From 553d89bd1293919585a14966b420c1af7455c52c Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 5 Apr 2023 00:12:01 -0700 Subject: [PATCH 001/155] add support for the new gwfailover bit --- common/const_var.go | 4 ++ http/supported_groups_handler_test.go | 2 + util/firmware_bitmap_test.go | 62 +++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 9 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index d1ad66d..e7e02a5 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -169,6 +169,9 @@ var ( 14: { {1, 24}, }, + 15: { + {1, 26}, + }, } ) @@ -199,6 +202,7 @@ var ( "wanfailover": 23, "cellularconfig": 24, "telcovoice": 25, + "gwfailover": 26, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index a8e5bc4..ccefd16 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -105,6 +105,7 @@ func TestSupportedGroupsHandler(t *testing.T) { assert.NilError(t, err) expectedEnabled["wanfailover"] = false expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) // ==== step 2 add lan data ==== @@ -285,5 +286,6 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) } diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 7ac8457..e8e55bf 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -22,7 +22,6 @@ import ( "strings" "testing" - "github.com/rdkcentral/webconfig/common" "gotest.tools/assert" ) @@ -158,6 +157,7 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { parsedSupportedMap := GetSupportedMap(cpeBitmap) expectedEnabled["wanfailover"] = false expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -203,6 +203,7 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { parsedSupportedMap := GetSupportedMap(cpeBitmap) expectedEnabled["wanfailover"] = false expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -248,6 +249,7 @@ func TestBitmapParsing(t *testing.T) { parsedSupportedMap := GetSupportedMap(cpeBitmap) expectedEnabled["wanfailover"] = false expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -293,6 +295,7 @@ func TestParseVoiceService(t *testing.T) { parsedSupportedMap := GetSupportedMap(cpeBitmap) expectedEnabled["wanfailover"] = false expectedEnabled["cellularconfig"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -306,14 +309,8 @@ func TestManualBitmap(t *testing.T) { } func TestParseSupportedDocsWithNewGroups(t *testing.T) { - // get the max metadata group_id - var maxMetadataGroupId int - for mgid := range common.SupportedDocsBitMaskMap { - if mgid > maxMetadataGroupId { - maxMetadataGroupId = mgid - } - } - xBitValue := (maxMetadataGroupId << 24) + 1 + cellularBitGroupId := 14 + xBitValue := (cellularBitGroupId << 24) + 1 ss := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809,234881025" rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", ss, xBitValue) @@ -354,6 +351,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { parsedSupportedMap := GetSupportedMap(cpeBitmap) expectedEnabled["wanfailover"] = false + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -397,6 +395,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { } parsedSupportedMap := GetSupportedMap(cpeBitmap) + expectedEnabled["gwfailover"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -439,6 +438,51 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { assert.Equal(t, parsedEnabled, enabled) } + parsedSupportedMap := GetSupportedMap(cpeBitmap) + expectedEnabled["gwfailover"] = false + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) +} + +func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241" + + // build expected + expectedEnabled := map[string]bool{ + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + parsedSupportedMap := GetSupportedMap(cpeBitmap) assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } From 149f7ff73b827e88287b87dbc289e118c050e1a1 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 20 May 2023 11:39:08 -0700 Subject: [PATCH 002/155] enable sarama client logger --- main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/main.go b/main.go index 8ac1ca3..343859e 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,7 @@ import ( "os/signal" "syscall" + "github.com/Shopify/sarama" "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" "github.com/rdkcentral/webconfig/common" @@ -90,6 +91,9 @@ func main() { } log.SetLevel(logLevel) + // setup sarama logger + sarama.Logger = log.StandardLogger() + // setup router router := server.GetRouter(false) From 8dd03a06dd989aabcb15a775e49972cd0a3e3025 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 13 Nov 2023 22:12:48 -0800 Subject: [PATCH 003/155] add support for parsing bitmap of subdoc clienttosteeringprofile --- common/const_var.go | 64 +++---- http/supported_groups_handler_test.go | 183 ++++++++++---------- util/firmware_bitmap_test.go | 240 ++++++++++++++++---------- 3 files changed, 276 insertions(+), 211 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index ddba5f8..1d7cf70 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -143,7 +143,8 @@ var ( {1, 12}, }, 6: { - {1, 13}, + {1, 13}, // mesh + {2, 31}, // clienttosteeringprofile }, 7: { {1, 14}, @@ -188,36 +189,37 @@ var ( var ( SubdocBitIndexMap = map[string]int{ - "portforwarding": 1, - "lan": 2, - "wan": 3, - "macbinding": 4, - "hotspot": 5, - "bridge": 6, - "privatessid": 7, - "homessid": 8, - "radio": 9, - "moca": 10, - "xdns": 11, - "advsecurity": 12, - "mesh": 13, - "aker": 14, - "telemetry": 15, - "statusreport": 16, - "trafficreport": 17, - "interfacereport": 18, - "radioreport": 19, - "telcovoip": 20, - "wanmanager": 21, - "voiceservice": 22, - "wanfailover": 23, - "cellularconfig": 24, - "telcovoice": 25, - "gwfailover": 26, - "gwrestore": 27, - "prioritizedmacs": 28, - "connectedbuilding": 29, - "lldqoscontrol": 30, + "portforwarding": 1, + "lan": 2, + "wan": 3, + "macbinding": 4, + "hotspot": 5, + "bridge": 6, + "privatessid": 7, + "homessid": 8, + "radio": 9, + "moca": 10, + "xdns": 11, + "advsecurity": 12, + "mesh": 13, + "aker": 14, + "telemetry": 15, + "statusreport": 16, + "trafficreport": 17, + "interfacereport": 18, + "radioreport": 19, + "telcovoip": 20, + "wanmanager": 21, + "voiceservice": 22, + "wanfailover": 23, + "cellularconfig": 24, + "telcovoice": 25, + "gwfailover": 26, + "gwrestore": 27, + "prioritizedmacs": 28, + "connectedbuilding": 29, + "lldqoscontrol": 30, + "clienttosteeringprofile": 31, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index fa5a641..80ae1b1 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -67,36 +67,37 @@ func TestSupportedGroupsHandler(t *testing.T) { // call GET /supported_groups expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, - "wanfailover": false, - "cellularconfig": false, - "gwfailover": false, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, + "portforwarding": true, + "lan": true, + "wan": true, + "macbinding": true, + "hotspot": false, + "bridge": false, + "privatessid": true, + "homessid": true, + "radio": false, + "moca": true, + "xdns": true, + "advsecurity": true, + "mesh": true, + "aker": true, + "telemetry": true, + "statusreport": false, + "trafficreport": false, + "interfacereport": false, + "radioreport": false, + "telcovoip": false, + "telcovoice": false, + "wanmanager": false, + "voiceservice": false, + "wanfailover": false, + "cellularconfig": false, + "gwfailover": false, + "gwrestore": false, + "prioritizedmacs": false, + "connectedbuilding": false, + "lldqoscontrol": false, + "clienttosteeringprofile": false, } // call GET /supported_groups to verify response @@ -251,36 +252,37 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { // call GET /supported_groups expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": false, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": true, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, - "gwfailover": false, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "homessid": false, + "hotspot": false, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": false, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoip": false, + "telcovoice": true, + "telemetry": true, + "trafficreport": false, + "voiceservice": false, + "wan": true, + "wanfailover": true, + "wanmanager": true, + "xdns": true, + "gwfailover": false, + "gwrestore": false, + "prioritizedmacs": false, + "connectedbuilding": false, + "lldqoscontrol": false, + "clienttosteeringprofile": false, } // call GET /supported_groups to verify response @@ -332,36 +334,37 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin // call GET /supported_groups expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwfailover": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoip": false, + "telcovoice": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwfailover": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": true, + "lldqoscontrol": true, + "clienttosteeringprofile": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 6cae866..4475183 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -162,6 +162,8 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -212,6 +214,7 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -262,6 +265,7 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -313,6 +317,7 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -374,6 +379,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -423,6 +429,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -472,6 +479,7 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["prioritizedmacs"] = false expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false + expectedEnabled["clienttosteeringprofile"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -481,36 +489,37 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { // build expected expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": false, + "connectedbuilding": false, + "lldqoscontrol": false, + "clienttosteeringprofile": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -529,36 +538,37 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { // build expected expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": false, - "lldqoscontrol": false, + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": false, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": false, + "lldqoscontrol": false, + "clienttosteeringprofile": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -577,36 +587,86 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test // build expected expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": true, + "lldqoscontrol": true, + "clienttosteeringprofile": false, + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + + parsedSupportedMap := GetSupportedMap(cpeBitmap) + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) +} + +func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + expectedEnabled := map[string]bool{ + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": true, + "lldqoscontrol": true, + "clienttosteeringprofile": true, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From f78ac32659f23d59c1aa392e64a1dd9f3d34925d Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 20 Dec 2023 23:21:53 -0800 Subject: [PATCH 004/155] add a flag to disable appending query params in supplementary GET calls --- config/sample_webconfig.conf | 2 + http/supplementary_handler.go | 18 ++-- http/supplementary_handler_test.go | 141 +++++++++++++++++++++++++++++ http/webconfig_server.go | 107 ++++++++++++---------- 4 files changed, 213 insertions(+), 55 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 2655e1f..f47c838 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -230,4 +230,6 @@ webconfig { // if valid_partners is empty, then all partners are accepted // if valid partners is not empty, all parters NOT in the list will be stored as "unknown" valid_partners = ["company", "vendor1", "vendor2"] + + supplementary_appending_enabled = true } diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 880fe57..c4488c5 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -51,15 +51,17 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r // append the extra query_params if any var queryParams string - rootdoc, err := s.GetRootDocument(mac) - if err != nil { - if !s.IsDbNotFound(err) { - Error(w, http.StatusInternalServerError, common.NewError(err)) - return + if s.SupplementaryAppendingEnabled() { + rootdoc, err := s.GetRootDocument(mac) + if err != nil { + if !s.IsDbNotFound(err) { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + } + if rootdoc != nil { + queryParams = rootdoc.QueryParams } - } - if rootdoc != nil { - queryParams = rootdoc.QueryParams } // partner handling diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index 8e1e0a3..ff08613 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -619,3 +619,144 @@ func TestSupplementaryApiBadPartnerHeader(t *testing.T) { assert.NilError(t, err) assert.DeepEqual(t, coreProfile1, srcItf) } + +func TestSupplementaryAppendingFlag(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== setup mock server ==== + var ss string + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ss = r.URL.String() + w.Header().Set(common.HeaderReqUrl, r.URL.String()) + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockProfileResponse)) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 set up the query params ==== + bitmap1 := 32479 + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + etag := strconv.Itoa(int(time.Now().Unix())) + queryParams1 := "stormReadyWifi=true" + srcDoc1 := common.NewRootDocument(bitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, etag, queryParams1) + bbytes, err := json.Marshal(srcDoc1) + assert.NilError(t, err) + + rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) + req, err := http.NewRequest("POST", rootdocUrl, bytes.NewReader(bbytes)) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 2 set append flag true ==== + appendEnabled := true + server.SetSupplementaryAppendingEnabled(appendEnabled) + assert.Assert(t, appendEnabled == server.SupplementaryAppendingEnabled()) + + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // TODO verify the resHeader + xReqUrl := res.Header.Get(common.HeaderReqUrl) + assert.Assert(t, strings.Contains(xReqUrl, queryParams1)) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok := mparts["telemetry"] + assert.Assert(t, ok) + + output := common.TR181Output{} + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes := []byte(output.Parameters[0].Value) + + var itf util.Dict + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok := itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok := profilesItf.([]interface{}) + assert.Assert(t, ok) + + profile1Itf := profilesJs[0] + + profile1, ok := profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok := profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok := coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + var srcItf map[string]interface{} + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 3 verify the query params ==== + assert.Assert(t, strings.Contains(ss, "&stormReadyWifi=true")) + + // okok := false + // assert.Assert(t, okok) + + // ==== step 4 set append flag false ==== + appendEnabled = false + server.SetSupplementaryAppendingEnabled(appendEnabled) + assert.Assert(t, appendEnabled == server.SupplementaryAppendingEnabled()) + + req, err = http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 5 verify the query params ==== + assert.Assert(t, !strings.Contains(ss, "&stormReadyWifi=true")) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 932b8bf..94eed50 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -48,15 +48,16 @@ const ( ) const ( - MetricsEnabledDefault = true - FactoryResetEnabledDefault = false - serverApiTokenAuthEnabledDefault = false - deviceApiTokenAuthEnabledDefault = true - tokenApiEnabledDefault = false - activeDriverDefault = "cassandra" - defaultJwksEnabled = false - defaultTraceparentParentID = "0000000000000001" - defaultTracestateVendorID = "webconfig" + MetricsEnabledDefault = true + FactoryResetEnabledDefault = false + serverApiTokenAuthEnabledDefault = false + deviceApiTokenAuthEnabledDefault = true + tokenApiEnabledDefault = false + activeDriverDefault = "cassandra" + defaultJwksEnabled = false + defaultTraceparentParentID = "0000000000000001" + defaultTracestateVendorID = "webconfig" + defaultSupplementaryAppendingEnabled = true ) var ( @@ -85,21 +86,22 @@ type WebconfigServer struct { *XconfConnector *MqttConnector *UpstreamConnector - tlsConfig *tls.Config - notLoggedHeaders []string - metricsEnabled bool - factoryResetEnabled bool - serverApiTokenAuthEnabled bool - deviceApiTokenAuthEnabled bool - tokenApiEnabled bool - kafkaEnabled bool - upstreamEnabled bool - appName string - validateMacEnabled bool - validPartners []string - jwksEnabled bool - traceparentParentID string - tracestateVendorID string + tlsConfig *tls.Config + notLoggedHeaders []string + metricsEnabled bool + factoryResetEnabled bool + serverApiTokenAuthEnabled bool + deviceApiTokenAuthEnabled bool + tokenApiEnabled bool + kafkaEnabled bool + upstreamEnabled bool + appName string + validateMacEnabled bool + validPartners []string + jwksEnabled bool + traceparentParentID string + tracestateVendorID string + supplementaryAppendingEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -243,35 +245,38 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) + supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) + return &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, WriteTimeout: time.Duration(conf.GetInt32("webconfig.server.write_timeout_in_secs", 3)) * time.Second, }, - DatabaseClient: dbclient, - TokenManager: security.NewTokenManager(conf), - JwksManager: jwksManager, - ServerConfig: sc, - WebpaConnector: NewWebpaConnector(conf, tlsConfig), - XconfConnector: NewXconfConnector(conf, tlsConfig), - MqttConnector: NewMqttConnector(conf, tlsConfig), - UpstreamConnector: NewUpstreamConnector(conf, tlsConfig), - tlsConfig: tlsConfig, - notLoggedHeaders: notLoggedHeaders, - metricsEnabled: metricsEnabled, - factoryResetEnabled: factoryResetEnabled, - serverApiTokenAuthEnabled: serverApiTokenAuthEnabled, - deviceApiTokenAuthEnabled: deviceApiTokenAuthEnabled, - tokenApiEnabled: tokenApiEnabled, - kafkaEnabled: kafkaEnabled, - upstreamEnabled: upstreamEnabled, - appName: appName, - validateMacEnabled: validateMacEnabled, - validPartners: validPartners, - jwksEnabled: jwksEnabled, - traceparentParentID: traceparentParentID, - tracestateVendorID: tracestateVendorID, + DatabaseClient: dbclient, + TokenManager: security.NewTokenManager(conf), + JwksManager: jwksManager, + ServerConfig: sc, + WebpaConnector: NewWebpaConnector(conf, tlsConfig), + XconfConnector: NewXconfConnector(conf, tlsConfig), + MqttConnector: NewMqttConnector(conf, tlsConfig), + UpstreamConnector: NewUpstreamConnector(conf, tlsConfig), + tlsConfig: tlsConfig, + notLoggedHeaders: notLoggedHeaders, + metricsEnabled: metricsEnabled, + factoryResetEnabled: factoryResetEnabled, + serverApiTokenAuthEnabled: serverApiTokenAuthEnabled, + deviceApiTokenAuthEnabled: deviceApiTokenAuthEnabled, + tokenApiEnabled: tokenApiEnabled, + kafkaEnabled: kafkaEnabled, + upstreamEnabled: upstreamEnabled, + appName: appName, + validateMacEnabled: validateMacEnabled, + validPartners: validPartners, + jwksEnabled: jwksEnabled, + traceparentParentID: traceparentParentID, + tracestateVendorID: tracestateVendorID, + supplementaryAppendingEnabled: supplementaryAppendingEnabled, } } @@ -556,6 +561,14 @@ func (s *WebconfigServer) SetTracestateVendorID(x string) { s.tracestateVendorID = x } +func (s *WebconfigServer) SupplementaryAppendingEnabled() bool { + return s.supplementaryAppendingEnabled +} + +func (s *WebconfigServer) SetSupplementaryAppendingEnabled(enabled bool) { + s.supplementaryAppendingEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { From 5d20f85c46c751712d19dc3e7d0430db39818caf Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 8 Jan 2024 11:24:41 -0800 Subject: [PATCH 005/155] Update gocql lib version --- README.md | 2 +- go.mod | 4 ++-- go.sum | 10 ++++++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c1ea263..c489fb1 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Webconfig supports 2 types of transport between cloud and devices 2. mqtt ## Install go -This project is written and tested with Go **1.17**. +This project is written and tested with Go **1.21**. ## Build the binary ```shell diff --git a/go.mod b/go.mod index 62ba922..a38b9a1 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ module github.com/rdkcentral/webconfig -go 1.19 +go 1.21 require ( github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/Shopify/sarama v1.38.1 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 - github.com/gocql/gocql v1.2.1 + github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/uuid v1.3.0 github.com/gorilla/mux v1.8.0 diff --git a/go.sum b/go.sum index ed18197..2174f5b 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4Oe github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= +github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -77,6 +78,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665/go.mod h1:19bUnum2ZAeftfwwLZ/wRe7idyfoW2MfmXO464Hrfbw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -91,8 +93,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gocql/gocql v1.2.1 h1:G/STxUzD6pGvRHzG0Fi7S04SXejMKBbRZb7pwre1edU= -github.com/gocql/gocql v1.2.1/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= +github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= +github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= @@ -200,6 +202,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -224,6 +227,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -270,6 +274,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -566,6 +571,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= From 274072822e700c64431846e45a840d25bdde52a4 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 22 Jan 2024 21:48:42 -0800 Subject: [PATCH 006/155] add factory reset NONE/NONE-REBOOT handling --- common/const_var.go | 1 + common/server_config.go | 3 +- db/service.go | 36 +++- db/service_test.go | 28 +++ go.mod | 20 +- go.sum | 62 ++++--- http/document_handler_test.go | 66 +++---- http/factory_reset_upstream_test.go | 272 ++++++++++++++++++++++++++++ http/http_client.go | 3 +- http/multipart.go | 120 +++++++++++- http/multipart_test.go | 86 ++++----- http/poke_handler_test.go | 30 +-- http/rootdocument_handler_test.go | 18 +- http/simple_handler_test.go | 10 +- http/validator_test.go | 44 ++--- http/webconfig_server.go | 9 +- kafka/consumer.go | 4 +- kafka/consumer_group_test.go | 2 +- kafka/kafka_consumer_group.go | 2 +- main.go | 2 +- security/token.go | 6 +- 21 files changed, 641 insertions(+), 183 deletions(-) create mode 100644 http/factory_reset_upstream_test.go diff --git a/common/const_var.go b/common/const_var.go index 1d7cf70..4370c7e 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -105,6 +105,7 @@ const ( HeaderSourceAppName = "X-Source-App-Name" HeaderTraceparent = "Traceparent" HeaderTracestate = "Tracestate" + HeaderContentLength = "Content-Length" ) // header X-System-Supported-Docs diff --git a/common/server_config.go b/common/server_config.go index b37da7f..44fa5a7 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -19,7 +19,6 @@ package common import ( "fmt" - "io/ioutil" "os" "github.com/go-akka/configuration" @@ -40,7 +39,7 @@ type ServerConfig struct { } func NewServerConfig(configFile string) (*ServerConfig, error) { - configBytes, err := ioutil.ReadFile(configFile) + configBytes, err := os.ReadFile(configFile) if err != nil { return nil, NewError(err) } diff --git a/db/service.go b/db/service.go index 3da868c..4c2a598 100644 --- a/db/service.go +++ b/db/service.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "net/http" + "sort" "strings" "time" @@ -247,8 +248,13 @@ func HashRootVersion(itf interface{}) string { // if len(mparts) == 0, then the murmur hash value is 0 buffer := bytes.NewBufferString("") - for _, version := range versionMap { - buffer.WriteString(version) + keys := []string{} + for k := range versionMap { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + buffer.WriteString(versionMap[k]) } return util.GetMurmur3Hash(buffer.Bytes()) } @@ -386,7 +392,7 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old return nil } -func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string, newDoc *common.Document, d *common.Document, fields log.Fields) error { +func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string, newDoc *common.Document, d *common.Document, toDelete bool, fields log.Fields) error { newRootVersion := upstreamRespEtag if d.RootVersion() != newRootVersion { err := c.SetRootDocumentVersion(cpeMac, newRootVersion) @@ -395,6 +401,11 @@ func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string } } + oldMap := map[string]struct{}{} + for k := range d.Items() { + oldMap[k] = struct{}{} + } + // need to set "state" to proper values like the download is complete for subdocId, newSubdoc := range newDoc.Items() { oldSubdoc := d.SubDocument(subdocId) @@ -402,6 +413,16 @@ func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string if err != nil { return common.NewError(err) } + delete(oldMap, subdocId) + } + + if toDelete { + for subdocId := range oldMap { + err := c.DeleteSubDocument(cpeMac, subdocId) + if err != nil { + return common.NewError(err) + } + } } return nil } @@ -450,3 +471,12 @@ func RebuildDeviceVersionMap(versions []string, cloudVersionMap map[string]strin } return m } + +func RefreshRootDocumentVersion(doc *common.Document) { + versionMap := doc.VersionMap() + rootVersion := HashRootVersion(versionMap) + rootDoc := doc.GetRootDocument() + if rootDoc != nil { + rootDoc.Version = rootVersion + } +} diff --git a/db/service_test.go b/db/service_test.go index 1afe700..c0226b0 100644 --- a/db/service_test.go +++ b/db/service_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -47,3 +48,30 @@ func TestHashRootVersion(t *testing.T) { root = HashRootVersion(doc.VersionMap()) assert.Assert(t, root != "0") } + +func TestUpdateRootVersion(t *testing.T) { + doc := common.NewDocument(nil) + tt := int(123) + + bbytes1 := util.RandomBytes(100, 150) + v1 := util.GetMurmur3Hash(bbytes1) + subdoc1 := common.NewSubDocument(bbytes1, &v1, nil, &tt, nil, nil) + doc.SetSubDocument("advsecurity", subdoc1) + + bbytes2 := util.RandomBytes(100, 150) + v2 := util.GetMurmur3Hash(bbytes2) + subdoc2 := common.NewSubDocument(bbytes2, &v2, nil, &tt, nil, nil) + doc.SetSubDocument("mesh", subdoc2) + + rootVersion := HashRootVersion(doc.VersionMap()) + assert.Assert(t, rootVersion != "0") + rootDoc := common.NewRootDocument(123, "fw_ver_123", "model_123", "partner_123", "", rootVersion, "") + doc.SetRootDocument(rootDoc) + + doc.DeleteSubDocument("mesh") + + RefreshRootDocumentVersion(doc) + + newRootVersion := doc.RootVersion() + assert.Assert(t, rootVersion != newRootVersion) +} diff --git a/go.mod b/go.mod index a38b9a1..bbdcee4 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/rdkcentral/webconfig go 1.21 require ( + github.com/IBM/sarama v1.42.1 github.com/MicahParks/keyfunc/v2 v2.1.0 - github.com/Shopify/sarama v1.38.1 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt/v5 v5.0.0 @@ -19,7 +19,7 @@ require ( github.com/vmihailenco/msgpack/v4 v4.3.12 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.1.0 + golang.org/x/sync v0.4.0 gotest.tools v2.2.0+incompatible ) @@ -28,8 +28,8 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.3.0 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect + github.com/eapache/go-resiliency v1.4.0 // indirect + github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -41,21 +41,21 @@ require ( github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect - github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.15.14 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect - golang.org/x/net v0.5.0 // indirect - golang.org/x/sys v0.4.0 // indirect + golang.org/x/crypto v0.17.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.15.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 2174f5b..f502ae6 100644 --- a/go.sum +++ b/go.sum @@ -33,12 +33,10 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= +github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= -github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= -github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= -github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= -github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -67,10 +65,10 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= -github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= -github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM= -github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= +github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= +github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= +github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -181,8 +179,8 @@ github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVET github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= -github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -195,8 +193,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= -github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -218,8 +216,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= -github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -266,6 +264,7 @@ github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -273,8 +272,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -287,6 +287,7 @@ github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgq github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -304,8 +305,10 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -336,6 +339,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -364,13 +368,15 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= -golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -387,8 +393,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -427,12 +434,15 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -440,6 +450,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -483,6 +494,7 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/http/document_handler_test.go b/http/document_handler_test.go index 5b87fcf..f879914 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -21,7 +21,7 @@ import ( "bytes" "encoding/hex" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "strings" @@ -51,7 +51,7 @@ func TestSubDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -61,7 +61,7 @@ func TestSubDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -76,7 +76,7 @@ func TestSubDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -85,7 +85,7 @@ func TestSubDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -110,7 +110,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -120,7 +120,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -140,7 +140,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -149,7 +149,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -167,7 +167,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req, err = http.NewRequest("DELETE", url, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -176,7 +176,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -185,7 +185,7 @@ func TestDeleteDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -217,7 +217,7 @@ func TestPostWithDeviceId(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -229,7 +229,7 @@ func TestPostWithDeviceId(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -249,7 +249,7 @@ func TestPostWithDeviceId(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -260,7 +260,7 @@ func TestPostWithDeviceId(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -292,7 +292,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -304,7 +304,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) assert.Equal(t, reqHeaderVersion, resHeaderVersion) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, gwrestoreBytes) @@ -333,7 +333,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -348,7 +348,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, remotedebuggerBytes) @@ -367,7 +367,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -377,7 +377,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -392,7 +392,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -426,7 +426,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { ifnonematch := strings.Join(subdocVersions, ",") req.Header.Set(common.HeaderIfNoneMatch, ifnonematch) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -452,7 +452,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -464,7 +464,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) assert.Equal(t, reqHeaderVersion, resHeaderVersion) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, gwrestoreBytes) @@ -493,7 +493,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -508,7 +508,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { resHeaderExpiry := res.Header.Get(common.HeaderSubdocumentExpiry) assert.Equal(t, reqHeaderExpiry, resHeaderExpiry) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, remotedebuggerBytes) @@ -527,7 +527,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -537,7 +537,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -552,7 +552,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -580,7 +580,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { ifnonematch := strings.Join(subdocVersions, ",") req.Header.Set(common.HeaderIfNoneMatch, ifnonematch) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -609,7 +609,7 @@ func TestBadHeaderExpiryHandler(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) } diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go new file mode 100644 index 0000000..03ce19c --- /dev/null +++ b/http/factory_reset_upstream_test.go @@ -0,0 +1,272 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/rdkcentral/webconfig/util" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestFactoryResetWithoutUpstream(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== GET /config ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== GET /config but with header changes without mock ==== + server.SetUpstreamEnabled(false) + + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // verify Document is now empty + fields := make(log.Fields) + _, err = server.GetDocument(cpeMac, fields) + assert.Assert(t, server.IsDbNotFound(err)) +} + +func TestFactoryResetWithUpstream(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== GET /config ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + // lanVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== setup mock upstream server ==== + fields := make(log.Fields) + mockDoc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + + mockRootDoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + mockDoc.SetRootDocument(mockRootDoc) + + mockDoc.DeleteSubDocument("wan") + assert.NilError(t, err) + + mockBytes, err := mockDoc.Bytes() + assert.NilError(t, err) + + db.RefreshRootDocumentVersion(mockDoc) + refreshedRootVersion := mockDoc.RootVersion() + + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + w.Header().Set(k, r.Header.Get(k)) + } + w.Header().Set(common.HeaderContentLength, strconv.Itoa(len(mockBytes))) + ifNoneMatch := refreshedRootVersion + w.Header().Set(common.HeaderEtag, ifNoneMatch) + w.WriteHeader(http.StatusOK) + w.Write(mockBytes) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + // ==== GET /config but with header changes without mock ==== + server.SetUpstreamEnabled(true) + + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + mpart, ok = mparts["wan"] + assert.Assert(t, !ok) + + // verify Document is now empty + doc, err := server.GetDocument(cpeMac, fields) + assert.NilError(t, err) + assert.Equal(t, doc.Length(), 1) +} diff --git a/http/http_client.go b/http/http_client.go index 833c333..dfc0459 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -25,7 +25,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net" "net/http" neturl "net/url" @@ -260,7 +259,7 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] } fields[fmt.Sprintf("%v_status", loggerName)] = res.StatusCode - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) if err != nil { fields[errorKey] = err.Error() if userAgent != "mget" { diff --git a/http/multipart.go b/http/multipart.go index 6c1c663..81b42bf 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -116,6 +116,16 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin respHeader := make(http.Header) userAgent := rHeader.Get("User-Agent") + // factory reset handling + ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) + if ifNoneMatch == "NONE" || ifNoneMatch == "NONE-REBOOT" { + status, respHeader, rbytes, err := BuildFactoryResetResponse(s, rHeader, fields) + if err != nil { + return status, respHeader, rbytes, common.NewError(err) + } + return status, respHeader, rbytes, nil + } + document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, err := db.BuildGetDocument(c, rHeader, route, fields) if uconn == nil { if err != nil { @@ -263,7 +273,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalFilteredDocument, document, fields) + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalFilteredDocument, document, false, fields) if err != nil { return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) } @@ -280,3 +290,111 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusOK, upstreamRespHeader, finalFilteredBytes, nil } + +func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { + c := s.DatabaseClient + uconn := s.GetUpstreamConnector() + mac := rHeader.Get(common.HeaderDeviceId) + respHeader := make(http.Header) + + document, err := c.GetDocument(mac, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + rootDocument, err := c.GetRootDocument(mac) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + document.SetRootDocument(rootDocument) + oldDocBytes, err := document.Bytes() + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + + if uconn == nil { + err := c.DeleteDocument(mac) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + return http.StatusNotFound, respHeader, nil, nil + } + + // handle partner + partnerId := rHeader.Get(common.HeaderPartnerID) + if len(partnerId) == 0 && rootDocument != nil { + partnerId = rootDocument.PartnerId + } + + // ============================= + // upstream handling + // ============================= + upstreamHeader := make(http.Header) + upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) + upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) + if itf, ok := fields["audit_id"]; ok { + auditId := itf.(string) + if len(auditId) > 0 { + upstreamHeader.Set(common.HeaderAuditid, auditId) + } + } + if x := rHeader.Get(common.HeaderTransactionId); len(x) > 0 { + upstreamHeader.Set(common.HeaderTransactionId, x) + } + + if s.TokenManager != nil { + token := rHeader.Get("Authorization") + if len(token) > 0 { + upstreamHeader.Set("Authorization", token) + } else { + token = s.Generate(mac, 86400) + upstreamHeader.Set("Authorization", "Bearer "+token) + } + } + + // call /upstream to handle factory reset + ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) + upstreamHeader.Set(common.HeaderIfNoneMatch, ifNoneMatch) + + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) + if err != nil { + var rherr common.RemoteHttpError + if errors.As(err, &rherr) { + return rherr.StatusCode, respHeader, nil, common.NewError(err) + } + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + + // ==== parse the upstreamRespBytes and store them ==== + finalMparts, err := util.ParseMultipartAsList(upstreamRespHeader, upstreamRespBytes) + if err != nil { + return http.StatusInternalServerError, respHeader, oldDocBytes, common.NewError(err) + } + upstreamRespEtag := upstreamRespHeader.Get(common.HeaderEtag) + + // filter by versionMap and filter by blockedIds + finalRootDocument := common.NewRootDocument(0, "", "", "", "", upstreamRespEtag, "") + finalDocument := common.NewDocument(finalRootDocument) + finalDocument.SetSubDocuments(finalMparts) + for _, subdocId := range c.BlockedSubdocIds() { + finalDocument.DeleteSubDocument(subdocId) + } + + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } + + // 304 + if finalDocument.Length() == 0 { + return http.StatusNotModified, upstreamRespHeader, nil, nil + } + + finalBytes, err := finalDocument.Bytes() + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, finalBytes, common.NewError(err) + } + + return http.StatusOK, upstreamRespHeader, finalBytes, nil +} diff --git a/http/multipart_test.go b/http/multipart_test.go index e4ebd4c..0dc6be5 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -21,7 +21,7 @@ import ( "bytes" "encoding/hex" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "strings" @@ -52,7 +52,7 @@ func TestMultipartConfigHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -62,7 +62,7 @@ func TestMultipartConfigHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -81,7 +81,7 @@ func TestMultipartConfigHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -91,7 +91,7 @@ func TestMultipartConfigHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -102,7 +102,7 @@ func TestMultipartConfigHandler(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -139,7 +139,7 @@ func TestMultipartConfigHandler(t *testing.T) { assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, etag) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -151,7 +151,7 @@ func TestMultipartConfigHandler(t *testing.T) { ifNoneMatch := fmt.Sprintf("foo,%v,bar", lanMpartVersion) req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -196,7 +196,7 @@ func TestCpeMiddleware(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -206,7 +206,7 @@ func TestCpeMiddleware(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -220,7 +220,7 @@ func TestCpeMiddleware(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router1).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusForbidden) @@ -231,7 +231,7 @@ func TestCpeMiddleware(t *testing.T) { assert.NilError(t, err) req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", token)) res = ExecuteRequest(req, router1).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -253,7 +253,7 @@ func TestVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -263,7 +263,7 @@ func TestVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -279,7 +279,7 @@ func TestVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -289,7 +289,7 @@ func TestVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -300,7 +300,7 @@ func TestVersionFiltering(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -325,7 +325,7 @@ func TestVersionFiltering(t *testing.T) { assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, etag) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -337,7 +337,7 @@ func TestVersionFiltering(t *testing.T) { ifNoneMatch := fmt.Sprintf("foo,%v,bar", lanMpartVersion) req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -397,7 +397,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -407,7 +407,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -423,7 +423,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -433,7 +433,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -444,7 +444,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -472,7 +472,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusServiceUnavailable) @@ -481,7 +481,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusServiceUnavailable) @@ -494,7 +494,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { w.Header().Set(k, r.Header.Get(k)) } w.WriteHeader(http.StatusOK) - if rbytes, err := ioutil.ReadAll(r.Body); err == nil { + if rbytes, err := io.ReadAll(r.Body); err == nil { _, err := w.Write(rbytes) assert.NilError(t, err) } @@ -509,7 +509,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -531,7 +531,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderIfNoneMatch, matchedIfNoneMatch) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) @@ -542,7 +542,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { req.Header.Set(common.HeaderIfNoneMatch, mismatchedIfNoneMatch) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -572,7 +572,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -582,7 +582,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -598,7 +598,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -608,7 +608,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -669,7 +669,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { w.Header().Set(k, r.Header.Get(k)) } w.WriteHeader(http.StatusOK) - if rbytes, err := ioutil.ReadAll(r.Body); err == nil { + if rbytes, err := io.ReadAll(r.Body); err == nil { _, err := w.Write(rbytes) assert.NilError(t, err) } @@ -749,7 +749,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -759,7 +759,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -776,7 +776,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -786,7 +786,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -797,7 +797,7 @@ func TestMultipartConfigMismatch(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -826,7 +826,7 @@ func TestMultipartConfigMismatch(t *testing.T) { header1 := "NONE," + lanMpartVersion req.Header.Set(common.HeaderIfNoneMatch, header1) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -837,7 +837,7 @@ func TestMultipartConfigMismatch(t *testing.T) { header2 := "NONE,123" req.Header.Set(common.HeaderIfNoneMatch, header2) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) @@ -848,7 +848,7 @@ func TestMultipartConfigMismatch(t *testing.T) { header3 := etag + ",123" req.Header.Set(common.HeaderIfNoneMatch, header3) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index bbd0884..29c3364 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -20,7 +20,7 @@ package http import ( "bytes" "fmt" - "io/ioutil" + "io" "net/http" "net/http/httptest" "strconv" @@ -62,7 +62,7 @@ func TestPokeHandler(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusNoContent) - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -89,7 +89,7 @@ func TestPokeHandlerWithCpe(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() } @@ -121,7 +121,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -130,7 +130,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -153,7 +153,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -162,7 +162,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -189,7 +189,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req, err = http.NewRequest("POST", mqttUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusAccepted) @@ -198,7 +198,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) state, err = strconv.Atoi(res.Header.Get("X-Subdocument-State")) @@ -210,7 +210,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) state, err = strconv.Atoi(res.Header.Get("X-Subdocument-State")) @@ -229,7 +229,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -238,7 +238,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes2) @@ -255,7 +255,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req, err = http.NewRequest("POST", mqttUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusAccepted) @@ -264,7 +264,7 @@ func TestBuildMqttSendDocument(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) state, err = strconv.Atoi(res.Header.Get("X-Subdocument-State")) @@ -301,7 +301,7 @@ func TestPokeHandlerInvalidMac(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusBadRequest) - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() } diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 5debc71..7bb66c6 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -22,7 +22,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strconv" "testing" @@ -51,7 +51,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/json") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - _, err = ioutil.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -97,7 +97,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -106,7 +106,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -124,7 +124,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -133,7 +133,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -147,7 +147,7 @@ func TestRootDocumentHandler(t *testing.T) { req.Header.Set(common.HeaderPartnerID, partner1) req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.Equal(t, res.StatusCode, http.StatusOK) assert.NilError(t, err) res.Body.Close() @@ -182,7 +182,7 @@ func TestRootDocumentHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -223,7 +223,7 @@ func TestPostRootDocumentHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() diff --git a/http/simple_handler_test.go b/http/simple_handler_test.go index 0e4e093..28109fa 100644 --- a/http/simple_handler_test.go +++ b/http/simple_handler_test.go @@ -18,7 +18,7 @@ package http import ( - "io/ioutil" + "io" "net/http" "testing" @@ -36,7 +36,7 @@ func TestSimpleHandler(t *testing.T) { assert.NilError(t, err) res := ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, 200) - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -45,7 +45,7 @@ func TestSimpleHandler(t *testing.T) { assert.NilError(t, err) res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, 200) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, len(rbytes), 0) @@ -56,7 +56,7 @@ func TestSimpleHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, 200) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, len(rbytes), 0) @@ -70,7 +70,7 @@ func TestNotifHandler(t *testing.T) { req, err := http.NewRequest("GET", "/notif", nil) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) _ = rbytes assert.NilError(t, err) res.Body.Close() diff --git a/http/validator_test.go b/http/validator_test.go index f368804..8d36dcf 100644 --- a/http/validator_test.go +++ b/http/validator_test.go @@ -22,7 +22,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "strings" "testing" @@ -49,7 +49,7 @@ func TestValidatorDisabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -58,7 +58,7 @@ func TestValidatorDisabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -75,7 +75,7 @@ func TestValidatorDisabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -89,7 +89,7 @@ func TestValidatorDisabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -98,7 +98,7 @@ func TestValidatorDisabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -113,7 +113,7 @@ func TestValidatorDisabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -142,7 +142,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) @@ -152,7 +152,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -162,7 +162,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) @@ -172,7 +172,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -188,7 +188,7 @@ func TestValidatorEnabled(t *testing.T) { req, err = http.NewRequest("GET", rootdocUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) @@ -199,7 +199,7 @@ func TestValidatorEnabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -214,7 +214,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusBadRequest) @@ -224,7 +224,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -233,7 +233,7 @@ func TestValidatorEnabled(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) @@ -248,7 +248,7 @@ func TestValidatorEnabled(t *testing.T) { res = ExecuteRequest(req, router).Result() assert.Equal(t, res.StatusCode, http.StatusOK) - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() @@ -275,7 +275,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res := ExecuteRequest(req, router).Result() - rbytes, err := ioutil.ReadAll(res.Body) + rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -284,7 +284,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, lanBytes) @@ -304,7 +304,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) @@ -313,7 +313,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req.Header.Set("Content-Type", "application/msgpack") assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusOK) assert.DeepEqual(t, rbytes, wanBytes) @@ -331,7 +331,7 @@ func TestValidatorWithLowerCase(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() - rbytes, err = ioutil.ReadAll(res.Body) + rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 94eed50..0c2d146 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -22,14 +22,13 @@ import ( "crypto/tls" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "os" "strings" "time" "github.com/rdkcentral/webconfig/common" - owcommon "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/db/sqlite" @@ -301,7 +300,7 @@ func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { if r.Method == "POST" { if r.Body != nil { - if rbytes, err := ioutil.ReadAll(r.Body); err == nil { + if rbytes, err := io.ReadAll(r.Body); err == nil { xw.SetBodyBytes(rbytes) } } @@ -625,7 +624,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } // extract traceparent from the header - traceparent := r.Header.Get(owcommon.HeaderTraceparent) + traceparent := r.Header.Get(common.HeaderTraceparent) if len(traceparent) == 55 { traceId = traceparent[3:35] outTraceparent = traceparent[:36] + s.TraceparentParentID() + traceparent[52:55] @@ -698,7 +697,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if r.Method == "POST" { if r.Body != nil { - bbytes, err := ioutil.ReadAll(r.Body) + bbytes, err := io.ReadAll(r.Body) if err != nil { fields["error"] = err log.WithFields(fields).Error("request starts") diff --git a/kafka/consumer.go b/kafka/consumer.go index b97268c..1135d5d 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" wchttp "github.com/rdkcentral/webconfig/http" @@ -170,7 +170,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram // NOTE: // Do not move the code below to a goroutine. // The `ConsumeClaim` itself is called within a goroutine, see: - // https://github.com/Shopify/sarama/blob/master/consumer_group.go#L27-L29 + // https://github.com/IBM/sarama/blob/master/consumer_group.go#L27-L29 rl := ratelimit.New(c.ratelimitMessagesPerSecond, ratelimit.WithoutSlack) // per second, no slack. for { diff --git a/kafka/consumer_group_test.go b/kafka/consumer_group_test.go index 5db165f..d720eb2 100644 --- a/kafka/consumer_group_test.go +++ b/kafka/consumer_group_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "gotest.tools/assert" ) diff --git a/kafka/kafka_consumer_group.go b/kafka/kafka_consumer_group.go index 298a3a2..c768af7 100644 --- a/kafka/kafka_consumer_group.go +++ b/kafka/kafka_consumer_group.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" wchttp "github.com/rdkcentral/webconfig/http" diff --git a/main.go b/main.go index 4c32552..11c8893 100644 --- a/main.go +++ b/main.go @@ -25,7 +25,7 @@ import ( "os/signal" "syscall" - "github.com/Shopify/sarama" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" wchttp "github.com/rdkcentral/webconfig/http" "github.com/rdkcentral/webconfig/kafka" diff --git a/security/token.go b/security/token.go index 52592e0..51695e2 100644 --- a/security/token.go +++ b/security/token.go @@ -22,7 +22,7 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io/ioutil" + "os" "strings" "time" @@ -110,7 +110,7 @@ func NewTokenManager(conf *configuration.Config) *TokenManager { } func loadDecodeKey(keyfile string) (*rsa.PublicKey, error) { - kbytes, err := ioutil.ReadFile(keyfile) + kbytes, err := os.ReadFile(keyfile) if err != nil { return nil, common.NewError(err) } @@ -122,7 +122,7 @@ func loadDecodeKey(keyfile string) (*rsa.PublicKey, error) { } func loadEncodeKey(keyfile string) (*rsa.PrivateKey, error) { - kbytes, err := ioutil.ReadFile(keyfile) + kbytes, err := os.ReadFile(keyfile) if err != nil { return nil, common.NewError(err) } From 521a46df5040dacde541d7f064caa97c8a7f7917 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 24 Jan 2024 22:27:00 -0800 Subject: [PATCH 007/155] fix a bug that factory reset NONE returns 500 when no data in db --- http/factory_reset_upstream_test.go | 19 +++++++++++++++++++ http/multipart.go | 3 +++ 2 files changed, 22 insertions(+) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 03ce19c..3f587a8 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -33,6 +33,25 @@ import ( "gotest.tools/assert" ) +func TestFactoryResetWithoutData(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetUpstreamEnabled(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== no data ==== + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} + func TestFactoryResetWithoutUpstream(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) diff --git a/http/multipart.go b/http/multipart.go index 81b42bf..287d5aa 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -299,6 +299,9 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { + if s.IsDbNotFound(err) { + return http.StatusNotFound, respHeader, nil, nil + } return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } rootDocument, err := c.GetRootDocument(mac) From b5a355481fa2d93cd1a34fc635ced632d8a12de2 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 7 Feb 2024 23:12:42 -0800 Subject: [PATCH 008/155] Fix a bug that 304 was returned when all subdocs were wiped out on factory reset header none/none-reboot --- http/multipart.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index 287d5aa..6117b85 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -389,9 +389,8 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) } - // 304 if finalDocument.Length() == 0 { - return http.StatusNotModified, upstreamRespHeader, nil, nil + return http.StatusOK, upstreamRespHeader, nil, nil } finalBytes, err := finalDocument.Bytes() From 898df13dbd4dbddbd80ea0531de295c349b6d9b6 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 8 Feb 2024 16:27:12 -0800 Subject: [PATCH 009/155] return 200 empty on if-none-match:none and no data in db --- http/factory_reset_upstream_test.go | 2 +- http/multipart.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 3f587a8..1d0e7bf 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -49,7 +49,7 @@ func TestFactoryResetWithoutData(t *testing.T) { _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() - assert.Equal(t, res.StatusCode, http.StatusNotFound) + assert.Equal(t, res.StatusCode, http.StatusOK) } func TestFactoryResetWithoutUpstream(t *testing.T) { diff --git a/http/multipart.go b/http/multipart.go index 6117b85..595c0ab 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -300,7 +300,8 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { if s.IsDbNotFound(err) { - return http.StatusNotFound, respHeader, nil, nil + // return http.StatusNotFound, respHeader, nil, nil + return http.StatusOK, respHeader, nil, nil } return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } From 3d44fe21d5d352dcefcf48be2c3080a090aac30b Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 8 Feb 2024 23:31:56 -0800 Subject: [PATCH 010/155] fix a bug that root_document was not updated when if-none-match:NONE-REBOOT --- db/service.go | 39 +++++++++++++++++++++++++++++ http/factory_reset_upstream_test.go | 16 ++++++++++++ http/multipart.go | 23 +++++++++-------- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/db/service.go b/db/service.go index 4c2a598..582c94e 100644 --- a/db/service.go +++ b/db/service.go @@ -480,3 +480,42 @@ func RefreshRootDocumentVersion(doc *common.Document) { rootDoc.Version = rootVersion } } + +func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerId string, fields log.Fields) (*common.RootDocument, error) { + fieldsDict := make(util.Dict) + fieldsDict.Update(fields) + + // ==== deviceRootDocument should always be created from request header ==== + var bitmap int + var err error + supportedDocs := rHeader.Get(common.HeaderSupportedDocs) + if len(supportedDocs) > 0 { + bitmap, err = util.GetCpeBitmap(supportedDocs) + if err != nil { + log.WithFields(fields).Warn(common.NewError(err)) + } + } + + schemaVersion := strings.ToLower(rHeader.Get(common.HeaderSchemaVersion)) + modelName := rHeader.Get(common.HeaderModelName) + firmwareVersion := rHeader.Get(common.HeaderFirmwareVersion) + + // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers + deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "") + + // ==== read the cloudRootDocument from db ==== + cloudRootDocument, err := c.GetRootDocument(mac) + if err != nil { + if !c.IsDbNotFound(err) { + return cloudRootDocument, common.NewError(err) + } + cloudRootDocument = deviceRootDocument.Clone() + } else { + cloudRootDocument.Update(deviceRootDocument) + } + + if err := c.SetRootDocument(mac, cloudRootDocument); err != nil { + return cloudRootDocument, common.NewError(err) + } + return cloudRootDocument, nil +} diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 1d0e7bf..429814a 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -41,15 +41,31 @@ func TestFactoryResetWithoutData(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() // ==== no data ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err := http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "NONE") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) + + rootDocument, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + + assert.Equal(t, rootDocument.Bitmap, 32479) } func TestFactoryResetWithoutUpstream(t *testing.T) { diff --git a/http/multipart.go b/http/multipart.go index 595c0ab..fefc94d 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -297,6 +297,18 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l mac := rHeader.Get(common.HeaderDeviceId) respHeader := make(http.Header) + fieldsDict := make(util.Dict) + fieldsDict.Update(fields) + partnerId := rHeader.Get(common.HeaderPartnerID) + if len(partnerId) == 0 { + partnerId = fieldsDict.GetString("partner") + } + + rootDocument, err := db.PreprocessRootDocument(c, rHeader, mac, partnerId, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + document, err := c.GetDocument(mac, fields) if err != nil { if s.IsDbNotFound(err) { @@ -305,10 +317,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - rootDocument, err := c.GetRootDocument(mac) - if err != nil { - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) - } + document.SetRootDocument(rootDocument) oldDocBytes, err := document.Bytes() if err != nil { @@ -323,12 +332,6 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusNotFound, respHeader, nil, nil } - // handle partner - partnerId := rHeader.Get(common.HeaderPartnerID) - if len(partnerId) == 0 && rootDocument != nil { - partnerId = rootDocument.PartnerId - } - // ============================= // upstream handling // ============================= From a9933b51889f698ef1cf35a15fc769c46b88ee68 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 12 Feb 2024 12:38:55 -0800 Subject: [PATCH 011/155] change the none-reboot no subdoc response from 200 to 404 --- http/factory_reset_upstream_test.go | 2 +- http/multipart.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 429814a..e697661 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -60,7 +60,7 @@ func TestFactoryResetWithoutData(t *testing.T) { _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() - assert.Equal(t, res.StatusCode, http.StatusOK) + assert.Equal(t, res.StatusCode, http.StatusNotFound) rootDocument, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) diff --git a/http/multipart.go b/http/multipart.go index fefc94d..154beb6 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -312,8 +312,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { if s.IsDbNotFound(err) { - // return http.StatusNotFound, respHeader, nil, nil - return http.StatusOK, respHeader, nil, nil + return http.StatusNotFound, respHeader, nil, nil } return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } @@ -394,7 +393,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } if finalDocument.Length() == 0 { - return http.StatusOK, upstreamRespHeader, nil, nil + return http.StatusNotFound, upstreamRespHeader, nil, nil } finalBytes, err := finalDocument.Bytes() From aa019fb9fde387f1f8a9f7d3b6ba3202c08aa4c8 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 15 Feb 2024 13:47:55 -0800 Subject: [PATCH 012/155] always call upstream on factory reset none-reboot --- common/document.go | 4 ++ http/factory_reset_upstream_test.go | 85 ++++++++++++++++++++++++++++- http/multipart.go | 11 ++-- 3 files changed, 95 insertions(+), 5 deletions(-) diff --git a/common/document.go b/common/document.go index f400945..61b054c 100644 --- a/common/document.go +++ b/common/document.go @@ -148,6 +148,10 @@ func (d *Document) FilterForGet(versionMap map[string]string) *Document { } func (d *Document) Bytes() ([]byte, error) { + if len(d.docmap) == 0 { + return nil, nil + } + // build the http stream mparts := []Multipart{} for subdocId, subdoc := range d.docmap { diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index e697661..f6bbe0c 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -25,6 +25,7 @@ import ( "net/http/httptest" "strconv" "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" @@ -40,6 +41,25 @@ func TestFactoryResetWithoutData(t *testing.T) { cpeMac := util.GenerateRandomCpeMac() + // ==== setup upstream mock server ==== + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // build the response + for k := range r.Header { + if k == common.HeaderContentLength { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + w.WriteHeader(http.StatusNotFound) + w.Write(nil) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + // ==== no data ==== supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" @@ -64,7 +84,6 @@ func TestFactoryResetWithoutData(t *testing.T) { rootDocument, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - assert.Equal(t, rootDocument.Bitmap, 32479) } @@ -305,3 +324,67 @@ func TestFactoryResetWithUpstream(t *testing.T) { assert.NilError(t, err) assert.Equal(t, doc.Length(), 1) } + +func TestFactoryResetUpstreamAddData(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetUpstreamEnabled(true) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== setup upstream mock server ==== + upstreamMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for k := range r.Header { + if k == common.HeaderContentLength { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + + // add a subdoc from upsream + mparts := []common.Multipart{ + { + Bytes: util.RandomBytes(100, 150), + Version: strconv.Itoa(int(time.Now().Unix())), + Name: "network", + State: common.PendingDownload, + }, + } + respBytes, err := common.WriteMultipartBytes(mparts) + assert.NilError(t, err) + w.WriteHeader(http.StatusOK) + w.Write(respBytes) + })) + + server.SetUpstreamHost(upstreamMockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, upstreamMockServer.URL, targetUpstreamHost) + defer upstreamMockServer.Close() + + // ==== no data ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + rootDocument, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, rootDocument.Bitmap, 32479) +} diff --git a/http/multipart.go b/http/multipart.go index 154beb6..ea54b8c 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -311,13 +311,16 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { - if s.IsDbNotFound(err) { - return http.StatusNotFound, respHeader, nil, nil + if !s.IsDbNotFound(err) { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + if document == nil { + document = common.NewDocument(rootDocument) + } else { + document.SetRootDocument(rootDocument) } - document.SetRootDocument(rootDocument) oldDocBytes, err := document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) From 73fa2d5c46b4bc07b08657dc915462e8327d8006 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 26 Feb 2024 19:29:40 -0800 Subject: [PATCH 013/155] fix a bug that auto migration did not update xpc_group_config table when devices reporting same versions as the newly automigrated versions --- common/document.go | 3 ++ common/document_test.go | 46 +++++++++++++++++++++++++++++-- common/subdocument.go | 34 +++++++++++------------ db/cassandra/document_test.go | 18 ++++++++---- db/cassandra/state_update_test.go | 6 ++-- db/service.go | 14 ++++++++-- db/sqlite/document_test.go | 12 +++++--- db/sqlite/state_metrics_test.go | 3 +- http/multipart.go | 13 +++++---- 9 files changed, 108 insertions(+), 41 deletions(-) diff --git a/common/document.go b/common/document.go index 61b054c..d99f245 100644 --- a/common/document.go +++ b/common/document.go @@ -105,6 +105,9 @@ func (d *Document) GetRootDocument() *RootDocument { } func (d *Document) RootVersion() string { + if d.rootDocument == nil { + return "" + } return d.rootDocument.Version } diff --git a/common/document_test.go b/common/document_test.go index 26caeb7..bb222d9 100644 --- a/common/document_test.go +++ b/common/document_test.go @@ -18,6 +18,9 @@ package common import ( + crand "crypto/rand" + "math/rand" + "strconv" "testing" "time" @@ -25,7 +28,44 @@ import ( ) func TestDocument(t *testing.T) { - // TODO a place holder for now - ts := int(time.Now().UnixNano() / 1000000) - assert.Assert(t, ts > 0) + document := NewDocument(nil) + assert.Equal(t, len(document.RootVersion()), 0) + + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + document = NewDocument(rootdoc) + + subdocIds := []string{"red", "orange", "yellow", "green"} + mparts := []Multipart{} + versionMap := make(map[string]string) + for _, subdocId := range subdocIds { + blen := rand.Intn(10) + 10 + bbytes := make([]byte, blen) + crand.Read(bbytes) + version := strconv.Itoa(int(time.Now().Unix())) + mpart := Multipart{ + Bytes: bbytes, + Version: version, + Name: subdocId, + State: Deployed, + } + mparts = append(mparts, mpart) + versionMap[subdocId] = version + } + + document.SetSubDocuments(mparts) + assert.Equal(t, len(document.Items()), len(subdocIds)) + + filteredDocument := document.FilterForGet(versionMap) + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 0) + + filteredDocument = document.FilterForGet(nil) + assert.Assert(t, filteredDocument != nil) + assert.Equal(t, len(filteredDocument.Items()), 4) } diff --git a/common/subdocument.go b/common/subdocument.go index 836eb37..336b22b 100644 --- a/common/subdocument.go +++ b/common/subdocument.go @@ -142,59 +142,59 @@ func (d *SubDocument) SetExpiry(expiry *int) { d.expiry = expiry } -func (d *SubDocument) Equals(tdoc *SubDocument) error { +func (d *SubDocument) Equals(tdoc *SubDocument) (bool, error) { if d.HasPayload() && tdoc.HasPayload() { if !bytes.Equal(d.Payload(), tdoc.Payload()) { - err := fmt.Errorf("d.Payload() != tdoc.Payload(), len(d.Payload())=%v, len(tdoc.Payload())=%v", len(d.payload), len(tdoc.payload)) - return NewError(err) + err := fmt.Errorf("d.Payload() != tdoc.Payload(), d.Payload()=%v, tdoc.Payload()=%v", d.payload, tdoc.payload) + return false, NewError(err) } } else { if d.HasPayload() != tdoc.HasPayload() { err := fmt.Errorf("d.HasPayload() != tdoc.HasPayload()") - return NewError(err) + return false, NewError(err) } } if d.Version() != nil && tdoc.Version() != nil { if *d.Version() != *tdoc.Version() { err := fmt.Errorf("*d.Version()[%v] != *tdoc.Version()[%v]", *d.Version(), *tdoc.Version()) - return NewError(err) + return false, NewError(err) } } else { if d.Version() != tdoc.Version() { err := fmt.Errorf("d.Version()[%v] != tdoc.Version()[%v]", d.Version(), tdoc.Version()) - return NewError(err) + return false, NewError(err) } } if d.UpdatedTime() != nil && tdoc.UpdatedTime() != nil { if *d.UpdatedTime() != *tdoc.UpdatedTime() { err := fmt.Errorf("*d.UpdatedTime()[%v] != *tdoc.UpdatedTime()[%v]", *d.UpdatedTime(), *tdoc.UpdatedTime()) - return NewError(err) + return false, NewError(err) } } else { if d.UpdatedTime() != tdoc.UpdatedTime() { err := fmt.Errorf("d.UpdatedTime()[%v] != tdoc.UpdatedTime()[%v]", d.UpdatedTime(), tdoc.UpdatedTime()) - return NewError(err) + return false, NewError(err) } } if d.State() != nil && tdoc.State() != nil { if *d.State() != *tdoc.State() { err := fmt.Errorf("*d.State()[%v] != *tdoc.State()[%v]", *d.State(), *tdoc.State()) - return NewError(err) + return false, NewError(err) } } else { if d.State() != tdoc.State() { err := fmt.Errorf("d.State()[%v] != tdoc.State()[%v]", d.State(), tdoc.State()) - return NewError(err) + return false, NewError(err) } } if d.ErrorCode() != nil && tdoc.ErrorCode() != nil { if *d.ErrorCode() != *tdoc.ErrorCode() { err := fmt.Errorf("*d.ErrorCode()[%v] != *tdoc.ErrorCode()[%v]", *d.ErrorCode(), *tdoc.ErrorCode()) - return NewError(err) + return false, NewError(err) } } else { var dErrorCode, tdocErrorCode int @@ -206,14 +206,14 @@ func (d *SubDocument) Equals(tdoc *SubDocument) error { } if dErrorCode != tdocErrorCode { err := fmt.Errorf("d.ErrorCode()[%v] != tdoc.ErrorCode()[%v]", d.ErrorCode(), tdoc.ErrorCode()) - return NewError(err) + return false, NewError(err) } } if d.ErrorDetails() != nil && tdoc.ErrorDetails() != nil { if *d.ErrorDetails() != *tdoc.ErrorDetails() { err := fmt.Errorf("*d.ErrorDetails()[%v] != *tdoc.ErrorDetails()[%v]", *d.ErrorDetails(), *tdoc.ErrorDetails()) - return NewError(err) + return false, NewError(err) } } else { var dErrorDetails, tdocErrorDetails string @@ -225,23 +225,23 @@ func (d *SubDocument) Equals(tdoc *SubDocument) error { } if dErrorDetails != tdocErrorDetails { err := fmt.Errorf("d.ErrorDetails()[%v] != tdoc.ErrorDetails()[%v]", d.ErrorDetails(), tdoc.ErrorDetails()) - return NewError(err) + return false, NewError(err) } } if d.Expiry() != nil && tdoc.Expiry() != nil { if *d.Expiry() != *tdoc.Expiry() { err := fmt.Errorf("*d.Expiry()[%v] != *tdoc.Expiry()[%v]", *d.Expiry(), *tdoc.Expiry()) - return NewError(err) + return false, NewError(err) } } else { if d.Expiry() != tdoc.Expiry() { err := fmt.Errorf("d.Expiry()[%v] != tdoc.Expiry()[%v]", d.Expiry(), tdoc.Expiry()) - return NewError(err) + return false, NewError(err) } } - return nil + return true, nil } func (d *SubDocument) NeedsUpdateForHttp304() bool { diff --git a/db/cassandra/document_test.go b/db/cassandra/document_test.go index 24bf0ac..b314690 100644 --- a/db/cassandra/document_test.go +++ b/db/cassandra/document_test.go @@ -57,7 +57,9 @@ func TestMocaSubDocument(t *testing.T) { fetchedSubdoc, err := tdbclient.GetSubDocument(cpeMac, subdocId) assert.NilError(t, err) - assert.Assert(t, srcSubdoc.Equals(fetchedSubdoc)) + ok, err := srcSubdoc.Equals(fetchedSubdoc) + assert.NilError(t, err) + assert.Assert(t, ok) } func TestPrivatessidSubDocument(t *testing.T) { @@ -82,7 +84,9 @@ func TestPrivatessidSubDocument(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - assert.Assert(t, srcDoc.Equals(fetchedDoc)) + ok, err := srcDoc.Equals(fetchedDoc) + assert.NilError(t, err) + assert.Assert(t, ok) } func TestMultiSubDocuments(t *testing.T) { @@ -114,8 +118,9 @@ func TestMultiSubDocuments(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) } doc, err := tdbclient.GetDocument(cpeMac) @@ -125,7 +130,9 @@ func TestMultiSubDocuments(t *testing.T) { for k, v := range srcmap { dv := doc.SubDocument(k) assert.Assert(t, dv != nil) - assert.Assert(t, v.Equals(dv)) + ok, err := v.Equals(dv) + assert.NilError(t, err) + assert.Assert(t, ok) } // ==== delete a document ==== @@ -178,8 +185,9 @@ func TestBlockedSubdocIds(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) } // add version1 and bitmap1 version1 := "indigo violet" diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 4b57677..0857770 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -48,8 +48,9 @@ func TestStateUpdate1(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) // update to state failure template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` @@ -98,8 +99,9 @@ func TestStateUpdate2(t *testing.T) { fetchedDoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = srcDoc.Equals(fetchedDoc) + ok, err := srcDoc.Equals(fetchedDoc) assert.NilError(t, err) + assert.Assert(t, ok) // update to state failure template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` diff --git a/db/service.go b/db/service.go index 582c94e..2cc326d 100644 --- a/db/service.go +++ b/db/service.go @@ -369,7 +369,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage return nil } -func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, fields log.Fields) error { +func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { var oldState int if oldSubdoc != nil && oldSubdoc.State() != nil { oldState = *oldSubdoc.State() @@ -384,6 +384,14 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old updatedTime := int(time.Now().UnixNano() / 1000000) newSubdoc.SetUpdatedTime(&updatedTime) newState := common.InDeployment + if oldVersion, ok := versionMap[subdocId]; ok { + if newSubdoc.Version() != nil { + if oldVersion == *newSubdoc.Version() { + newState = common.Deployed + } + } + } + newSubdoc.SetState(&newState) err = c.SetSubDocument(cpeMac, subdocId, newSubdoc, oldState, labels, fields) if err != nil { @@ -392,7 +400,7 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old return nil } -func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string, newDoc *common.Document, d *common.Document, toDelete bool, fields log.Fields) error { +func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string, newDoc *common.Document, d *common.Document, toDelete bool, versionMap map[string]string, fields log.Fields) error { newRootVersion := upstreamRespEtag if d.RootVersion() != newRootVersion { err := c.SetRootDocumentVersion(cpeMac, newRootVersion) @@ -409,7 +417,7 @@ func WriteDocumentFromUpstream(c DatabaseClient, cpeMac, upstreamRespEtag string // need to set "state" to proper values like the download is complete for subdocId, newSubdoc := range newDoc.Items() { oldSubdoc := d.SubDocument(subdocId) - err := UpdateSubDocument(c, cpeMac, subdocId, &newSubdoc, oldSubdoc, fields) + err := UpdateSubDocument(c, cpeMac, subdocId, &newSubdoc, oldSubdoc, versionMap, fields) if err != nil { return common.NewError(err) } diff --git a/db/sqlite/document_test.go b/db/sqlite/document_test.go index a633ee3..ea6e15e 100644 --- a/db/sqlite/document_test.go +++ b/db/sqlite/document_test.go @@ -48,8 +48,9 @@ func TestSubDocumentDb(t *testing.T) { // read a SubDocument from db and verify identical targetSubDocument, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = sourceDoc.Equals(targetSubDocument) + ok, err := sourceDoc.Equals(targetSubDocument) assert.NilError(t, err) + assert.Assert(t, ok) // ==== update an existing doc with the same cpeMac and groupId ==== srcVersion2 := "red white blue" @@ -61,8 +62,9 @@ func TestSubDocumentDb(t *testing.T) { assert.NilError(t, err) expectedDoc := common.NewSubDocument(srcBytes, &srcVersion2, &srcState, &srcUpdatedTime, nil, nil) - err = targetSubDocument.Equals(expectedDoc) + ok, err = targetSubDocument.Equals(expectedDoc) assert.NilError(t, err) + assert.Assert(t, ok) // ==== delete a doc ==== err = tdbclient.DeleteSubDocument(cpeMac, groupId) @@ -106,10 +108,12 @@ func TestDbReadDocument(t *testing.T) { assert.NilError(t, err) assert.Equal(t, Document.Length(), 2) - err = pdoc.Equals(Document.SubDocument("privatessid")) + ok, err := pdoc.Equals(Document.SubDocument("privatessid")) assert.NilError(t, err) - err = hdoc.Equals(Document.SubDocument("homessid")) + assert.Assert(t, ok) + ok, err = hdoc.Equals(Document.SubDocument("homessid")) assert.NilError(t, err) + assert.Assert(t, ok) // ==== delete all SubDocuments ==== err = tdbclient.DeleteDocument(cpeMac) diff --git a/db/sqlite/state_metrics_test.go b/db/sqlite/state_metrics_test.go index d8bcb02..bb2cb00 100644 --- a/db/sqlite/state_metrics_test.go +++ b/db/sqlite/state_metrics_test.go @@ -65,8 +65,9 @@ func TestStateMetrics(t *testing.T) { // read a SubDocument from db and verify identical doc1, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) - err = sourceDoc.Equals(doc1) + ok, err := sourceDoc.Equals(doc1) assert.NilError(t, err) + assert.Assert(t, ok) // ==== update an doc with the same cpeMac and a changed state ==== state2 := common.InDeployment diff --git a/http/multipart.go b/http/multipart.go index ea54b8c..5051082 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -267,17 +267,18 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin finalRootDocument := common.NewRootDocument(0, "", "", "", "", upstreamRespEtag, "") finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) - finalFilteredDocument := finalDocument.FilterForGet(deviceVersionMap) - for _, subdocId := range c.BlockedSubdocIds() { - finalFilteredDocument.DeleteSubDocument(subdocId) - } // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalFilteredDocument, document, false, fields) + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) if err != nil { return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) } + finalFilteredDocument := finalDocument.FilterForGet(deviceVersionMap) + for _, subdocId := range c.BlockedSubdocIds() { + finalFilteredDocument.DeleteSubDocument(subdocId) + } + // 304 if finalFilteredDocument.Length() == 0 { return http.StatusNotModified, upstreamRespHeader, nil, nil @@ -390,7 +391,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, fields) + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) if err != nil { return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) } From 51e6afe0d3f814dcc1132ab2fdc88a5184a3b020 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 1 Mar 2024 09:41:30 -0800 Subject: [PATCH 014/155] return updated root_version in POST /document call --- common/error.go | 33 +++++++++++++++++++++++++-------- http/document_handler.go | 13 +++++++++++-- http/main_test.go | 3 +-- http/webconfig_server.go | 6 ++---- security/token.go | 4 ++-- 5 files changed, 41 insertions(+), 18 deletions(-) diff --git a/common/error.go b/common/error.go index 0bb87e5..e733968 100644 --- a/common/error.go +++ b/common/error.go @@ -22,11 +22,16 @@ import ( "fmt" "path/filepath" "runtime" + "slices" + "strings" +) + +const ( + defaultTracebackLevel = 3 ) var ( - NotOK = fmt.Errorf("!ok") - ProfileNotFound = fmt.Errorf("profile not found") + ErrNotOK = fmt.Errorf("!ok") ) type Http400Error struct { @@ -121,12 +126,24 @@ func UnwrapAll(wrappedErr error) error { return err } -func GetCaller() string { - _, file, line, _ := runtime.Caller(1) - filename := filepath.Base(file) - fulldir := filepath.Dir(file) - dir := filepath.Base(fulldir) - return fmt.Sprintf("%v/%v[%v]", dir, filename, line) +func GetCaller(args ...int) string { + items := []string{} + + tracebackLevel := defaultTracebackLevel + if len(args) > 0 { + tracebackLevel = args[0] + } + + for i := 0; i < tracebackLevel; i++ { + _, file, line, _ := runtime.Caller(i + 1) + filename := filepath.Base(file) + fulldir := filepath.Dir(file) + dir := filepath.Base(fulldir) + items = append(items, fmt.Sprintf("%v/%v[%v]", dir, filename, line)) + } + + slices.Reverse(items) + return strings.Join(items, " ") } type NoCapabilitiesError struct { diff --git a/http/document_handler.go b/http/document_handler.go index 2f38afd..f41f7ff 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -160,6 +160,8 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. metricsAgent = "default" } + rootVersionMap := make(map[string]string) + var newRootVersion string for _, deviceId := range deviceIds { fields["src_caller"] = common.GetCaller() @@ -189,15 +191,22 @@ func (s *WebconfigServer) PostSubDocumentHandler(w http.ResponseWriter, r *http. } doc.SetSubDocument(subdocId, subdoc) - newRootVersion := db.HashRootVersion(doc.VersionMap()) + newRootVersion = db.HashRootVersion(doc.VersionMap()) err = s.SetRootDocumentVersion(deviceId, newRootVersion) if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) return } + rootVersionMap[deviceId] = newRootVersion + } + d := make(util.Dict) + if len(rootVersionMap) == 1 { + d["root_version"] = newRootVersion + } else { + d["root_version"] = rootVersionMap } - WriteOkResponse(w, nil) + WriteByMarshal(w, http.StatusOK, d) } func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *http.Request) { diff --git a/http/main_test.go b/http/main_test.go index 7ccd1f2..6da258a 100644 --- a/http/main_test.go +++ b/http/main_test.go @@ -45,9 +45,8 @@ func TestMain(m *testing.M) { panic(err) } + _ = GetTestDatabaseClient(sc) log.SetOutput(io.Discard) - returnCode := m.Run() - os.Exit(returnCode) } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 0c2d146..0d82f31 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -151,10 +151,6 @@ func GetTestDatabaseClient(sc *common.ServerConfig) db.DatabaseClient { err = fmt.Errorf("Unsupported database.active_driver %v is configured", activeDriver) panic(err) } - err = tdbclient.SetUp() - if err != nil { - panic(err) - } return tdbclient } @@ -767,6 +763,8 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { err1 := common.NewError(err) fields["response"] = ObfuscatedMap fields["response_text"] = err1.Error() + } else { + fields["response"] = itf } } else { fields["response"] = ObfuscatedMap diff --git a/security/token.go b/security/token.go index 51695e2..169cd0f 100644 --- a/security/token.go +++ b/security/token.go @@ -213,11 +213,11 @@ func ParseKidFromTokenHeader(tokenString string) (string, error) { rawKid, ok := headers["kid"] if !ok { - return kid, common.NewError(common.NotOK) + return kid, common.NewError(common.ErrNotOK) } kid, ok = rawKid.(string) if !ok { - return kid, common.NewError(common.NotOK) + return kid, common.NewError(common.ErrNotOK) } return kid, nil From 8eebeb8122f17720960d0978de2cc61561998e5c Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 7 Mar 2024 13:40:58 -0800 Subject: [PATCH 015/155] copy all incoming headers when calling upstream --- http/multipart.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index 5051082..cb445a6 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -207,7 +207,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // ============================= // upstream handling // ============================= - upstreamHeader := make(http.Header) + upstreamHeader := rHeader.Clone() upstreamHeader.Set("Content-type", common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) if itf, ok := fields["audit_id"]; ok { @@ -338,19 +338,17 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l // ============================= // upstream handling // ============================= - upstreamHeader := make(http.Header) + upstreamHeader := rHeader.Clone() upstreamHeader.Set("Content-type", common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) + if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) if len(auditId) > 0 { upstreamHeader.Set(common.HeaderAuditid, auditId) } } - if x := rHeader.Get(common.HeaderTransactionId); len(x) > 0 { - upstreamHeader.Set(common.HeaderTransactionId, x) - } if s.TokenManager != nil { token := rHeader.Get("Authorization") @@ -363,9 +361,6 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } // call /upstream to handle factory reset - ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) - upstreamHeader.Set(common.HeaderIfNoneMatch, ifNoneMatch) - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) if err != nil { var rherr common.RemoteHttpError From 6e829bb8ca46c1b23ab63b7312f79ff3b128a45f Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 8 Mar 2024 22:22:39 -0800 Subject: [PATCH 016/155] fix a bug that some responses are not of json format when there are errors --- http/multipart.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index cb445a6..227364f 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -88,9 +88,6 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. } status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) - if err != nil && respBytes == nil { - respBytes = []byte(err.Error()) - } // REMINDER 404 use standard response if status == http.StatusNotFound { @@ -102,6 +99,11 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. w.Header().Set(k, respHeader.Get(k)) } + if err != nil && respBytes == nil { + Error(w, status, common.NewError(err)) + return + } + w.WriteHeader(status) _, _ = w.Write(respBytes) } From 19a57e97d2c02e08b49ef79c6f75ccd5f8e3b262 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 7 Mar 2024 13:40:58 -0800 Subject: [PATCH 017/155] copy all incoming headers when calling upstream --- http/multipart.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index 5051082..cb445a6 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -207,7 +207,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // ============================= // upstream handling // ============================= - upstreamHeader := make(http.Header) + upstreamHeader := rHeader.Clone() upstreamHeader.Set("Content-type", common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) if itf, ok := fields["audit_id"]; ok { @@ -338,19 +338,17 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l // ============================= // upstream handling // ============================= - upstreamHeader := make(http.Header) + upstreamHeader := rHeader.Clone() upstreamHeader.Set("Content-type", common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) + if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) if len(auditId) > 0 { upstreamHeader.Set(common.HeaderAuditid, auditId) } } - if x := rHeader.Get(common.HeaderTransactionId); len(x) > 0 { - upstreamHeader.Set(common.HeaderTransactionId, x) - } if s.TokenManager != nil { token := rHeader.Get("Authorization") @@ -363,9 +361,6 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } // call /upstream to handle factory reset - ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) - upstreamHeader.Set(common.HeaderIfNoneMatch, ifNoneMatch) - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) if err != nil { var rherr common.RemoteHttpError From 5f4eedd82191418791326f05ccfc596f9661aaf6 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 16 Mar 2024 11:28:12 -0700 Subject: [PATCH 018/155] add support of using reference_document --- common/const_var.go | 1 + common/refsubdocument.go | 81 +++++++++++++++ common/refsubdocument_test.go | 40 ++++++++ db/cassandra/refsubdocument.go | 81 +++++++++++++++ db/cassandra/refsubdocument_test.go | 59 +++++++++++ db/cassandra/schema.go | 5 + db/database_client.go | 5 + db/service.go | 36 +++++++ db/sqlite/refsubdocument.go | 153 ++++++++++++++++++++++++++++ db/sqlite/refsubdocument_test.go | 59 +++++++++++ db/sqlite/schema.go | 5 + http/multipart.go | 23 +++++ http/refsubdocument_handler.go | 125 +++++++++++++++++++++++ http/refsubdocument_handler_test.go | 76 ++++++++++++++ http/router.go | 14 +++ http/validator.go | 37 +++++++ 16 files changed, 800 insertions(+) create mode 100644 common/refsubdocument.go create mode 100644 common/refsubdocument_test.go create mode 100644 db/cassandra/refsubdocument.go create mode 100644 db/cassandra/refsubdocument_test.go create mode 100644 db/sqlite/refsubdocument.go create mode 100644 db/sqlite/refsubdocument_test.go create mode 100644 http/refsubdocument_handler.go create mode 100644 http/refsubdocument_handler_test.go diff --git a/common/const_var.go b/common/const_var.go index 4370c7e..e024f97 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -106,6 +106,7 @@ const ( HeaderTraceparent = "Traceparent" HeaderTracestate = "Tracestate" HeaderContentLength = "Content-Length" + HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" ) // header X-System-Supported-Docs diff --git a/common/refsubdocument.go b/common/refsubdocument.go new file mode 100644 index 0000000..cad7ffc --- /dev/null +++ b/common/refsubdocument.go @@ -0,0 +1,81 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "bytes" +) + +type RefSubDocument struct { + payload []byte + version *string +} + +func NewRefSubDocument(payload []byte, version *string) *RefSubDocument { + return &RefSubDocument{ + payload: payload, + version: version, + } +} + +func (d *RefSubDocument) Payload() []byte { + return d.payload +} + +func (d *RefSubDocument) SetPayload(payload []byte) { + d.payload = payload +} + +func (d *RefSubDocument) HasPayload() bool { + if d.payload != nil && len(d.payload) > 0 { + return true + } else { + return false + } +} + +func (d *RefSubDocument) Version() *string { + return d.version +} + +func (d *RefSubDocument) SetVersion(version *string) { + d.version = version +} + +func (d *RefSubDocument) Equals(tdoc *RefSubDocument) bool { + if d.HasPayload() && tdoc.HasPayload() { + if !bytes.Equal(d.Payload(), tdoc.Payload()) { + return false + } + } else { + if d.HasPayload() != tdoc.HasPayload() { + return false + } + } + + if d.Version() != nil && tdoc.Version() != nil { + if *d.Version() != *tdoc.Version() { + return false + } + } else { + if d.Version() != tdoc.Version() { + return false + } + } + return true +} diff --git a/common/refsubdocument_test.go b/common/refsubdocument_test.go new file mode 100644 index 0000000..416c123 --- /dev/null +++ b/common/refsubdocument_test.go @@ -0,0 +1,40 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestRefSubDocument(t *testing.T) { + bbytes1 := []byte("hello world") + version1 := "12345" + refsubdoc1 := NewRefSubDocument(bbytes1, &version1) + + bbytes2 := []byte("hello world") + version2 := "12345" + refsubdoc2 := NewRefSubDocument(bbytes2, &version2) + assert.Assert(t, refsubdoc1.Equals(refsubdoc2)) + + bbytes3 := []byte("foo bar") + version3 := "12345" + refsubdoc3 := NewRefSubDocument(bbytes3, &version3) + assert.Assert(t, !refsubdoc1.Equals(refsubdoc3)) +} diff --git a/db/cassandra/refsubdocument.go b/db/cassandra/refsubdocument.go new file mode 100644 index 0000000..1c9ca9f --- /dev/null +++ b/db/cassandra/refsubdocument.go @@ -0,0 +1,81 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "fmt" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/gocql/gocql" +) + +func (c *CassandraClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { + var payload []byte + var version string + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt := "SELECT payload,version FROM reference_document WHERE ref_id=?" + if err := c.Query(stmt, refId).Scan(&payload, &version); err != nil { + return nil, common.NewError(err) + } + + if len(payload) == 0 { + return nil, common.NewError(gocql.ErrNotFound) + } + + refsubdoc := common.NewRefSubDocument(payload, &version) + return refsubdoc, nil +} + +func (c *CassandraClient) SetRefSubDocument(refId string, refsubdoc *common.RefSubDocument) (fnerr error) { + // build the statement and avoid unnecessary fields/columns + columns := []string{"ref_id"} + values := []interface{}{refId} + if refsubdoc.Payload() != nil && len(refsubdoc.Payload()) > 0 { + columns = append(columns, "payload") + values = append(values, refsubdoc.Payload()) + } + + if refsubdoc.Version() != nil { + columns = append(columns, "version") + values = append(values, refsubdoc.Version()) + } + stmt := fmt.Sprintf("INSERT INTO reference_document(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + if err := c.Query(stmt, values...).Exec(); err != nil { + return common.NewError(err) + } + return nil +} + +func (c *CassandraClient) DeleteRefSubDocument(refId string) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt := "DELETE FROM reference_document WHERE ref_id=?" + if err := c.Query(stmt, refId).Exec(); err != nil { + return common.NewError(err) + } + return nil +} diff --git a/db/cassandra/refsubdocument_test.go b/db/cassandra/refsubdocument_test.go new file mode 100644 index 0000000..22e0ba8 --- /dev/null +++ b/db/cassandra/refsubdocument_test.go @@ -0,0 +1,59 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "crypto/rand" + "testing" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentOperation(t *testing.T) { + refId := uuid.New().String() + + // prepare the source data + slen := util.RandomInt(100) + 16 + srcBytes := make([]byte, slen) + rand.Read(srcBytes) + srcVersion := util.GetMurmur3Hash(srcBytes) + + // verify empty before start + var err error + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // write into db + srcRefsubdoc := common.NewRefSubDocument(srcBytes, &srcVersion) + err = tdbclient.SetRefSubDocument(refId, srcRefsubdoc) + assert.NilError(t, err) + + fetchedRefsubdoc, err := tdbclient.GetRefSubDocument(refId) + assert.NilError(t, err) + assert.Assert(t, srcRefsubdoc.Equals(fetchedRefsubdoc)) + + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) + + // verify not found in db now + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) +} diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index 4d0ac95..b013804 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -44,6 +44,11 @@ var ( route text, schema_version text, version text +)`, + `CREATE TABLE IF NOT EXISTS reference_document ( + ref_id text PRIMARY KEY, + payload blob, + version text )`, } diff --git a/db/database_client.go b/db/database_client.go index b074e59..7f9a8bf 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -59,4 +59,9 @@ type DatabaseClient interface { FactoryReset(string) error FirmwareUpdate(string, int, *common.RootDocument) error AppendProfiles(string, []byte) ([]byte, error) + + // reference subdocument + GetRefSubDocument(string) (*common.RefSubDocument, error) + SetRefSubDocument(string, *common.RefSubDocument) error + DeleteRefSubDocument(string) error } diff --git a/db/service.go b/db/service.go index 2cc326d..c80e240 100644 --- a/db/service.go +++ b/db/service.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "net/http" + "slices" "sort" "strings" "time" @@ -31,6 +32,14 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + referenceIndicatorByteLength = 4 +) + +var ( + referenceIndicatorBytes = make([]byte, referenceIndicatorByteLength) +) + // TODO s.MultipartSupplementaryHandler(w, r) should be handled separately // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream @@ -527,3 +536,30 @@ func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerI } return cloudRootDocument, nil } + +func GetRefId(payload []byte) (string, bool) { + if len(payload) > referenceIndicatorByteLength { + prefixBytes := payload[:referenceIndicatorByteLength] + if slices.Equal(referenceIndicatorBytes, prefixBytes) { + suffixBytes := payload[referenceIndicatorByteLength:] + return string(suffixBytes), true + } + } + return "", false +} + +func LoadRefSubDocuments(c DatabaseClient, document *common.Document, fields log.Fields) (*common.Document, error) { + newDocument := common.NewDocument(document.GetRootDocument()) + for subdocId, subDocument := range document.Items() { + payload := subDocument.Payload() + if refId, ok := GetRefId(payload); ok { + refsubdocument, err := c.GetRefSubDocument(refId) + if err != nil { + return nil, common.NewError(err) + } + subDocument.SetPayload(refsubdocument.Payload()) + } + newDocument.SetSubDocument(subdocId, &subDocument) + } + return newDocument, nil +} diff --git a/db/sqlite/refsubdocument.go b/db/sqlite/refsubdocument.go new file mode 100644 index 0000000..b93d4eb --- /dev/null +++ b/db/sqlite/refsubdocument.go @@ -0,0 +1,153 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "database/sql" + "fmt" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + _ "github.com/mattn/go-sqlite3" +) + +func (c *SqliteClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + rows, err := c.Query("SELECT payload,version FROM reference_document WHERE ref_id=?", refId) + if err != nil { + return nil, common.NewError(err) + } + + var ns1 sql.NullString + var b1 []byte + + if !rows.Next() { + return nil, sql.ErrNoRows + } + err = rows.Scan(&b1, &ns1) + defer rows.Close() + if err != nil { + return nil, common.NewError(err) + } + + var s1 *string + if ns1.Valid { + s1 = &(ns1.String) + } + + refsubdoc := common.NewRefSubDocument(b1, s1) + return refsubdoc, nil +} + +func (c *SqliteClient) insertRefSubDocument(refId string, refsubdoc *common.RefSubDocument) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // build the statement and avoid unnecessary fields/columns + columns := []string{"ref_id"} + values := []interface{}{refId} + if refsubdoc.Payload() != nil { + columns = append(columns, "payload") + values = append(values, refsubdoc.Payload()) + } + if refsubdoc.Version() != nil { + columns = append(columns, "version") + values = append(values, refsubdoc.Version()) + } + qstr := fmt.Sprintf("INSERT INTO reference_document(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(values...) + if err != nil { + return common.NewError(err) + } + return nil +} + +func (c *SqliteClient) updateRefSubDocument(refId string, doc *common.RefSubDocument) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // build the statement and avoid unnecessary fields/columns + columns := []string{} + values := []interface{}{} + if doc.Payload() != nil { + columns = append(columns, "payload") + values = append(values, doc.Payload()) + } + if doc.Version() != nil { + columns = append(columns, "version") + values = append(values, doc.Version()) + } + values = append(values, refId) + qstr := fmt.Sprintf("UPDATE reference_document SET %v WHERE cpe_mac=? AND ref_id=?", db.GetSetColumnsStr(columns)) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(values...) + if err != nil { + return common.NewError(err) + } + return nil +} + +func (c *SqliteClient) SetRefSubDocument(refId string, refsubdoc *common.RefSubDocument) error { + _, err := c.GetRefSubDocument(refId) + if err != nil { + if c.IsDbNotFound(err) { + err1 := c.insertRefSubDocument(refId, refsubdoc) + if err1 != nil { + return common.NewError(err1) + } + } else { + // unexpected error + return common.NewError(err) + } + } else { + // normal dbNotFound should not happen + err = c.updateRefSubDocument(refId, refsubdoc) + if err != nil { + return common.NewError(err) + } + } + + return nil +} + +func (c *SqliteClient) DeleteRefSubDocument(refId string) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt, err := c.Prepare("DELETE FROM reference_document WHERE ref_id=?") + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(refId) + if err != nil { + return common.NewError(err) + } + return nil +} diff --git a/db/sqlite/refsubdocument_test.go b/db/sqlite/refsubdocument_test.go new file mode 100644 index 0000000..01478f9 --- /dev/null +++ b/db/sqlite/refsubdocument_test.go @@ -0,0 +1,59 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "crypto/rand" + "testing" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentOperation(t *testing.T) { + refId := uuid.New().String() + + // prepare the source data + slen := util.RandomInt(100) + 16 + srcBytes := make([]byte, slen) + rand.Read(srcBytes) + srcVersion := util.GetMurmur3Hash(srcBytes) + + // verify empty before start + var err error + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // write into db + srcRefsubdoc := common.NewRefSubDocument(srcBytes, &srcVersion) + err = tdbclient.SetRefSubDocument(refId, srcRefsubdoc) + assert.NilError(t, err) + + fetchedRefsubdoc, err := tdbclient.GetRefSubDocument(refId) + assert.NilError(t, err) + assert.Assert(t, srcRefsubdoc.Equals(fetchedRefsubdoc)) + + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) + + // verify not found in db now + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) +} diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 8ee8d6e..3e4c132 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -50,6 +50,11 @@ var ( route text, schema_version, version text +)`, + `CREATE TABLE IF NOT EXISTS reference_document ( + ref_id text PRIMARY KEY, + payload blob, + version text )`, } ) diff --git a/http/multipart.go b/http/multipart.go index 227364f..b2860af 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -147,6 +147,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin document.DeleteSubDocument(subdocId) } + document, err = db.LoadRefSubDocuments(c, document, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } respBytes, err := document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -178,9 +182,20 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin document = common.NewDocument(rootDocument) } + if userAgent == "mget" { + postUpstream = false + } + var respBytes []byte respStatus := http.StatusNotModified if document.Length() > 0 { + + if !postUpstream { + document, err = db.LoadRefSubDocuments(c, document, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + } respBytes, err = document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -286,6 +301,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusNotModified, upstreamRespHeader, nil, nil } + finalFilteredDocument, err = db.LoadRefSubDocuments(c, finalFilteredDocument, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + } finalFilteredBytes, err := finalFilteredDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalFilteredBytes, common.NewError(err) @@ -397,6 +416,10 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusNotFound, upstreamRespHeader, nil, nil } + finalDocument, err = db.LoadRefSubDocuments(c, finalDocument, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + } finalBytes, err := finalDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalBytes, common.NewError(err) diff --git a/http/refsubdocument_handler.go b/http/refsubdocument_handler.go new file mode 100644 index 0000000..6413ee4 --- /dev/null +++ b/http/refsubdocument_handler.go @@ -0,0 +1,125 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "errors" + "net/http" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" +) + +func (s *WebconfigServer) GetRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, _, _, err := s.ValidateRefData(w, r, false) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + refsubdoc, err := s.GetRefSubDocument(refId) + if err != nil { + if s.IsDbNotFound(err) { + Error(w, http.StatusNotFound, nil) + } else { + LogError(w, err) + Error(w, http.StatusInternalServerError, common.NewError(err)) + } + return + } + + w.Header().Set("Content-Type", "application/msgpack") + if refsubdoc.Version() != nil { + w.Header().Set(common.HeaderRefSubdocumentVersion, *refsubdoc.Version()) + } + w.WriteHeader(http.StatusOK) + w.Write(refsubdoc.Payload()) +} + +func (s *WebconfigServer) PostRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, bbytes, _, err := s.ValidateRefData(w, r, true) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + // handle version header + version := r.Header.Get(common.HeaderSubdocumentVersion) + if len(version) == 0 { + version = util.GetMurmur3Hash(bbytes) + } + + refsubdoc := common.NewRefSubDocument(bbytes, &version) + + err = s.SetRefSubDocument(refId, refsubdoc) + if err != nil { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + + WriteOkResponse(w, nil) +} + +func (s *WebconfigServer) DeleteRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, _, _, err := s.ValidateRefData(w, r, false) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + err = s.DeleteRefSubDocument(refId) + if err != nil { + if s.IsDbNotFound(err) { + Error(w, http.StatusNotFound, nil) + } else { + Error(w, http.StatusInternalServerError, common.NewError(err)) + } + return + } + WriteOkResponse(w, nil) +} diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go new file mode 100644 index 0000000..49171b0 --- /dev/null +++ b/http/refsubdocument_handler_test.go @@ -0,0 +1,76 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "testing" + + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentHandler(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + refId := uuid.New().String() + bbytes := util.RandomBytes(100, 150) + + // post + url := fmt.Sprintf("/api/v1/reference/%v/document", refId) + req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes) + + // delete + req, err = http.NewRequest("DELETE", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get but expect 404 + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} diff --git a/http/router.go b/http/router.go index 7645c51..746bec0 100644 --- a/http/router.go +++ b/http/router.go @@ -127,5 +127,19 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { } sub4.HandleFunc("", s.DeleteDocumentHandler).Methods("DELETE") + sub5 := router.Path("/api/v1/reference/{ref}/document").Subrouter() + if testOnly { + sub5.Use(s.TestingMiddleware) + } else { + if s.ServerApiTokenAuthEnabled() { + sub5.Use(s.ApiMiddleware) + } else { + sub5.Use(s.NoAuthMiddleware) + } + } + sub5.HandleFunc("", s.GetRefSubDocumentHandler).Methods("GET") + sub5.HandleFunc("", s.PostRefSubDocumentHandler).Methods("POST") + sub5.HandleFunc("", s.DeleteRefSubDocumentHandler).Methods("DELETE") + return router } diff --git a/http/validator.go b/http/validator.go index 10683ec..d85921e 100644 --- a/http/validator.go +++ b/http/validator.go @@ -71,3 +71,40 @@ func (s *WebconfigServer) Validate(w http.ResponseWriter, r *http.Request, valid } return mac, subdocId, bodyBytes, fields, nil } + +func (s *WebconfigServer) ValidateRefData(w http.ResponseWriter, r *http.Request, validateContent bool) (string, []byte, log.Fields, error) { + var fields log.Fields + + // check mac + params := mux.Vars(r) + refId := params["ref"] + + // check for safety, but it should not fail + xw, ok := w.(*XResponseWriter) + if !ok { + err := *common.NewHttp500Error("responsewriter cast error") + return refId, nil, nil, common.NewError(err) + } + fields = xw.Audit() + + if !validateContent { + return refId, nil, fields, nil + } + + // ==== validate content ==== + // check content-type + contentType := r.Header.Get("Content-type") + if contentType != "application/msgpack" { + // TODO (1) if we should validate this header + // (2) if unexpected, return 400 or 415 + err := *common.NewHttp400Error("content-type not msgpack") + return refId, nil, nil, common.NewError(err) + } + + bodyBytes := xw.BodyBytes() + if len(bodyBytes) == 0 { + err := *common.NewHttp400Error("empty body") + return refId, nil, nil, common.NewError(err) + } + return refId, bodyBytes, fields, nil +} From a56833234cc80a15321770ca5bfcd063973b2904 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 18 Mar 2024 23:25:56 -0700 Subject: [PATCH 019/155] add an optional feature to correct states --- common/subdocument.go | 16 ++ common/subdocument_test.go | 10 +- config/sample_webconfig.conf | 3 + db/cassandra/cassandra_client.go | 33 ++-- db/cassandra/cassandra_client_test.go | 8 + db/database_client.go | 4 + db/service.go | 31 +++ db/sqlite/sqlite_client.go | 22 ++- db/sqlite/sqlite_client_test.go | 43 +++++ http/document_handler_test.go | 2 +- http/multipart_test.go | 261 +++++++++++++++++++++++++- 11 files changed, 409 insertions(+), 24 deletions(-) create mode 100644 db/sqlite/sqlite_client_test.go diff --git a/common/subdocument.go b/common/subdocument.go index 336b22b..bd54947 100644 --- a/common/subdocument.go +++ b/common/subdocument.go @@ -102,6 +102,14 @@ func (d *SubDocument) SetVersion(version *string) { d.version = version } +// convenient function +func (d *SubDocument) GetVersion() string { + if d.version != nil { + return *d.version + } + return "" +} + func (d *SubDocument) State() *int { return d.state } @@ -110,6 +118,14 @@ func (d *SubDocument) SetState(state *int) { d.state = state } +// convenient function +func (d *SubDocument) GetState() int { + if d.state != nil { + return *d.state + } + return 0 +} + func (d *SubDocument) UpdatedTime() *int { return d.updatedTime } diff --git a/common/subdocument_test.go b/common/subdocument_test.go index 3b4bf1a..379ad1b 100644 --- a/common/subdocument_test.go +++ b/common/subdocument_test.go @@ -32,6 +32,12 @@ func TestSubDocumentString(t *testing.T) { errorCode := 103 errorDetails := "cannot parse" - doc := NewSubDocument(bbytes, &version, &state, &updatedTime, &errorCode, &errorDetails) - assert.Assert(t, doc != nil) + subdoc := NewSubDocument(bbytes, &version, &state, &updatedTime, &errorCode, &errorDetails) + assert.Assert(t, subdoc != nil) + + subdoc = &SubDocument{} + tgtVersion := subdoc.GetVersion() + assert.Equal(t, tgtVersion, "") + tgtState := subdoc.GetState() + assert.Equal(t, tgtState, 0) } diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index f47c838..894f513 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -232,4 +232,7 @@ webconfig { valid_partners = ["company", "vendor1", "vendor2"] supplementary_appending_enabled = true + + // correct subdoc states if versions match but not "deployed" + state_correction_enabled= false } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index d9ca1f7..7855b9f 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -48,10 +48,11 @@ type CassandraClient struct { *gocql.ClusterConfig *security.AesCodec *common.AppMetrics - concurrentQueries chan bool - localDc string - blockedSubdocIds []string - encryptedSubdocIds []string + concurrentQueries chan bool + localDc string + blockedSubdocIds []string + encryptedSubdocIds []string + stateCorrectionEnabled bool } /* @@ -159,15 +160,17 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") encryptedSubdocIds := conf.GetStringList("webconfig.encrypted_subdoc_ids") + stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") return &CassandraClient{ - Session: session, - ClusterConfig: cluster, - AesCodec: codec, - concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), - localDc: localDc, - blockedSubdocIds: blockedSubdocIds, - encryptedSubdocIds: encryptedSubdocIds, + Session: session, + ClusterConfig: cluster, + AesCodec: codec, + concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), + localDc: localDc, + blockedSubdocIds: blockedSubdocIds, + encryptedSubdocIds: encryptedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, }, nil } @@ -216,6 +219,14 @@ func (c *CassandraClient) SetEncryptedSubdocIds(x []string) { c.encryptedSubdocIds = x } +func (c *CassandraClient) StateCorrectionEnabled() bool { + return c.stateCorrectionEnabled +} + +func (c *CassandraClient) SetStateCorrectionEnabled(enabled bool) { + c.stateCorrectionEnabled = enabled +} + // TODO we hardcoded for now but it should be changed to be configurable func (c *CassandraClient) IsEncryptedGroup(subdocId string) bool { return util.Contains(c.EncryptedSubdocIds(), subdocId) diff --git a/db/cassandra/cassandra_client_test.go b/db/cassandra/cassandra_client_test.go index 57c1734..355b701 100644 --- a/db/cassandra/cassandra_client_test.go +++ b/db/cassandra/cassandra_client_test.go @@ -38,6 +38,14 @@ func TestCassandraClient(t *testing.T) { tgtSubdocIds := tdbclient.EncryptedSubdocIds() assert.Assert(t, len(tgtSubdocIds) == 4) + + // state correction flag + enabled := true + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + enabled = false + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) } func TestGetConfig(t *testing.T) { diff --git a/db/database_client.go b/db/database_client.go index 7f9a8bf..fa1065f 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -64,4 +64,8 @@ type DatabaseClient interface { GetRefSubDocument(string) (*common.RefSubDocument, error) SetRefSubDocument(string, *common.RefSubDocument) error DeleteRefSubDocument(string) error + + // enable state correction + StateCorrectionEnabled() bool + SetStateCorrectionEnabled(bool) } diff --git a/db/service.go b/db/service.go index c80e240..882d0c4 100644 --- a/db/service.go +++ b/db/service.go @@ -29,6 +29,7 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -147,6 +148,36 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field } } + if c.StateCorrectionEnabled() { + if document == nil { + document, err = c.GetDocument(mac, fields) + if err != nil { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + } + } + for subdocId, subdocument := range document.Items() { + cloudVersion := subdocument.GetVersion() + cloudState := subdocument.GetState() + if len(cloudVersion) == 0 { + continue + } + deviceVersion := deviceVersionMap[subdocId] + if cloudVersion == deviceVersion && cloudState >= common.PendingDownload && cloudState <= common.Failure { + labels := prometheus.Labels{ + "model": modelName, + "fwversion": firmwareVersion, + } + // update state + newState := common.Deployed + subdocument.SetState(&newState) + if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + } + } + } + + } + switch rootCmpEnum { case common.RootDocumentEquals: // create an empty "document" diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index d623d07..9ab3b12 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -43,8 +43,9 @@ type SqliteClient struct { db.BaseClient *sql.DB *common.AppMetrics - concurrentQueries chan bool - blockedSubdocIds []string + concurrentQueries chan bool + blockedSubdocIds []string + stateCorrectionEnabled bool } func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { @@ -58,15 +59,18 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") + stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + db, err := sql.Open("sqlite3", dbfile) if err != nil { return nil, common.NewError(err) } return &SqliteClient{ - DB: db, - concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), - blockedSubdocIds: blockedSubdocIds, + DB: db, + concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), + blockedSubdocIds: blockedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, }, nil } @@ -136,6 +140,14 @@ func (c *SqliteClient) SetBlockedSubdocIds(x []string) { c.blockedSubdocIds = x } +func (c *SqliteClient) StateCorrectionEnabled() bool { + return c.stateCorrectionEnabled +} + +func (c *SqliteClient) SetStateCorrectionEnabled(enabled bool) { + c.stateCorrectionEnabled = enabled +} + func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { if tdbclient != nil { return tdbclient, nil diff --git a/db/sqlite/sqlite_client_test.go b/db/sqlite/sqlite_client_test.go new file mode 100644 index 0000000..3a3f74f --- /dev/null +++ b/db/sqlite/sqlite_client_test.go @@ -0,0 +1,43 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "testing" + + "github.com/rdkcentral/webconfig/common" + "gotest.tools/assert" +) + +func TestSqliteClient(t *testing.T) { + configFile := "../../config/sample_webconfig.conf" + sc, err := common.GetTestServerConfig(configFile) + + assert.NilError(t, err) + dbc, err := GetTestSqliteClient(sc.Config, true) + assert.NilError(t, err) + assert.Assert(t, dbc != nil) + + // state correction flag + enabled := true + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + enabled = false + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) +} diff --git a/http/document_handler_test.go b/http/document_handler_test.go index f879914..7d11287 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -419,7 +419,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { subdocVersions = append(subdocVersions, mpart.Version) // ==== step 5 get document again ==== - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,gwrestore,remotedebugger,lan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) diff --git a/http/multipart_test.go b/http/multipart_test.go index 0dc6be5..30919f8 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -133,7 +133,7 @@ func TestMultipartConfigHandler(t *testing.T) { wanMpartVersion := mpart.Version _ = wanMpartVersion - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -144,7 +144,7 @@ func TestMultipartConfigHandler(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) - // ==== cal GET /config with if-none-match partial match ==== + // ==== call GET /config with if-none-match partial match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -819,7 +819,7 @@ func TestMultipartConfigMismatch(t *testing.T) { wanMpartVersion := mpart.Version _ = wanMpartVersion - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -831,7 +831,7 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header2 := "NONE,123" @@ -842,7 +842,7 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header3 := etag + ",123" @@ -853,3 +853,254 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) } + +func TestStateCorrectionEnabled(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== group 3 mesh ==== + subdocId = "mesh" + meshBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(meshBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, meshBytes) + + // ==== group 3 moca ==== + subdocId = "moca" + mocaBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(mocaBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, mocaBytes) + + // ==== GET /config ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 4) + etag := res.Header.Get(common.HeaderEtag) + + // parse the actual data + mpart, ok := mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + mpart, ok = mpartMap["mesh"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, meshBytes) + meshVersion := mpart.Version + + mpart, ok = mpartMap["moca"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, mocaBytes) + mocaVersion := mpart.Version + "x" + + // verify all states are in-deployment + lanSubdocument, err := server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.InDeployment) + + wanSubdocument, err := server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + + meshSubdocument, err := server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.InDeployment) + + mocaSubdocument, err := server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.InDeployment) + + // ==== setup special error conditions to test state correction scenario ==== + lanState := common.PendingDownload + lanSubdocument.SetState(&lanState) + err = server.SetSubDocument(cpeMac, "lan", lanSubdocument) + assert.NilError(t, err) + + wanState := common.InDeployment + wanSubdocument.SetState(&wanState) + err = server.SetSubDocument(cpeMac, "wan", wanSubdocument) + assert.NilError(t, err) + + meshState := common.Failure + meshSubdocument.SetState(&meshState) + err = server.SetSubDocument(cpeMac, "mesh", meshSubdocument) + assert.NilError(t, err) + + mocaState := common.PendingDownload + mocaSubdocument.SetState(&mocaState) + err = server.SetSubDocument(cpeMac, "moca", mocaSubdocument) + assert.NilError(t, err) + + // ==== call GET /config again with if-none-match and expect 304 ==== + server.SetStateCorrectionEnabled(false) + + configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan,mesh,moca", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + header1 := fmt.Sprintf("%v,%v,%v,%v,%v", etag, lanVersion, wanVersion, meshVersion, mocaVersion) + req.Header.Set(common.HeaderIfNoneMatch, header1) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // verify the states remain unchanged in the case of 304 + lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.PendingDownload) + + wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + + meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.Failure) + + mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + + // ==== enable the state correction flag and call GET /config again with if-none-match and expect 304 ==== + server.SetStateCorrectionEnabled(true) + defer func() { + server.SetStateCorrectionEnabled(false) + }() + + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, header1) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // verify all version-matched states remain unchanged in the case of 304 + lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.Deployed) + + wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.Deployed) + + meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.Deployed) + + mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) +} From 395cba1c0e7e6c0db1d64c2f454323e50e15489b Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 19 Mar 2024 13:36:33 -0700 Subject: [PATCH 020/155] add support of rfc subdoc bitmap --- common/const_var.go | 2 ++ http/supported_groups_handler_test.go | 3 +++ util/firmware_bitmap_test.go | 12 ++++++++++++ 3 files changed, 17 insertions(+) diff --git a/common/const_var.go b/common/const_var.go index e024f97..9206be1 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -153,6 +153,7 @@ var ( }, 8: { {1, 15}, + {2, 32}, }, 9: { {1, 16}, @@ -222,6 +223,7 @@ var ( "connectedbuilding": 29, "lldqoscontrol": 30, "clienttosteeringprofile": 31, + "rfc": 32, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 80ae1b1..51820cb 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -98,6 +98,7 @@ func TestSupportedGroupsHandler(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response @@ -283,6 +284,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response @@ -365,6 +367,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 4475183..841827d 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -163,6 +163,7 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -215,6 +216,8 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -266,6 +269,7 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -318,6 +322,7 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -380,6 +385,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -430,6 +436,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -480,6 +487,7 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -520,6 +528,7 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -569,6 +578,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -618,6 +628,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -667,6 +678,7 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From cb919c3b2dc26b4326ba1f290fa6d8e83777fc7a Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 18 Mar 2024 23:25:56 -0700 Subject: [PATCH 021/155] cherrypick and resolve a conflict --- common/subdocument.go | 16 ++ common/subdocument_test.go | 10 +- config/sample_webconfig.conf | 3 + db/cassandra/cassandra_client.go | 33 ++-- db/cassandra/cassandra_client_test.go | 8 + db/database_client.go | 4 + db/service.go | 31 +++ db/sqlite/sqlite_client.go | 22 ++- db/sqlite/sqlite_client_test.go | 43 +++++ http/document_handler_test.go | 2 +- http/multipart_test.go | 261 +++++++++++++++++++++++++- 11 files changed, 409 insertions(+), 24 deletions(-) create mode 100644 db/sqlite/sqlite_client_test.go diff --git a/common/subdocument.go b/common/subdocument.go index 336b22b..bd54947 100644 --- a/common/subdocument.go +++ b/common/subdocument.go @@ -102,6 +102,14 @@ func (d *SubDocument) SetVersion(version *string) { d.version = version } +// convenient function +func (d *SubDocument) GetVersion() string { + if d.version != nil { + return *d.version + } + return "" +} + func (d *SubDocument) State() *int { return d.state } @@ -110,6 +118,14 @@ func (d *SubDocument) SetState(state *int) { d.state = state } +// convenient function +func (d *SubDocument) GetState() int { + if d.state != nil { + return *d.state + } + return 0 +} + func (d *SubDocument) UpdatedTime() *int { return d.updatedTime } diff --git a/common/subdocument_test.go b/common/subdocument_test.go index 3b4bf1a..379ad1b 100644 --- a/common/subdocument_test.go +++ b/common/subdocument_test.go @@ -32,6 +32,12 @@ func TestSubDocumentString(t *testing.T) { errorCode := 103 errorDetails := "cannot parse" - doc := NewSubDocument(bbytes, &version, &state, &updatedTime, &errorCode, &errorDetails) - assert.Assert(t, doc != nil) + subdoc := NewSubDocument(bbytes, &version, &state, &updatedTime, &errorCode, &errorDetails) + assert.Assert(t, subdoc != nil) + + subdoc = &SubDocument{} + tgtVersion := subdoc.GetVersion() + assert.Equal(t, tgtVersion, "") + tgtState := subdoc.GetState() + assert.Equal(t, tgtState, 0) } diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index f47c838..894f513 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -232,4 +232,7 @@ webconfig { valid_partners = ["company", "vendor1", "vendor2"] supplementary_appending_enabled = true + + // correct subdoc states if versions match but not "deployed" + state_correction_enabled= false } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index d9ca1f7..7855b9f 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -48,10 +48,11 @@ type CassandraClient struct { *gocql.ClusterConfig *security.AesCodec *common.AppMetrics - concurrentQueries chan bool - localDc string - blockedSubdocIds []string - encryptedSubdocIds []string + concurrentQueries chan bool + localDc string + blockedSubdocIds []string + encryptedSubdocIds []string + stateCorrectionEnabled bool } /* @@ -159,15 +160,17 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") encryptedSubdocIds := conf.GetStringList("webconfig.encrypted_subdoc_ids") + stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") return &CassandraClient{ - Session: session, - ClusterConfig: cluster, - AesCodec: codec, - concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), - localDc: localDc, - blockedSubdocIds: blockedSubdocIds, - encryptedSubdocIds: encryptedSubdocIds, + Session: session, + ClusterConfig: cluster, + AesCodec: codec, + concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), + localDc: localDc, + blockedSubdocIds: blockedSubdocIds, + encryptedSubdocIds: encryptedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, }, nil } @@ -216,6 +219,14 @@ func (c *CassandraClient) SetEncryptedSubdocIds(x []string) { c.encryptedSubdocIds = x } +func (c *CassandraClient) StateCorrectionEnabled() bool { + return c.stateCorrectionEnabled +} + +func (c *CassandraClient) SetStateCorrectionEnabled(enabled bool) { + c.stateCorrectionEnabled = enabled +} + // TODO we hardcoded for now but it should be changed to be configurable func (c *CassandraClient) IsEncryptedGroup(subdocId string) bool { return util.Contains(c.EncryptedSubdocIds(), subdocId) diff --git a/db/cassandra/cassandra_client_test.go b/db/cassandra/cassandra_client_test.go index 57c1734..355b701 100644 --- a/db/cassandra/cassandra_client_test.go +++ b/db/cassandra/cassandra_client_test.go @@ -38,6 +38,14 @@ func TestCassandraClient(t *testing.T) { tgtSubdocIds := tdbclient.EncryptedSubdocIds() assert.Assert(t, len(tgtSubdocIds) == 4) + + // state correction flag + enabled := true + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + enabled = false + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) } func TestGetConfig(t *testing.T) { diff --git a/db/database_client.go b/db/database_client.go index b074e59..397c995 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -59,4 +59,8 @@ type DatabaseClient interface { FactoryReset(string) error FirmwareUpdate(string, int, *common.RootDocument) error AppendProfiles(string, []byte) ([]byte, error) + + // enable state correction + StateCorrectionEnabled() bool + SetStateCorrectionEnabled(bool) } diff --git a/db/service.go b/db/service.go index 2cc326d..ca77dc2 100644 --- a/db/service.go +++ b/db/service.go @@ -28,6 +28,7 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -138,6 +139,36 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field } } + if c.StateCorrectionEnabled() { + if document == nil { + document, err = c.GetDocument(mac, fields) + if err != nil { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + } + } + for subdocId, subdocument := range document.Items() { + cloudVersion := subdocument.GetVersion() + cloudState := subdocument.GetState() + if len(cloudVersion) == 0 { + continue + } + deviceVersion := deviceVersionMap[subdocId] + if cloudVersion == deviceVersion && cloudState >= common.PendingDownload && cloudState <= common.Failure { + labels := prometheus.Labels{ + "model": modelName, + "fwversion": firmwareVersion, + } + // update state + newState := common.Deployed + subdocument.SetState(&newState) + if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + } + } + } + + } + switch rootCmpEnum { case common.RootDocumentEquals: // create an empty "document" diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index d623d07..9ab3b12 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -43,8 +43,9 @@ type SqliteClient struct { db.BaseClient *sql.DB *common.AppMetrics - concurrentQueries chan bool - blockedSubdocIds []string + concurrentQueries chan bool + blockedSubdocIds []string + stateCorrectionEnabled bool } func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { @@ -58,15 +59,18 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") + stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + db, err := sql.Open("sqlite3", dbfile) if err != nil { return nil, common.NewError(err) } return &SqliteClient{ - DB: db, - concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), - blockedSubdocIds: blockedSubdocIds, + DB: db, + concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), + blockedSubdocIds: blockedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, }, nil } @@ -136,6 +140,14 @@ func (c *SqliteClient) SetBlockedSubdocIds(x []string) { c.blockedSubdocIds = x } +func (c *SqliteClient) StateCorrectionEnabled() bool { + return c.stateCorrectionEnabled +} + +func (c *SqliteClient) SetStateCorrectionEnabled(enabled bool) { + c.stateCorrectionEnabled = enabled +} + func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { if tdbclient != nil { return tdbclient, nil diff --git a/db/sqlite/sqlite_client_test.go b/db/sqlite/sqlite_client_test.go new file mode 100644 index 0000000..3a3f74f --- /dev/null +++ b/db/sqlite/sqlite_client_test.go @@ -0,0 +1,43 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "testing" + + "github.com/rdkcentral/webconfig/common" + "gotest.tools/assert" +) + +func TestSqliteClient(t *testing.T) { + configFile := "../../config/sample_webconfig.conf" + sc, err := common.GetTestServerConfig(configFile) + + assert.NilError(t, err) + dbc, err := GetTestSqliteClient(sc.Config, true) + assert.NilError(t, err) + assert.Assert(t, dbc != nil) + + // state correction flag + enabled := true + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + enabled = false + tdbclient.SetStateCorrectionEnabled(enabled) + assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) +} diff --git a/http/document_handler_test.go b/http/document_handler_test.go index f879914..7d11287 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -419,7 +419,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { subdocVersions = append(subdocVersions, mpart.Version) // ==== step 5 get document again ==== - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,gwrestore,remotedebugger,lan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) diff --git a/http/multipart_test.go b/http/multipart_test.go index 0dc6be5..30919f8 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -133,7 +133,7 @@ func TestMultipartConfigHandler(t *testing.T) { wanMpartVersion := mpart.Version _ = wanMpartVersion - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -144,7 +144,7 @@ func TestMultipartConfigHandler(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) - // ==== cal GET /config with if-none-match partial match ==== + // ==== call GET /config with if-none-match partial match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -819,7 +819,7 @@ func TestMultipartConfigMismatch(t *testing.T) { wanMpartVersion := mpart.Version _ = wanMpartVersion - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) @@ -831,7 +831,7 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header2 := "NONE,123" @@ -842,7 +842,7 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) - // ==== cal GET /config with if-none-match ==== + // ==== call GET /config with if-none-match ==== req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) header3 := etag + ",123" @@ -853,3 +853,254 @@ func TestMultipartConfigMismatch(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusNotModified) } + +func TestStateCorrectionEnabled(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== group 3 mesh ==== + subdocId = "mesh" + meshBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(meshBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, meshBytes) + + // ==== group 3 moca ==== + subdocId = "moca" + mocaBytes := util.RandomBytes(m, n) + assert.NilError(t, err) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(mocaBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, mocaBytes) + + // ==== GET /config ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 4) + etag := res.Header.Get(common.HeaderEtag) + + // parse the actual data + mpart, ok := mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + mpart, ok = mpartMap["mesh"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, meshBytes) + meshVersion := mpart.Version + + mpart, ok = mpartMap["moca"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, mocaBytes) + mocaVersion := mpart.Version + "x" + + // verify all states are in-deployment + lanSubdocument, err := server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.InDeployment) + + wanSubdocument, err := server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + + meshSubdocument, err := server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.InDeployment) + + mocaSubdocument, err := server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.InDeployment) + + // ==== setup special error conditions to test state correction scenario ==== + lanState := common.PendingDownload + lanSubdocument.SetState(&lanState) + err = server.SetSubDocument(cpeMac, "lan", lanSubdocument) + assert.NilError(t, err) + + wanState := common.InDeployment + wanSubdocument.SetState(&wanState) + err = server.SetSubDocument(cpeMac, "wan", wanSubdocument) + assert.NilError(t, err) + + meshState := common.Failure + meshSubdocument.SetState(&meshState) + err = server.SetSubDocument(cpeMac, "mesh", meshSubdocument) + assert.NilError(t, err) + + mocaState := common.PendingDownload + mocaSubdocument.SetState(&mocaState) + err = server.SetSubDocument(cpeMac, "moca", mocaSubdocument) + assert.NilError(t, err) + + // ==== call GET /config again with if-none-match and expect 304 ==== + server.SetStateCorrectionEnabled(false) + + configUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan,mesh,moca", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + header1 := fmt.Sprintf("%v,%v,%v,%v,%v", etag, lanVersion, wanVersion, meshVersion, mocaVersion) + req.Header.Set(common.HeaderIfNoneMatch, header1) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // verify the states remain unchanged in the case of 304 + lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.PendingDownload) + + wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + + meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.Failure) + + mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + + // ==== enable the state correction flag and call GET /config again with if-none-match and expect 304 ==== + server.SetStateCorrectionEnabled(true) + defer func() { + server.SetStateCorrectionEnabled(false) + }() + + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, header1) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) + + // verify all version-matched states remain unchanged in the case of 304 + lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") + assert.NilError(t, err) + assert.Equal(t, lanSubdocument.GetState(), common.Deployed) + + wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") + assert.NilError(t, err) + assert.Equal(t, wanSubdocument.GetState(), common.Deployed) + + meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") + assert.NilError(t, err) + assert.Equal(t, meshSubdocument.GetState(), common.Deployed) + + mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") + assert.NilError(t, err) + assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) +} From 26ce641b91ea1a51ccbc48c64679fec0587791e3 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 11 Apr 2024 23:02:04 -0700 Subject: [PATCH 022/155] Add support for defaultrfc bitmap --- common/const_var.go | 4 +- http/supported_groups_handler_test.go | 3 ++ util/firmware_bitmap_test.go | 62 +++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/common/const_var.go b/common/const_var.go index 9206be1..fdcda6f 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -154,6 +154,7 @@ var ( 8: { {1, 15}, {2, 32}, + {3, 33}, }, 9: { {1, 16}, @@ -223,7 +224,8 @@ var ( "connectedbuilding": 29, "lldqoscontrol": 30, "clienttosteeringprofile": 31, - "rfc": 32, + "defaultrfc": 32, + "rfc": 33, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 51820cb..d14dc56 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -99,6 +99,7 @@ func TestSupportedGroupsHandler(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response @@ -285,6 +286,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response @@ -368,6 +370,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "lldqoscontrol": true, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 841827d..f68affa 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -164,6 +164,7 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -217,6 +218,7 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -270,6 +272,7 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -323,6 +326,7 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -386,6 +390,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -437,6 +442,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -488,6 +494,7 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -529,6 +536,7 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -579,6 +587,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -629,6 +638,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "lldqoscontrol": true, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -679,6 +689,58 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "lldqoscontrol": true, "clienttosteeringprofile": true, "rfc": false, + "defaultrfc": false, + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + + parsedSupportedMap := GetSupportedMap(cpeBitmap) + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) +} + +func TestParseSupportedDocsHeaderRfc(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" + + // build expected + expectedEnabled := map[string]bool{ + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": true, + "lldqoscontrol": true, + "clienttosteeringprofile": true, + "rfc": true, + "defaultrfc": true, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From 84ff8a51ca8e47fe25255800b5f690c3b0a054a7 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 19 Apr 2024 15:44:34 -0700 Subject: [PATCH 023/155] add bitmaps for wifimotionsettings and xmspeedboost --- common/const_var.go | 10 +++++- http/supported_groups_handler_test.go | 12 +++++-- util/firmware_bitmap_test.go | 46 ++++++++++++++++++++------- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index fdcda6f..2fcf2af 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -128,7 +128,8 @@ var ( {4, 4}, {5, 5}, {6, 6}, - {7, 29}, + {7, 29}, // connectedbuilding + {8, 35}, // xmspeedboost }, 2: { {1, 7}, @@ -147,6 +148,11 @@ var ( 6: { {1, 13}, // mesh {2, 31}, // clienttosteeringprofile + {3, 36}, // meshsteeringprofiles + {4, 37}, // wifistatsconfig + {5, 38}, // mwoconfigs + {6, 39}, // interference + {7, 34}, // wifimotionsettings }, 7: { {1, 14}, @@ -226,6 +232,8 @@ var ( "clienttosteeringprofile": 31, "defaultrfc": 32, "rfc": 33, + "wifimotionsettings": 34, + "xmspeedboost": 35, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index d14dc56..13ffbe9 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -98,8 +98,10 @@ func TestSupportedGroupsHandler(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response @@ -285,8 +287,10 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response @@ -369,8 +373,10 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index f68affa..28c3159 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -163,8 +163,10 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -217,8 +219,10 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -271,8 +275,10 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -325,8 +331,10 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -389,8 +397,10 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -441,8 +451,10 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -493,8 +505,10 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -537,6 +551,8 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "clienttosteeringprofile": false, "rfc": false, "defaultrfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -586,8 +602,10 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -637,8 +655,10 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -688,8 +708,10 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -739,8 +761,10 @@ func TestParseSupportedDocsHeaderRfc(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, - "rfc": true, "defaultrfc": true, + "rfc": true, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From 02765e2f8f053df76a2d910259d10786080f50de Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 19 Mar 2024 13:36:33 -0700 Subject: [PATCH 024/155] add support of rfc subdoc bitmap --- common/const_var.go | 2 ++ http/supported_groups_handler_test.go | 3 +++ util/firmware_bitmap_test.go | 12 ++++++++++++ 3 files changed, 17 insertions(+) diff --git a/common/const_var.go b/common/const_var.go index 4370c7e..893bdf7 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -152,6 +152,7 @@ var ( }, 8: { {1, 15}, + {2, 32}, }, 9: { {1, 16}, @@ -221,6 +222,7 @@ var ( "connectedbuilding": 29, "lldqoscontrol": 30, "clienttosteeringprofile": 31, + "rfc": 32, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 80ae1b1..51820cb 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -98,6 +98,7 @@ func TestSupportedGroupsHandler(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response @@ -283,6 +284,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response @@ -365,6 +367,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, + "rfc": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 4475183..841827d 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -163,6 +163,7 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -215,6 +216,8 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -266,6 +269,7 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -318,6 +322,7 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -380,6 +385,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -430,6 +436,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -480,6 +487,7 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false + expectedEnabled["rfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -520,6 +528,7 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -569,6 +578,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -618,6 +628,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -667,6 +678,7 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, + "rfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From 437c523271b588d5864205a5aaae3b90a889a641 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 11 Apr 2024 23:02:04 -0700 Subject: [PATCH 025/155] Add support for defaultrfc bitmap --- common/const_var.go | 4 +- http/supported_groups_handler_test.go | 3 ++ util/firmware_bitmap_test.go | 62 +++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/common/const_var.go b/common/const_var.go index 893bdf7..a2fb8cc 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -153,6 +153,7 @@ var ( 8: { {1, 15}, {2, 32}, + {3, 33}, }, 9: { {1, 16}, @@ -222,7 +223,8 @@ var ( "connectedbuilding": 29, "lldqoscontrol": 30, "clienttosteeringprofile": 31, - "rfc": 32, + "defaultrfc": 32, + "rfc": 33, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 51820cb..d14dc56 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -99,6 +99,7 @@ func TestSupportedGroupsHandler(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response @@ -285,6 +286,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response @@ -368,6 +370,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "lldqoscontrol": true, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 841827d..f68affa 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -164,6 +164,7 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -217,6 +218,7 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -270,6 +272,7 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -323,6 +326,7 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -386,6 +390,7 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -437,6 +442,7 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -488,6 +494,7 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false expectedEnabled["rfc"] = false + expectedEnabled["defaultrfc"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -529,6 +536,7 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -579,6 +587,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "lldqoscontrol": false, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -629,6 +638,7 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "lldqoscontrol": true, "clienttosteeringprofile": false, "rfc": false, + "defaultrfc": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -679,6 +689,58 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "lldqoscontrol": true, "clienttosteeringprofile": true, "rfc": false, + "defaultrfc": false, + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + + parsedSupportedMap := GetSupportedMap(cpeBitmap) + assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) +} + +func TestParseSupportedDocsHeaderRfc(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" + + // build expected + expectedEnabled := map[string]bool{ + "advsecurity": true, + "aker": true, + "bridge": false, + "cellularconfig": false, + "gwfailover": true, + "homessid": true, + "hotspot": true, + "interfacereport": false, + "lan": true, + "macbinding": true, + "mesh": true, + "moca": true, + "portforwarding": true, + "privatessid": true, + "radio": false, + "radioreport": false, + "statusreport": false, + "telcovoice": false, + "telcovoip": false, + "telemetry": true, + "trafficreport": false, + "voiceservice": true, + "wan": true, + "wanfailover": true, + "wanmanager": false, + "xdns": true, + "gwrestore": false, + "prioritizedmacs": true, + "connectedbuilding": true, + "lldqoscontrol": true, + "clienttosteeringprofile": true, + "rfc": true, + "defaultrfc": true, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From 8720d84d2304d74a78d7e3433e82b81111bcc1d9 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 19 Apr 2024 15:44:34 -0700 Subject: [PATCH 026/155] add bitmaps for wifimotionsettings and xmspeedboost --- common/const_var.go | 10 +++++- http/supported_groups_handler_test.go | 12 +++++-- util/firmware_bitmap_test.go | 46 ++++++++++++++++++++------- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index a2fb8cc..8efae88 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -127,7 +127,8 @@ var ( {4, 4}, {5, 5}, {6, 6}, - {7, 29}, + {7, 29}, // connectedbuilding + {8, 35}, // xmspeedboost }, 2: { {1, 7}, @@ -146,6 +147,11 @@ var ( 6: { {1, 13}, // mesh {2, 31}, // clienttosteeringprofile + {3, 36}, // meshsteeringprofiles + {4, 37}, // wifistatsconfig + {5, 38}, // mwoconfigs + {6, 39}, // interference + {7, 34}, // wifimotionsettings }, 7: { {1, 14}, @@ -225,6 +231,8 @@ var ( "clienttosteeringprofile": 31, "defaultrfc": 32, "rfc": 33, + "wifimotionsettings": 34, + "xmspeedboost": 35, } ) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index d14dc56..13ffbe9 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -98,8 +98,10 @@ func TestSupportedGroupsHandler(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response @@ -285,8 +287,10 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response @@ -369,8 +373,10 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } // call GET /supported_groups to verify response diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index f68affa..28c3159 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -163,8 +163,10 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -217,8 +219,10 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -271,8 +275,10 @@ func TestBitmapParsing(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -325,8 +331,10 @@ func TestParseVoiceService(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -389,8 +397,10 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -441,8 +451,10 @@ func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -493,8 +505,10 @@ func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { expectedEnabled["connectedbuilding"] = false expectedEnabled["lldqoscontrol"] = false expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["rfc"] = false expectedEnabled["defaultrfc"] = false + expectedEnabled["rfc"] = false + expectedEnabled["wifimotionsettings"] = false + expectedEnabled["xmspeedboost"] = false assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) } @@ -537,6 +551,8 @@ func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { "clienttosteeringprofile": false, "rfc": false, "defaultrfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -586,8 +602,10 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { "connectedbuilding": false, "lldqoscontrol": false, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -637,8 +655,10 @@ func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *test "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": false, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -688,8 +708,10 @@ func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, - "rfc": false, "defaultrfc": false, + "rfc": false, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) @@ -739,8 +761,10 @@ func TestParseSupportedDocsHeaderRfc(t *testing.T) { "connectedbuilding": true, "lldqoscontrol": true, "clienttosteeringprofile": true, - "rfc": true, "defaultrfc": true, + "rfc": true, + "wifimotionsettings": false, + "xmspeedboost": false, } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) From 6b89ce52d8cd67f6f1df186fca13a5b3ed9b67b5 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 9 May 2024 23:20:02 -0700 Subject: [PATCH 027/155] do urlencode for xconf telemetry url and handle 400 response properly --- http/supplementary_handler.go | 7 ++---- http/supplementary_handler_test.go | 40 ++++++++++++++++++++++++++++++ util/string.go | 4 +++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index c4488c5..52e5660 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -77,11 +77,8 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - if rherr.StatusCode == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) - return - } - + Error(w, rherr.StatusCode, rherr) + return } Error(w, http.StatusInternalServerError, common.NewError(err)) return diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index ff08613..b7007e5 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -760,3 +760,43 @@ func TestSupplementaryAppendingFlag(t *testing.T) { // ==== step 5 verify the query params ==== assert.Assert(t, !strings.Contains(ss, "&stormReadyWifi=true")) } + +func TestSupplementaryApiBadRequest(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + validPartners := []string{"comcast", "comcast-dev", "cox", "rogers", "shaw", "sky-de", "sky-italia", "sky-italia-dev", "sky-roi", "sky-roi-dev", "sky-uk", "sky-uk-dev", "videotron"} + server.SetValidPartners(validPartners) + + // ==== setup mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Bad Request")) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "R NOT") + req.Header.Set(common.HeaderAccountID, "ERROR: ld.so: object '/usr/lib/libwayland-egl.so.0' from LD_PRE") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) +} diff --git a/util/string.go b/util/string.go index a582544..439b0bb 100644 --- a/util/string.go +++ b/util/string.go @@ -103,6 +103,10 @@ func GetTelemetryQueryString(header http.Header, mac, queryParams, partnerId str if len(queryParams) > 0 && len(ret) > 0 { ret += "&" + queryParams } + + ret = url.QueryEscape(ret) + ret = strings.ReplaceAll(ret, "%3D", "=") + ret = strings.ReplaceAll(ret, "%26", "&") return ret } From b2e372a59ee10b5b1e4b4e60146ca63e216b74ff Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 9 May 2024 23:20:02 -0700 Subject: [PATCH 028/155] do urlencode for xconf telemetry url and handle 400 response properly --- http/supplementary_handler.go | 7 ++---- http/supplementary_handler_test.go | 40 ++++++++++++++++++++++++++++++ util/string.go | 4 +++ 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index c4488c5..52e5660 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -77,11 +77,8 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - if rherr.StatusCode == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) - return - } - + Error(w, rherr.StatusCode, rherr) + return } Error(w, http.StatusInternalServerError, common.NewError(err)) return diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index ff08613..b7007e5 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -760,3 +760,43 @@ func TestSupplementaryAppendingFlag(t *testing.T) { // ==== step 5 verify the query params ==== assert.Assert(t, !strings.Contains(ss, "&stormReadyWifi=true")) } + +func TestSupplementaryApiBadRequest(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + validPartners := []string{"comcast", "comcast-dev", "cox", "rogers", "shaw", "sky-de", "sky-italia", "sky-italia-dev", "sky-roi", "sky-roi-dev", "sky-uk", "sky-uk-dev", "videotron"} + server.SetValidPartners(validPartners) + + // ==== setup mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Bad Request")) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "R NOT") + req.Header.Set(common.HeaderAccountID, "ERROR: ld.so: object '/usr/lib/libwayland-egl.so.0' from LD_PRE") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) +} diff --git a/util/string.go b/util/string.go index a582544..439b0bb 100644 --- a/util/string.go +++ b/util/string.go @@ -103,6 +103,10 @@ func GetTelemetryQueryString(header http.Header, mac, queryParams, partnerId str if len(queryParams) > 0 && len(ret) > 0 { ret += "&" + queryParams } + + ret = url.QueryEscape(ret) + ret = strings.ReplaceAll(ret, "%3D", "=") + ret = strings.ReplaceAll(ret, "%26", "&") return ret } From cc31901635c1cb29f02e979a6ce53f2b8d354a36 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Tue, 14 May 2024 16:44:31 -0700 Subject: [PATCH 029/155] Opentelemetry tracing option added Supports noop, stdouttracer and otlphttp tracer noop is the default, so no tracing will take place by default --- config/sample_webconfig.conf | 8 ++ go.mod | 38 +++++--- go.sum | 87 ++++++++++++----- http/otel.go | 178 +++++++++++++++++++++++++++++++++++ http/webconfig_server.go | 37 ++++++-- main.go | 1 + 6 files changed, 305 insertions(+), 44 deletions(-) create mode 100644 http/otel.go diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 894f513..a36a3d7 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -6,6 +6,14 @@ webconfig { panic_exit_enabled = false traceparent_parent_id = "0000000000000001" tracestate_vendor_id = "webconfig" + opentelemetry { + endpoint = "127.0.0.1:4318" + // Allowed values: "noop", "stdout", "http" + // "noop" will generate no trace + // "stdout" will use stdoutTracer and output spans to stdout + // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector + provider = "noop", + } // build info code_git_commit = "2ac7ff4" diff --git a/go.mod b/go.mod index bbdcee4..cfb4bb9 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt/v5 v5.0.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.0 github.com/mattn/go-sqlite3 v1.14.15 github.com/prometheus/client_golang v1.13.0 @@ -17,23 +17,32 @@ require ( github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 + go.opentelemetry.io/otel v1.26.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 + go.opentelemetry.io/otel/sdk v1.26.0 + go.opentelemetry.io/otel/trace v1.26.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.4.0 + golang.org/x/sync v0.6.0 gotest.tools v2.2.0+incompatible ) require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -44,19 +53,24 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/klauspost/compress v1.16.7 // indirect - github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index f502ae6..8505ce4 100644 --- a/go.sum +++ b/go.sum @@ -52,16 +52,18 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -90,6 +92,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= @@ -122,8 +129,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -138,8 +146,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -151,14 +159,16 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -199,8 +209,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -254,8 +264,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -273,8 +283,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -293,6 +303,22 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 h1:0W5o9SzoR15ocYHEQfvfipzcNog1lBxOLfnex91Hk6s= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0/go.mod h1:zVZ8nz+VSggWmnh6tTsJqXQ7rU4xLwRtna1M4x5jq58= +go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= +go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= @@ -307,8 +333,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -375,8 +401,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -394,8 +420,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -438,8 +464,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -450,7 +476,10 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -521,8 +550,8 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -552,6 +581,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= +google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -564,6 +599,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -576,8 +613,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http/otel.go b/http/otel.go new file mode 100644 index 0000000..6b3f838 --- /dev/null +++ b/http/otel.go @@ -0,0 +1,178 @@ +package http + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + + "github.com/go-akka/configuration" + + log "github.com/sirupsen/logrus" +) + +// Tracing contains the core dependencies to make tracing possible across an application. +type otelTracing struct { + providerName string + tracerProvider trace.TracerProvider + propagator propagation.TextMapPropagator +} + +type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) + +var ( + ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") + ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") + providersConfig = map[string]providerConstructor{ + "http": httpTraceProvider, + "stdout": stdoutTraceProvider, + "noop": noopTraceProvider, + } +) + +// DefaultTracerProvider is used when no provider is given. +// The Noop tracer provider turns all tracing related operations into +// noops essentially disabling tracing. +const defaultTracerProvider = "noop" + +// newOtel creates a structure with components that apps can use to initialize OpenTelemetry +// tracing instrumentation code. +func newOtel(conf *configuration.Config) (otelTracing, error) { + if IsNoOpTracing(conf) { + log.Debug("open telemetry tracing disabled (noop)") + } else { + log.Debug("opentelemetry tracing enabled") + } + + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + var tracing = otelTracing{ + providerName: providerName, + } + tracerProvider, err := newTracerProvider(conf) + if err != nil { + return otelTracing{}, err + } + tracing.tracerProvider = tracerProvider + otel.SetTracerProvider(tracerProvider) + + // Set up propagator. + prop := newPropagator() + tracing.propagator = prop + otel.SetTextMapPropagator(prop) + + return tracing, nil +} + +// IsNoOpTracing returns true if the provider is set to "noop" +func IsNoOpTracing(conf *configuration.Config) bool { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + return strings.EqualFold(providerName, "noop") +} + +// TracerProvider returns the tracer provider component. By default, the noop +// tracer provider is returned. +func (t otelTracing) TracerProvider() trace.TracerProvider { + if t.tracerProvider == nil { + return noop.NewTracerProvider() + } + return t.tracerProvider +} + +// Propagator returns the component that helps propagate trace context across +// API boundaries. By default, a W3C Trace Context format propagator is returned. +func (t otelTracing) Propagator() propagation.TextMapPropagator { + if t.propagator == nil { + return propagation.TraceContext{} + } + return t.propagator +} + +func newPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) +} + +// newTracerProvider creates the TracerProvider based on config setting +// If no config setting, a noop tracerProvider will be returned. +func newTracerProvider(conf *configuration.Config) (trace.TracerProvider, error) { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + if len(providerName) == 0 { + providerName = defaultTracerProvider + } + // Handling camelcase of provider. + providerName = strings.ToLower(providerName) + providerConfig := providersConfig[providerName] + if providerConfig == nil { + return nil, fmt.Errorf("%w for provider %s", ErrTracerProviderNotFound, providerName) + } + + traceProvider, err := providerConfig(conf) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + return traceProvider, nil +} + +func noopTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + return noop.NewTracerProvider(), nil +} + +func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + option := stdouttrace.WithPrettyPrint() + exporter, err := stdouttrace.New(option) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), + sdktrace.WithBatcher(exporter, + // Default is 5s. Set to 1s for demonstrative purposes. + sdktrace.WithBatchTimeout(time.Second)), + ) + return tp, nil +} + +func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + // Send traces over HTTP + endpoint := conf.GetString("webconfig.opentelemetry.endpoint") + if endpoint == "" { + return nil, ErrTracerProviderBuildFailed + } + exporter, err := otlptracehttp.New(context.Background(), + otlptracehttp.WithEndpoint(endpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + + appName := conf.GetString("webconfig.app_name") + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(appName), + ), + ), + ), nil +} + +func (s *WebconfigServer) OtelShutdown() { + sdkTraceProvider, ok := s.otelTracer.tracerProvider.(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 0d82f31..bd1385b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -28,6 +28,8 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/propagation" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" @@ -101,6 +103,7 @@ type WebconfigServer struct { traceparentParentID string tracestateVendorID string supplementaryAppendingEnabled bool + otelTracer *otelTracing // For OpenTelemetry Tracing } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -239,6 +242,11 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) + otelTracer, err := newOtel(conf) + if err != nil { + // Just log err and continue + log.Error("Could not initialize open telemetry for tracing, but continuing") + } supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) @@ -271,6 +279,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer jwksEnabled: jwksEnabled, traceparentParentID: traceparentParentID, tracestateVendorID: tracestateVendorID, + otelTracer: &otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } } @@ -308,8 +317,9 @@ func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { func (s *WebconfigServer) NoAuthMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw := s.logRequestStarts(w, r) + xw, ctx := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) + r = r.WithContext(ctx) next.ServeHTTP(xw, r) } return http.HandlerFunc(fn) @@ -318,8 +328,9 @@ func (s *WebconfigServer) NoAuthMiddleware(next http.Handler) http.Handler { // Token valid and mac must match func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw := s.logRequestStarts(w, r) + xw, ctx := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) + r = r.WithContext(ctx) isValid := false token := xw.Token() @@ -358,8 +369,9 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { // Token valid enough func (s *WebconfigServer) ApiMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw := s.logRequestStarts(w, r) + xw, ctx := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) + r = r.WithContext(ctx) isValid := false token := xw.Token() @@ -596,7 +608,7 @@ func getFilteredHeader(r *http.Request, notLoggedHeaders []string) http.Header { return header } -func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Request) *XResponseWriter { +func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Request) (*XResponseWriter, context.Context) { remoteIp := r.RemoteAddr host := r.Host header := getFilteredHeader(r, s.notLoggedHeaders) @@ -626,11 +638,12 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques outTraceparent = traceparent[:36] + s.TraceparentParentID() + traceparent[52:55] } - // extrac tracestate from the header + // extract tracestate from the header tracestate := r.Header.Get(common.HeaderTracestate) if len(tracestate) > 0 { outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) } + ctx := injectTrace(r, outTraceparent, outTracestate) // extract auditid from the header auditId := r.Header.Get("X-Auditid") @@ -697,7 +710,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if err != nil { fields["error"] = err log.WithFields(fields).Error("request starts") - return xwriter + return xwriter, ctx } xwriter.SetBodyBytes(bbytes) } @@ -708,7 +721,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques log.WithFields(tfields).Info("request starts") } - return xwriter + return xwriter, ctx } func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { @@ -843,3 +856,13 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { } return itf, "" } + +func injectTrace(r *http.Request, traceparent string, tracestate string) (context.Context) { + propagator := propagation.TraceContext{} + ctx := r.Context() + ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) + var textMapCarrier propagation.TextMapCarrier = propagation.MapCarrier{} + textMapCarrier.Set(common.HeaderTraceparent, traceparent) + textMapCarrier.Set(common.HeaderTracestate, tracestate) + return propagation.TraceContext{}.Extract(ctx, textMapCarrier) +} diff --git a/main.go b/main.go index 11c8893..d5684ef 100644 --- a/main.go +++ b/main.go @@ -60,6 +60,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) + defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From 315f2d890b1cc13bbe08213bff498cdb513cb625 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 1 Jun 2024 17:29:31 -0700 Subject: [PATCH 030/155] Fix a bug that poke api returns 500 if webpa returns 403 --- common/server_config.go | 1 + http/otel.go | 17 +++++++++++++++++ http/poke_handler.go | 4 +--- http/poke_handler_test.go | 36 +++++++++++++++++++++++++++++++++--- http/webconfig_server.go | 4 ++-- http/webpa_connector.go | 14 +++++++++++--- 6 files changed, 65 insertions(+), 11 deletions(-) diff --git a/common/server_config.go b/common/server_config.go index 44fa5a7..0873776 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -30,6 +30,7 @@ var ( "/app/webconfig/test_webconfig.conf", "../config/sample_webconfig.conf", "/app/webconfig/webconfig.conf", + "/app/webconfig/conf/webconfig.conf", } ) diff --git a/http/otel.go b/http/otel.go index 6b3f838..3028948 100644 --- a/http/otel.go +++ b/http/otel.go @@ -1,3 +1,20 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ package http import ( diff --git a/http/poke_handler.go b/http/poke_handler.go index 62c4092..ac50ba5 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -161,11 +161,9 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - status := http.StatusInternalServerError + status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 - } else if rherr.StatusCode > http.StatusInternalServerError { - status = rherr.StatusCode } // parse the core message diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 29c3364..ba63157 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -35,7 +35,8 @@ import ( ) var ( - mockedWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) ) func TestPokeHandler(t *testing.T) { @@ -47,7 +48,7 @@ func TestPokeHandler(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -286,7 +287,7 @@ func TestPokeHandlerInvalidMac(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -305,3 +306,32 @@ func TestPokeHandlerInvalidMac(t *testing.T) { assert.NilError(t, err) res.Body.Close() } + +func TestPokeHandlerWebpa403(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write(mockWebpaPoke403Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index bd1385b..c2dc929 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -857,8 +857,8 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func injectTrace(r *http.Request, traceparent string, tracestate string) (context.Context) { - propagator := propagation.TraceContext{} +func injectTrace(r *http.Request, traceparent string, tracestate string) context.Context { + propagator := propagation.TraceContext{} ctx := r.Context() ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) var textMapCarrier propagation.TextMapCarrier = propagation.MapCarrier{} diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 021f049..d7681f4 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -251,18 +251,26 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field } func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { - var cont bool - + tfields := common.FilterLogFields(fields) + tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { cbytes := make([]byte, len(bbytes)) copy(cbytes, bbytes) if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont = c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, _, cont := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { + msg := fmt.Sprintf("finished success after 1 retry") + if i > 1 { + fmt.Sprintf("finished success after %v retries", i) + } + log.WithFields(tfields).Info(msg) break } + if i == c.retries { + log.WithFields(tfields).Infof("finished failure after %v retries", i) + } } <-c.queue } From af7d8c83ab467c985a84e7e86dc2f7cc8a17c536 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 1 Jun 2024 17:29:31 -0700 Subject: [PATCH 031/155] cherrypick a poke403 handling fix and resolve conflicts --- common/server_config.go | 1 + http/poke_handler.go | 4 +--- http/poke_handler_test.go | 36 +++++++++++++++++++++++++++++++++--- http/webpa_connector.go | 14 +++++++++++--- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/common/server_config.go b/common/server_config.go index 44fa5a7..0873776 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -30,6 +30,7 @@ var ( "/app/webconfig/test_webconfig.conf", "../config/sample_webconfig.conf", "/app/webconfig/webconfig.conf", + "/app/webconfig/conf/webconfig.conf", } ) diff --git a/http/poke_handler.go b/http/poke_handler.go index 62c4092..ac50ba5 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -161,11 +161,9 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - status := http.StatusInternalServerError + status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 - } else if rherr.StatusCode > http.StatusInternalServerError { - status = rherr.StatusCode } // parse the core message diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 29c3364..ba63157 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -35,7 +35,8 @@ import ( ) var ( - mockedWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) ) func TestPokeHandler(t *testing.T) { @@ -47,7 +48,7 @@ func TestPokeHandler(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -286,7 +287,7 @@ func TestPokeHandlerInvalidMac(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -305,3 +306,32 @@ func TestPokeHandlerInvalidMac(t *testing.T) { assert.NilError(t, err) res.Body.Close() } + +func TestPokeHandlerWebpa403(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write(mockWebpaPoke403Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 021f049..d7681f4 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -251,18 +251,26 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field } func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { - var cont bool - + tfields := common.FilterLogFields(fields) + tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { cbytes := make([]byte, len(bbytes)) copy(cbytes, bbytes) if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont = c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, _, cont := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { + msg := fmt.Sprintf("finished success after 1 retry") + if i > 1 { + fmt.Sprintf("finished success after %v retries", i) + } + log.WithFields(tfields).Info(msg) break } + if i == c.retries { + log.WithFields(tfields).Infof("finished failure after %v retries", i) + } } <-c.queue } From 123411a0d2aa40b8d80d9f60d56808b37339af9f Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Thu, 13 Jun 2024 05:49:40 -0700 Subject: [PATCH 032/155] Otel Changes --- config/sample_webconfig.conf | 11 ++--- go.mod | 77 +++++++++++++++++----------------- go.sum | 80 ++++++++++++++++++++++++++++++++++++ http/http_client.go | 36 +++++++++------- http/otel.go | 37 +++++++++++++---- http/webconfig_server.go | 16 ++++++-- 6 files changed, 188 insertions(+), 69 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index a36a3d7..fbfc38b 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -8,11 +8,12 @@ webconfig { tracestate_vendor_id = "webconfig" opentelemetry { endpoint = "127.0.0.1:4318" - // Allowed values: "noop", "stdout", "http" - // "noop" will generate no trace - // "stdout" will use stdoutTracer and output spans to stdout - // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector - provider = "noop", + // Allowed values: "noop", "stdout", "http" + // "noop" will generate no trace + // "stdout" will use stdoutTracer and output spans to stdout + // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector + provider = "noop" + env_name = "dev" } // build info diff --git a/go.mod b/go.mod index cfb4bb9..188e4df 100644 --- a/go.mod +++ b/go.mod @@ -3,48 +3,51 @@ module github.com/rdkcentral/webconfig go 1.21 require ( - github.com/IBM/sarama v1.42.1 + github.com/IBM/sarama v1.43.2 github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 - github.com/golang-jwt/jwt/v5 v5.0.0 + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 - github.com/gorilla/mux v1.8.0 - github.com/mattn/go-sqlite3 v1.14.15 - github.com/prometheus/client_golang v1.13.0 - github.com/prometheus/client_model v0.2.0 - github.com/sirupsen/logrus v1.9.0 - github.com/twmb/murmur3 v1.1.6 + github.com/gorilla/mux v1.8.1 + github.com/mattn/go-sqlite3 v1.14.22 + github.com/prometheus/client_golang v1.19.1 + github.com/prometheus/client_model v0.6.1 + github.com/sirupsen/logrus v1.9.3 + github.com/twmb/murmur3 v1.1.8 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/otel v1.26.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 - go.opentelemetry.io/otel/sdk v1.26.0 - go.opentelemetry.io/otel/trace v1.26.0 - go.uber.org/automaxprocs v1.5.1 - go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.6.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 + go.uber.org/automaxprocs v1.5.3 + go.uber.org/ratelimit v0.3.1 + golang.org/x/sync v0.7.0 gotest.tools v2.2.0+incompatible ) require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.4.0 // indirect + github.com/eapache/go-resiliency v1.6.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -52,25 +55,25 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.54.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 // indirect - go.opentelemetry.io/otel/metric v1.26.0 // indirect - go.opentelemetry.io/proto/otlp v1.2.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/grpc v1.63.2 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index 8505ce4..9e2917d 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= +github.com/IBM/sarama v1.43.2 h1:HABeEqRUh32z8yzY2hGB/j8mHSzC/HA9zlEjqFNCzSw= +github.com/IBM/sarama v1.43.2/go.mod h1:Kyo4WkF24Z+1nz7xeVUFWIuKVV8RS3wM8mkvPKMdXFQ= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -44,6 +46,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -59,6 +63,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -69,6 +75,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30= +github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= @@ -77,6 +85,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= @@ -95,6 +105,8 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -103,6 +115,8 @@ github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJr github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -165,14 +179,20 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -205,6 +225,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -217,8 +239,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -228,6 +254,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -243,17 +271,23 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= +github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -261,6 +295,8 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -271,6 +307,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -287,6 +325,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= @@ -303,28 +343,50 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 h1:0W5o9SzoR15ocYHEQfvfipzcNog1lBxOLfnex91Hk6s= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0/go.mod h1:zVZ8nz+VSggWmnh6tTsJqXQ7rU4xLwRtna1M4x5jq58= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= +go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= +go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -335,6 +397,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -403,6 +467,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -422,6 +488,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -466,6 +534,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -480,6 +550,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -585,8 +657,12 @@ google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUE google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= +google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 h1:QW9+G6Fir4VcRXVH8x3LilNAb6cxBGLa6+GM4hRwexE= +google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3/go.mod h1:kdrSS/OiLkPrNUpzD4aHgCq2rVuC/YRxok32HXZ4vRE= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -601,6 +677,8 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -615,6 +693,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http/http_client.go b/http/http_client.go index dfc0459..c6cc70b 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -32,11 +32,13 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -82,20 +84,26 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) userAgent := conf.GetString("webconfig.http_client.user_agent") + var transport http.RoundTripper = &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: time.Duration(connectTimeout) * time.Second, + KeepAlive: time.Duration(keepaliveTimeout) * time.Second, + }).DialContext, + MaxIdleConns: 0, + MaxIdleConnsPerHost: maxIdleConnsPerHost, + IdleConnTimeout: time.Duration(keepaliveTimeout) * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: tlsConfig, + } + transport = otelhttp.NewTransport(transport, + otelhttp.WithPropagators(otelTracer.propagator), + otelhttp.WithTracerProvider(otelTracer.tracerProvider), + ) + return &HttpClient{ Client: &http.Client{ - Transport: &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: time.Duration(connectTimeout) * time.Second, - KeepAlive: time.Duration(keepaliveTimeout) * time.Second, - }).DialContext, - MaxIdleConns: 0, - MaxIdleConnsPerHost: maxIdleConnsPerHost, - IdleConnTimeout: time.Duration(keepaliveTimeout) * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - TLSClientConfig: tlsConfig, - }, + Transport: transport, Timeout: time.Duration(readTimeout) * time.Second, }, retries: retries, diff --git a/http/otel.go b/http/otel.go index 6b3f838..920bebd 100644 --- a/http/otel.go +++ b/http/otel.go @@ -1,3 +1,20 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ package http import ( @@ -25,6 +42,7 @@ import ( // Tracing contains the core dependencies to make tracing possible across an application. type otelTracing struct { providerName string + envName string tracerProvider trace.TracerProvider propagator propagation.TextMapPropagator } @@ -39,6 +57,8 @@ var ( "stdout": stdoutTraceProvider, "noop": noopTraceProvider, } + + otelTracer otelTracing ) // DefaultTracerProvider is used when no provider is given. @@ -48,30 +68,28 @@ const defaultTracerProvider = "noop" // newOtel creates a structure with components that apps can use to initialize OpenTelemetry // tracing instrumentation code. -func newOtel(conf *configuration.Config) (otelTracing, error) { +func newOtel(conf *configuration.Config) (*otelTracing, error) { if IsNoOpTracing(conf) { log.Debug("open telemetry tracing disabled (noop)") } else { log.Debug("opentelemetry tracing enabled") } - providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - var tracing = otelTracing{ - providerName: providerName, - } + otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") tracerProvider, err := newTracerProvider(conf) if err != nil { - return otelTracing{}, err + return &otelTracer, err } - tracing.tracerProvider = tracerProvider + otelTracer.tracerProvider = tracerProvider otel.SetTracerProvider(tracerProvider) // Set up propagator. prop := newPropagator() - tracing.propagator = prop + otelTracer.propagator = prop otel.SetTextMapPropagator(prop) - return tracing, nil + return &otelTracer, nil } // IsNoOpTracing returns true if the provider is set to "noop" @@ -165,6 +183,7 @@ func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), ), ), ), nil diff --git a/http/webconfig_server.go b/http/webconfig_server.go index bd1385b..a85c096 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -28,7 +28,9 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" @@ -279,7 +281,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer jwksEnabled: jwksEnabled, traceparentParentID: traceparentParentID, tracestateVendorID: tracestateVendorID, - otelTracer: &otelTracer, + otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } } @@ -371,8 +373,8 @@ func (s *WebconfigServer) ApiMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { xw, ctx := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) - r = r.WithContext(ctx) + r = r.WithContext(ctx) isValid := false token := xw.Token() if len(token) > 0 { @@ -857,12 +859,18 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func injectTrace(r *http.Request, traceparent string, tracestate string) (context.Context) { - propagator := propagation.TraceContext{} +func injectTrace(r *http.Request, traceparent string, tracestate string) context.Context { ctx := r.Context() + + propagator := propagation.TraceContext{} ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) var textMapCarrier propagation.TextMapCarrier = propagation.MapCarrier{} textMapCarrier.Set(common.HeaderTraceparent, traceparent) textMapCarrier.Set(common.HeaderTracestate, tracestate) + + span := trace.SpanFromContext(ctx) + attr := attribute.String("env", otelTracer.envName) + span.SetAttributes(attr) + return propagation.TraceContext{}.Extract(ctx, textMapCarrier) } From 7fdcc9b808e13d9d152e1d8fe94a89149035a07c Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Thu, 13 Jun 2024 14:39:49 -0700 Subject: [PATCH 033/155] Otel changes second iteration Restrict opentelemetry to poke only Instrument the API using middleware instead of instrumenting the client library --- http/otel.go | 14 ++++++-- http/router.go | 1 + http/webconfig_server.go | 77 +++++++++++++++++++++++++++++++++++----- 3 files changed, 81 insertions(+), 11 deletions(-) diff --git a/http/otel.go b/http/otel.go index 920bebd..1faa742 100644 --- a/http/otel.go +++ b/http/otel.go @@ -43,8 +43,10 @@ import ( type otelTracing struct { providerName string envName string + appName string tracerProvider trace.TracerProvider propagator propagation.TextMapPropagator + tracer trace.Tracer } type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) @@ -75,6 +77,7 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { log.Debug("opentelemetry tracing enabled") } + otelTracer.appName = conf.GetString("webconfig.app_name") otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") tracerProvider, err := newTracerProvider(conf) @@ -89,6 +92,7 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { otelTracer.propagator = prop otel.SetTextMapPropagator(prop) + otelTracer.tracer = otel.Tracer(otelTracer.appName) return &otelTracer, nil } @@ -158,6 +162,13 @@ func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, erro sdktrace.WithBatcher(exporter, // Default is 5s. Set to 1s for demonstrative purposes. sdktrace.WithBatchTimeout(time.Second)), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(otelTracer.appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), + ), + ), ) return tp, nil } @@ -176,13 +187,12 @@ func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) } - appName := conf.GetString("webconfig.app_name") return sdktrace.NewTracerProvider( sdktrace.WithBatcher(exporter), sdktrace.WithResource( resource.NewWithAttributes( semconv.SchemaURL, - semconv.ServiceNameKey.String(appName), + semconv.ServiceNameKey.String(otelTracer.appName), semconv.ServiceNamespaceKey.String(otelTracer.envName), ), ), diff --git a/http/router.go b/http/router.go index 746bec0..c6ddaf1 100644 --- a/http/router.go +++ b/http/router.go @@ -91,6 +91,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() + sub2.Use(spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { diff --git a/http/webconfig_server.go b/http/webconfig_server.go index a85c096..2069acb 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -20,6 +20,7 @@ package http import ( "context" "crypto/tls" + "encoding/hex" "encoding/json" "fmt" "io" @@ -29,7 +30,6 @@ import ( "time" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" "github.com/rdkcentral/webconfig/common" @@ -77,6 +77,7 @@ var ( "privatessid", "homessid", } + ws *WebconfigServer ) type WebconfigServer struct { @@ -252,7 +253,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) - return &WebconfigServer{ + ws = &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, @@ -284,6 +285,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } + return ws } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -645,7 +647,8 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if len(tracestate) > 0 { outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) } - ctx := injectTrace(r, outTraceparent, outTracestate) + ctx := r.Context() + // ctx := injectTrace(r, outTraceparent, outTracestate) // extract auditid from the header auditId := r.Header.Get("X-Auditid") @@ -859,18 +862,74 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } +/* func injectTrace(r *http.Request, traceparent string, tracestate string) context.Context { ctx := r.Context() - propagator := propagation.TraceContext{} - ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) var textMapCarrier propagation.TextMapCarrier = propagation.MapCarrier{} textMapCarrier.Set(common.HeaderTraceparent, traceparent) textMapCarrier.Set(common.HeaderTracestate, tracestate) - span := trace.SpanFromContext(ctx) - attr := attribute.String("env", otelTracer.envName) - span.SetAttributes(attr) + propagator := propagation.TraceContext{} + return propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) + // return propagator.Extract(ctx, textMapCarrier) +} +*/ + +func spanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + spanContext := trace.SpanContextFromContext(ctx) + remote := spanContext.IsRemote() + sc := trace.SpanContext{}.WithRemote(remote) + + traceIDStr, traceFlagsStr := parseTraceparent(r) + if traceIDStr != "" { + traceID, _ := trace.TraceIDFromHex(traceIDStr) + sc = sc.WithTraceID(traceID) + } + if traceFlagsStr != "" { + traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) + sc = sc.WithTraceFlags(traceFlags) + } + tracestateStr := getTracestate(r) + if tracestateStr != "" { + tracestate, _ := trace.ParseTraceState(tracestateStr) + sc = sc.WithTraceState(tracestate) + } + ctx = trace.ContextWithSpanContext(ctx, sc) + ctx, span := otelTracer.tracer.Start(ctx, "mytest") + defer span.End() + + attr := attribute.String("env", otelTracer.envName) + span.SetAttributes(attr) + + // Pass the context with the span to the next handler + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// extract traceparent from the header +func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { + inTraceparent := r.Header.Get(common.HeaderTraceparent) + if len(inTraceparent) == 55 { + traceID = inTraceparent[3:35] + traceFlags = inTraceparent[53:55] + } + return +} + +// extract tracestate from the header +func getTracestate(r *http.Request) string { + inTracestate := r.Header.Get(common.HeaderTracestate) + var outTracestate string + if len(inTracestate) > 0 { + outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, ws.TracestateVendorID(), ws.TraceparentParentID()) + } + return outTracestate +} - return propagation.TraceContext{}.Extract(ctx, textMapCarrier) +func hexStringToBytes(hexString string) []byte { + bytes, _ := hex.DecodeString(hexString) + return bytes } From e74912d0a4379be4e73dea04ddc2d143ca7dc690 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Thu, 13 Jun 2024 16:00:24 -0700 Subject: [PATCH 034/155] Code review feedback plus minor simplifications --- http/webconfig_server.go | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 2069acb..3a6082b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -321,9 +321,8 @@ func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { func (s *WebconfigServer) NoAuthMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw, ctx := s.logRequestStarts(w, r) + xw := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) - r = r.WithContext(ctx) next.ServeHTTP(xw, r) } return http.HandlerFunc(fn) @@ -332,9 +331,8 @@ func (s *WebconfigServer) NoAuthMiddleware(next http.Handler) http.Handler { // Token valid and mac must match func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw, ctx := s.logRequestStarts(w, r) + xw := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) - r = r.WithContext(ctx) isValid := false token := xw.Token() @@ -373,10 +371,9 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { // Token valid enough func (s *WebconfigServer) ApiMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { - xw, ctx := s.logRequestStarts(w, r) + xw := s.logRequestStarts(w, r) defer s.logRequestEnds(xw, r) - r = r.WithContext(ctx) isValid := false token := xw.Token() if len(token) > 0 { @@ -612,7 +609,7 @@ func getFilteredHeader(r *http.Request, notLoggedHeaders []string) http.Header { return header } -func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Request) (*XResponseWriter, context.Context) { +func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Request) *XResponseWriter { remoteIp := r.RemoteAddr host := r.Host header := getFilteredHeader(r, s.notLoggedHeaders) @@ -647,9 +644,6 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if len(tracestate) > 0 { outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) } - ctx := r.Context() - // ctx := injectTrace(r, outTraceparent, outTracestate) - // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { @@ -715,7 +709,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques if err != nil { fields["error"] = err log.WithFields(fields).Error("request starts") - return xwriter, ctx + return xwriter } xwriter.SetBodyBytes(bbytes) } @@ -726,7 +720,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques log.WithFields(tfields).Info("request starts") } - return xwriter, ctx + return xwriter } func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { @@ -862,20 +856,6 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -/* -func injectTrace(r *http.Request, traceparent string, tracestate string) context.Context { - ctx := r.Context() - - var textMapCarrier propagation.TextMapCarrier = propagation.MapCarrier{} - textMapCarrier.Set(common.HeaderTraceparent, traceparent) - textMapCarrier.Set(common.HeaderTracestate, tracestate) - - propagator := propagation.TraceContext{} - return propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) - // return propagator.Extract(ctx, textMapCarrier) -} -*/ - func spanMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -898,7 +878,7 @@ func spanMiddleware(next http.Handler) http.Handler { sc = sc.WithTraceState(tracestate) } ctx = trace.ContextWithSpanContext(ctx, sc) - ctx, span := otelTracer.tracer.Start(ctx, "mytest") + ctx, span := otelTracer.tracer.Start(ctx, "oswebconfig_poke_handler") defer span.End() attr := attribute.String("env", otelTracer.envName) From b0f9841116097eb2b76783fdddf0e24ea47e93ee Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 15 Jun 2024 12:00:33 -0700 Subject: [PATCH 035/155] handle corrupted encrypted blobs --- db/cassandra/document.go | 7 +- http/document_handler.go | 1 + http/http_client.go | 7 +- http/multipart_test.go | 164 +++++++++++++++++++++++++++++ http/otel.go | 10 +- http/router.go | 2 +- http/supplementary_handler_test.go | 3 - http/webconfig_server.go | 12 +-- 8 files changed, 184 insertions(+), 22 deletions(-) diff --git a/db/cassandra/document.go b/db/cassandra/document.go index 2eaf58c..df580b8 100644 --- a/db/cassandra/document.go +++ b/db/cassandra/document.go @@ -29,7 +29,6 @@ import ( log "github.com/sirupsen/logrus" ) -// NOTE this func (c *CassandraClient) GetSubDocument(cpeMac string, groupId string) (*common.SubDocument, error) { var err error var payload []byte @@ -283,7 +282,11 @@ func (c *CassandraClient) GetDocument(cpeMac string, xargs ...interface{}) (fndo if c.IsEncryptedGroup(groupId) { payload, err = c.DecryptBytes(payload) if err != nil { - return nil, common.NewError(err) + tfields := common.FilterLogFields(fields) + tfields["logger"] = "subdoc" + tfields["subdoc_id"] = groupId + log.WithFields(tfields).Warn(err) + continue } } diff --git a/http/document_handler.go b/http/document_handler.go index f41f7ff..3f3e834 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -301,6 +301,7 @@ func (s *WebconfigServer) DeleteDocumentHandler(w http.ResponseWriter, r *http.R if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) } + WriteOkResponse(w, nil) } else { Error(w, http.StatusInternalServerError, common.NewError(err)) } diff --git a/http/http_client.go b/http/http_client.go index c6cc70b..790d907 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -32,13 +32,12 @@ import ( "strings" "time" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" log "github.com/sirupsen/logrus" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" ) const ( @@ -104,7 +103,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl return &HttpClient{ Client: &http.Client{ Transport: transport, - Timeout: time.Duration(readTimeout) * time.Second, + Timeout: time.Duration(readTimeout) * time.Second, }, retries: retries, retryInMsecs: retryInMsecs, diff --git a/http/multipart_test.go b/http/multipart_test.go index 30919f8..2aed59a 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -28,6 +28,7 @@ import ( "testing" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" "github.com/vmihailenco/msgpack/v4" @@ -1104,3 +1105,166 @@ func TestStateCorrectionEnabled(t *testing.T) { assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) } + +func TestCorruptedEncryptedDocumentHandler(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + tdbclient, ok := server.DatabaseClient.(*cassandra.CassandraClient) + if !ok { + t.Skip("Only test in cassandra env") + } + + cpeMac := util.GenerateRandomCpeMac() + encSubdocIds := []string{} + tdbclient.SetEncryptedSubdocIds(encSubdocIds) + readSubDocIds := tdbclient.EncryptedSubdocIds() + assert.DeepEqual(t, encSubdocIds, readSubDocIds) + assert.Assert(t, !tdbclient.IsEncryptedGroup("privatessid")) + + // ==== step 1 setup lan subdoc ==== + // post + subdocId := "lan" + lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + lanBytes := util.RandomBytes(100, 150) + req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + _ = rbytes + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", lanUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 2 setup wan subdoc ==== + // post + subdocId = "wan" + wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + wanBytes := util.RandomBytes(100, 150) + req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", wanUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 3 setup privatessid subdoc ==== + // post + subdocId = "privatessid" + privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + privatessidBytes := util.RandomBytes(100, 150) + req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", privatessidUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidBytes) + + // ==== step 4 read the document ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 3) + + // parse the actual data + mpart, ok := mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + mpart, ok = mpartMap["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidBytes) + + // ==== step 5 set privatessid as an encrypted subdoc ==== + encSubdocIds = []string{"privatessid"} + tdbclient.SetEncryptedSubdocIds(encSubdocIds) + readSubDocIds = tdbclient.EncryptedSubdocIds() + assert.DeepEqual(t, encSubdocIds, readSubDocIds) + assert.Assert(t, tdbclient.IsEncryptedGroup("privatessid")) + + // ==== step 6 read the document expect no error but 1 less subdoc ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok = mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + _, ok = mpartMap["privatessid"] + assert.Assert(t, !ok) +} diff --git a/http/otel.go b/http/otel.go index 1faa742..e0a806e 100644 --- a/http/otel.go +++ b/http/otel.go @@ -27,10 +27,10 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" @@ -45,7 +45,7 @@ type otelTracing struct { envName string appName string tracerProvider trace.TracerProvider - propagator propagation.TextMapPropagator + propagator propagation.TextMapPropagator tracer trace.Tracer } @@ -54,10 +54,10 @@ type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, var ( ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") - providersConfig = map[string]providerConstructor{ - "http": httpTraceProvider, + providersConfig = map[string]providerConstructor{ + "http": httpTraceProvider, "stdout": stdoutTraceProvider, - "noop": noopTraceProvider, + "noop": noopTraceProvider, } otelTracer otelTracing diff --git a/http/router.go b/http/router.go index c6ddaf1..bc7b0d6 100644 --- a/http/router.go +++ b/http/router.go @@ -91,7 +91,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() - sub2.Use(spanMiddleware) + sub2.Use(s.spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index b7007e5..942b0f8 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -732,9 +732,6 @@ func TestSupplementaryAppendingFlag(t *testing.T) { // ==== step 3 verify the query params ==== assert.Assert(t, strings.Contains(ss, "&stormReadyWifi=true")) - // okok := false - // assert.Assert(t, okok) - // ==== step 4 set append flag false ==== appendEnabled = false server.SetSupplementaryAppendingEnabled(appendEnabled) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 3a6082b..e9800bb 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -77,7 +77,6 @@ var ( "privatessid", "homessid", } - ws *WebconfigServer ) type WebconfigServer struct { @@ -253,7 +252,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) - ws = &WebconfigServer{ + return &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, @@ -285,7 +284,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } - return ws } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -856,7 +854,7 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func spanMiddleware(next http.Handler) http.Handler { +func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() spanContext := trace.SpanContextFromContext(ctx) @@ -872,7 +870,7 @@ func spanMiddleware(next http.Handler) http.Handler { traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) sc = sc.WithTraceFlags(traceFlags) } - tracestateStr := getTracestate(r) + tracestateStr := s.getTracestate(r) if tracestateStr != "" { tracestate, _ := trace.ParseTraceState(tracestateStr) sc = sc.WithTraceState(tracestate) @@ -900,11 +898,11 @@ func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { } // extract tracestate from the header -func getTracestate(r *http.Request) string { +func (s *WebconfigServer) getTracestate(r *http.Request) string { inTracestate := r.Header.Get(common.HeaderTracestate) var outTracestate string if len(inTracestate) > 0 { - outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, ws.TracestateVendorID(), ws.TraceparentParentID()) + outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, s.TracestateVendorID(), s.TraceparentParentID()) } return outTracestate } From 6062b63266350f338e838ee08e34fa07141b95cb Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Mon, 17 Jun 2024 11:52:07 -0700 Subject: [PATCH 036/155] Cherrypick of otel changes --- common/const_var.go | 1 + common/refsubdocument.go | 81 +++++++++++ common/refsubdocument_test.go | 40 ++++++ common/server_config.go | 1 + config/sample_webconfig.conf | 9 ++ db/cassandra/refsubdocument.go | 81 +++++++++++ db/cassandra/refsubdocument_test.go | 59 ++++++++ db/cassandra/schema.go | 5 + db/database_client.go | 5 + db/service.go | 36 +++++ db/sqlite/refsubdocument.go | 153 ++++++++++++++++++++ db/sqlite/refsubdocument_test.go | 59 ++++++++ db/sqlite/schema.go | 5 + go.mod | 40 ++++-- go.sum | 89 ++++++++---- http/http_client.go | 36 +++-- http/multipart.go | 23 ++++ http/otel.go | 207 ++++++++++++++++++++++++++++ http/poke_handler.go | 4 +- http/poke_handler_test.go | 36 ++++- http/refsubdocument_handler.go | 125 +++++++++++++++++ http/refsubdocument_handler_test.go | 76 ++++++++++ http/router.go | 15 ++ http/validator.go | 37 +++++ http/webconfig_server.go | 76 +++++++++- http/webpa_connector.go | 14 +- main.go | 1 + 27 files changed, 1251 insertions(+), 63 deletions(-) create mode 100644 common/refsubdocument.go create mode 100644 common/refsubdocument_test.go create mode 100644 db/cassandra/refsubdocument.go create mode 100644 db/cassandra/refsubdocument_test.go create mode 100644 db/sqlite/refsubdocument.go create mode 100644 db/sqlite/refsubdocument_test.go create mode 100644 http/otel.go create mode 100644 http/refsubdocument_handler.go create mode 100644 http/refsubdocument_handler_test.go diff --git a/common/const_var.go b/common/const_var.go index 8efae88..2fcf2af 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -106,6 +106,7 @@ const ( HeaderTraceparent = "Traceparent" HeaderTracestate = "Tracestate" HeaderContentLength = "Content-Length" + HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" ) // header X-System-Supported-Docs diff --git a/common/refsubdocument.go b/common/refsubdocument.go new file mode 100644 index 0000000..cad7ffc --- /dev/null +++ b/common/refsubdocument.go @@ -0,0 +1,81 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "bytes" +) + +type RefSubDocument struct { + payload []byte + version *string +} + +func NewRefSubDocument(payload []byte, version *string) *RefSubDocument { + return &RefSubDocument{ + payload: payload, + version: version, + } +} + +func (d *RefSubDocument) Payload() []byte { + return d.payload +} + +func (d *RefSubDocument) SetPayload(payload []byte) { + d.payload = payload +} + +func (d *RefSubDocument) HasPayload() bool { + if d.payload != nil && len(d.payload) > 0 { + return true + } else { + return false + } +} + +func (d *RefSubDocument) Version() *string { + return d.version +} + +func (d *RefSubDocument) SetVersion(version *string) { + d.version = version +} + +func (d *RefSubDocument) Equals(tdoc *RefSubDocument) bool { + if d.HasPayload() && tdoc.HasPayload() { + if !bytes.Equal(d.Payload(), tdoc.Payload()) { + return false + } + } else { + if d.HasPayload() != tdoc.HasPayload() { + return false + } + } + + if d.Version() != nil && tdoc.Version() != nil { + if *d.Version() != *tdoc.Version() { + return false + } + } else { + if d.Version() != tdoc.Version() { + return false + } + } + return true +} diff --git a/common/refsubdocument_test.go b/common/refsubdocument_test.go new file mode 100644 index 0000000..416c123 --- /dev/null +++ b/common/refsubdocument_test.go @@ -0,0 +1,40 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestRefSubDocument(t *testing.T) { + bbytes1 := []byte("hello world") + version1 := "12345" + refsubdoc1 := NewRefSubDocument(bbytes1, &version1) + + bbytes2 := []byte("hello world") + version2 := "12345" + refsubdoc2 := NewRefSubDocument(bbytes2, &version2) + assert.Assert(t, refsubdoc1.Equals(refsubdoc2)) + + bbytes3 := []byte("foo bar") + version3 := "12345" + refsubdoc3 := NewRefSubDocument(bbytes3, &version3) + assert.Assert(t, !refsubdoc1.Equals(refsubdoc3)) +} diff --git a/common/server_config.go b/common/server_config.go index 44fa5a7..0873776 100644 --- a/common/server_config.go +++ b/common/server_config.go @@ -30,6 +30,7 @@ var ( "/app/webconfig/test_webconfig.conf", "../config/sample_webconfig.conf", "/app/webconfig/webconfig.conf", + "/app/webconfig/conf/webconfig.conf", } ) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 894f513..fbfc38b 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -6,6 +6,15 @@ webconfig { panic_exit_enabled = false traceparent_parent_id = "0000000000000001" tracestate_vendor_id = "webconfig" + opentelemetry { + endpoint = "127.0.0.1:4318" + // Allowed values: "noop", "stdout", "http" + // "noop" will generate no trace + // "stdout" will use stdoutTracer and output spans to stdout + // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector + provider = "noop" + env_name = "dev" + } // build info code_git_commit = "2ac7ff4" diff --git a/db/cassandra/refsubdocument.go b/db/cassandra/refsubdocument.go new file mode 100644 index 0000000..1c9ca9f --- /dev/null +++ b/db/cassandra/refsubdocument.go @@ -0,0 +1,81 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "fmt" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + "github.com/gocql/gocql" +) + +func (c *CassandraClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { + var payload []byte + var version string + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt := "SELECT payload,version FROM reference_document WHERE ref_id=?" + if err := c.Query(stmt, refId).Scan(&payload, &version); err != nil { + return nil, common.NewError(err) + } + + if len(payload) == 0 { + return nil, common.NewError(gocql.ErrNotFound) + } + + refsubdoc := common.NewRefSubDocument(payload, &version) + return refsubdoc, nil +} + +func (c *CassandraClient) SetRefSubDocument(refId string, refsubdoc *common.RefSubDocument) (fnerr error) { + // build the statement and avoid unnecessary fields/columns + columns := []string{"ref_id"} + values := []interface{}{refId} + if refsubdoc.Payload() != nil && len(refsubdoc.Payload()) > 0 { + columns = append(columns, "payload") + values = append(values, refsubdoc.Payload()) + } + + if refsubdoc.Version() != nil { + columns = append(columns, "version") + values = append(values, refsubdoc.Version()) + } + stmt := fmt.Sprintf("INSERT INTO reference_document(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) + + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + if err := c.Query(stmt, values...).Exec(); err != nil { + return common.NewError(err) + } + return nil +} + +func (c *CassandraClient) DeleteRefSubDocument(refId string) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt := "DELETE FROM reference_document WHERE ref_id=?" + if err := c.Query(stmt, refId).Exec(); err != nil { + return common.NewError(err) + } + return nil +} diff --git a/db/cassandra/refsubdocument_test.go b/db/cassandra/refsubdocument_test.go new file mode 100644 index 0000000..22e0ba8 --- /dev/null +++ b/db/cassandra/refsubdocument_test.go @@ -0,0 +1,59 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package cassandra + +import ( + "crypto/rand" + "testing" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentOperation(t *testing.T) { + refId := uuid.New().String() + + // prepare the source data + slen := util.RandomInt(100) + 16 + srcBytes := make([]byte, slen) + rand.Read(srcBytes) + srcVersion := util.GetMurmur3Hash(srcBytes) + + // verify empty before start + var err error + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // write into db + srcRefsubdoc := common.NewRefSubDocument(srcBytes, &srcVersion) + err = tdbclient.SetRefSubDocument(refId, srcRefsubdoc) + assert.NilError(t, err) + + fetchedRefsubdoc, err := tdbclient.GetRefSubDocument(refId) + assert.NilError(t, err) + assert.Assert(t, srcRefsubdoc.Equals(fetchedRefsubdoc)) + + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) + + // verify not found in db now + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) +} diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index 4d0ac95..b013804 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -44,6 +44,11 @@ var ( route text, schema_version text, version text +)`, + `CREATE TABLE IF NOT EXISTS reference_document ( + ref_id text PRIMARY KEY, + payload blob, + version text )`, } diff --git a/db/database_client.go b/db/database_client.go index 397c995..fa1065f 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -60,6 +60,11 @@ type DatabaseClient interface { FirmwareUpdate(string, int, *common.RootDocument) error AppendProfiles(string, []byte) ([]byte, error) + // reference subdocument + GetRefSubDocument(string) (*common.RefSubDocument, error) + SetRefSubDocument(string, *common.RefSubDocument) error + DeleteRefSubDocument(string) error + // enable state correction StateCorrectionEnabled() bool SetStateCorrectionEnabled(bool) diff --git a/db/service.go b/db/service.go index ca77dc2..882d0c4 100644 --- a/db/service.go +++ b/db/service.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "net/http" + "slices" "sort" "strings" "time" @@ -32,6 +33,14 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + referenceIndicatorByteLength = 4 +) + +var ( + referenceIndicatorBytes = make([]byte, referenceIndicatorByteLength) +) + // TODO s.MultipartSupplementaryHandler(w, r) should be handled separately // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream @@ -558,3 +567,30 @@ func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerI } return cloudRootDocument, nil } + +func GetRefId(payload []byte) (string, bool) { + if len(payload) > referenceIndicatorByteLength { + prefixBytes := payload[:referenceIndicatorByteLength] + if slices.Equal(referenceIndicatorBytes, prefixBytes) { + suffixBytes := payload[referenceIndicatorByteLength:] + return string(suffixBytes), true + } + } + return "", false +} + +func LoadRefSubDocuments(c DatabaseClient, document *common.Document, fields log.Fields) (*common.Document, error) { + newDocument := common.NewDocument(document.GetRootDocument()) + for subdocId, subDocument := range document.Items() { + payload := subDocument.Payload() + if refId, ok := GetRefId(payload); ok { + refsubdocument, err := c.GetRefSubDocument(refId) + if err != nil { + return nil, common.NewError(err) + } + subDocument.SetPayload(refsubdocument.Payload()) + } + newDocument.SetSubDocument(subdocId, &subDocument) + } + return newDocument, nil +} diff --git a/db/sqlite/refsubdocument.go b/db/sqlite/refsubdocument.go new file mode 100644 index 0000000..b93d4eb --- /dev/null +++ b/db/sqlite/refsubdocument.go @@ -0,0 +1,153 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "database/sql" + "fmt" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" + _ "github.com/mattn/go-sqlite3" +) + +func (c *SqliteClient) GetRefSubDocument(refId string) (*common.RefSubDocument, error) { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + rows, err := c.Query("SELECT payload,version FROM reference_document WHERE ref_id=?", refId) + if err != nil { + return nil, common.NewError(err) + } + + var ns1 sql.NullString + var b1 []byte + + if !rows.Next() { + return nil, sql.ErrNoRows + } + err = rows.Scan(&b1, &ns1) + defer rows.Close() + if err != nil { + return nil, common.NewError(err) + } + + var s1 *string + if ns1.Valid { + s1 = &(ns1.String) + } + + refsubdoc := common.NewRefSubDocument(b1, s1) + return refsubdoc, nil +} + +func (c *SqliteClient) insertRefSubDocument(refId string, refsubdoc *common.RefSubDocument) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // build the statement and avoid unnecessary fields/columns + columns := []string{"ref_id"} + values := []interface{}{refId} + if refsubdoc.Payload() != nil { + columns = append(columns, "payload") + values = append(values, refsubdoc.Payload()) + } + if refsubdoc.Version() != nil { + columns = append(columns, "version") + values = append(values, refsubdoc.Version()) + } + qstr := fmt.Sprintf("INSERT INTO reference_document(%v) VALUES(%v)", db.GetColumnsStr(columns), db.GetValuesStr(len(columns))) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(values...) + if err != nil { + return common.NewError(err) + } + return nil +} + +func (c *SqliteClient) updateRefSubDocument(refId string, doc *common.RefSubDocument) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + // build the statement and avoid unnecessary fields/columns + columns := []string{} + values := []interface{}{} + if doc.Payload() != nil { + columns = append(columns, "payload") + values = append(values, doc.Payload()) + } + if doc.Version() != nil { + columns = append(columns, "version") + values = append(values, doc.Version()) + } + values = append(values, refId) + qstr := fmt.Sprintf("UPDATE reference_document SET %v WHERE cpe_mac=? AND ref_id=?", db.GetSetColumnsStr(columns)) + stmt, err := c.Prepare(qstr) + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(values...) + if err != nil { + return common.NewError(err) + } + return nil +} + +func (c *SqliteClient) SetRefSubDocument(refId string, refsubdoc *common.RefSubDocument) error { + _, err := c.GetRefSubDocument(refId) + if err != nil { + if c.IsDbNotFound(err) { + err1 := c.insertRefSubDocument(refId, refsubdoc) + if err1 != nil { + return common.NewError(err1) + } + } else { + // unexpected error + return common.NewError(err) + } + } else { + // normal dbNotFound should not happen + err = c.updateRefSubDocument(refId, refsubdoc) + if err != nil { + return common.NewError(err) + } + } + + return nil +} + +func (c *SqliteClient) DeleteRefSubDocument(refId string) error { + c.concurrentQueries <- true + defer func() { <-c.concurrentQueries }() + + stmt, err := c.Prepare("DELETE FROM reference_document WHERE ref_id=?") + if err != nil { + return common.NewError(err) + } + + _, err = stmt.Exec(refId) + if err != nil { + return common.NewError(err) + } + return nil +} diff --git a/db/sqlite/refsubdocument_test.go b/db/sqlite/refsubdocument_test.go new file mode 100644 index 0000000..01478f9 --- /dev/null +++ b/db/sqlite/refsubdocument_test.go @@ -0,0 +1,59 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package sqlite + +import ( + "crypto/rand" + "testing" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentOperation(t *testing.T) { + refId := uuid.New().String() + + // prepare the source data + slen := util.RandomInt(100) + 16 + srcBytes := make([]byte, slen) + rand.Read(srcBytes) + srcVersion := util.GetMurmur3Hash(srcBytes) + + // verify empty before start + var err error + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) + + // write into db + srcRefsubdoc := common.NewRefSubDocument(srcBytes, &srcVersion) + err = tdbclient.SetRefSubDocument(refId, srcRefsubdoc) + assert.NilError(t, err) + + fetchedRefsubdoc, err := tdbclient.GetRefSubDocument(refId) + assert.NilError(t, err) + assert.Assert(t, srcRefsubdoc.Equals(fetchedRefsubdoc)) + + err = tdbclient.DeleteRefSubDocument(refId) + assert.NilError(t, err) + + // verify not found in db now + _, err = tdbclient.GetRefSubDocument(refId) + assert.Assert(t, tdbclient.IsDbNotFound(err)) +} diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 8ee8d6e..3e4c132 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -50,6 +50,11 @@ var ( route text, schema_version, version text +)`, + `CREATE TABLE IF NOT EXISTS reference_document ( + ref_id text PRIMARY KEY, + payload blob, + version text )`, } ) diff --git a/go.mod b/go.mod index bbdcee4..2c15107 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt/v5 v5.0.0 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.0 github.com/mattn/go-sqlite3 v1.14.15 github.com/prometheus/client_golang v1.13.0 @@ -17,23 +17,34 @@ require ( github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.4.0 + golang.org/x/sync v0.6.0 gotest.tools v2.2.0+incompatible ) require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -44,19 +55,24 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/klauspost/compress v1.16.7 // indirect - github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.15.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index f502ae6..b6af8c0 100644 --- a/go.sum +++ b/go.sum @@ -52,16 +52,18 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -75,6 +77,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= @@ -90,6 +94,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= @@ -122,8 +131,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -138,8 +148,8 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -151,14 +161,16 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -199,8 +211,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -254,8 +266,8 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -273,8 +285,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= @@ -293,6 +305,24 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= @@ -307,8 +337,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -375,8 +405,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -394,8 +424,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -438,8 +468,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -450,7 +480,10 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -521,8 +554,8 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -552,6 +585,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4 h1:Di6ANFilr+S60a4S61ZM00vLdw0IrQOSMS2/6mrnOU0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240617180043-68d350f18fd4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -564,6 +601,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -576,8 +615,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http/http_client.go b/http/http_client.go index dfc0459..c6cc70b 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -32,11 +32,13 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -82,20 +84,26 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) userAgent := conf.GetString("webconfig.http_client.user_agent") + var transport http.RoundTripper = &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: time.Duration(connectTimeout) * time.Second, + KeepAlive: time.Duration(keepaliveTimeout) * time.Second, + }).DialContext, + MaxIdleConns: 0, + MaxIdleConnsPerHost: maxIdleConnsPerHost, + IdleConnTimeout: time.Duration(keepaliveTimeout) * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: tlsConfig, + } + transport = otelhttp.NewTransport(transport, + otelhttp.WithPropagators(otelTracer.propagator), + otelhttp.WithTracerProvider(otelTracer.tracerProvider), + ) + return &HttpClient{ Client: &http.Client{ - Transport: &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: time.Duration(connectTimeout) * time.Second, - KeepAlive: time.Duration(keepaliveTimeout) * time.Second, - }).DialContext, - MaxIdleConns: 0, - MaxIdleConnsPerHost: maxIdleConnsPerHost, - IdleConnTimeout: time.Duration(keepaliveTimeout) * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - TLSClientConfig: tlsConfig, - }, + Transport: transport, Timeout: time.Duration(readTimeout) * time.Second, }, retries: retries, diff --git a/http/multipart.go b/http/multipart.go index 227364f..b2860af 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -147,6 +147,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin document.DeleteSubDocument(subdocId) } + document, err = db.LoadRefSubDocuments(c, document, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } respBytes, err := document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -178,9 +182,20 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin document = common.NewDocument(rootDocument) } + if userAgent == "mget" { + postUpstream = false + } + var respBytes []byte respStatus := http.StatusNotModified if document.Length() > 0 { + + if !postUpstream { + document, err = db.LoadRefSubDocuments(c, document, fields) + if err != nil { + return http.StatusInternalServerError, respHeader, nil, common.NewError(err) + } + } respBytes, err = document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -286,6 +301,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusNotModified, upstreamRespHeader, nil, nil } + finalFilteredDocument, err = db.LoadRefSubDocuments(c, finalFilteredDocument, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + } finalFilteredBytes, err := finalFilteredDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalFilteredBytes, common.NewError(err) @@ -397,6 +416,10 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusNotFound, upstreamRespHeader, nil, nil } + finalDocument, err = db.LoadRefSubDocuments(c, finalDocument, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, nil, common.NewError(err) + } finalBytes, err := finalDocument.Bytes() if err != nil { return http.StatusInternalServerError, upstreamRespHeader, finalBytes, common.NewError(err) diff --git a/http/otel.go b/http/otel.go new file mode 100644 index 0000000..1faa742 --- /dev/null +++ b/http/otel.go @@ -0,0 +1,207 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + + "github.com/go-akka/configuration" + + log "github.com/sirupsen/logrus" +) + +// Tracing contains the core dependencies to make tracing possible across an application. +type otelTracing struct { + providerName string + envName string + appName string + tracerProvider trace.TracerProvider + propagator propagation.TextMapPropagator + tracer trace.Tracer +} + +type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) + +var ( + ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") + ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") + providersConfig = map[string]providerConstructor{ + "http": httpTraceProvider, + "stdout": stdoutTraceProvider, + "noop": noopTraceProvider, + } + + otelTracer otelTracing +) + +// DefaultTracerProvider is used when no provider is given. +// The Noop tracer provider turns all tracing related operations into +// noops essentially disabling tracing. +const defaultTracerProvider = "noop" + +// newOtel creates a structure with components that apps can use to initialize OpenTelemetry +// tracing instrumentation code. +func newOtel(conf *configuration.Config) (*otelTracing, error) { + if IsNoOpTracing(conf) { + log.Debug("open telemetry tracing disabled (noop)") + } else { + log.Debug("opentelemetry tracing enabled") + } + + otelTracer.appName = conf.GetString("webconfig.app_name") + otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") + tracerProvider, err := newTracerProvider(conf) + if err != nil { + return &otelTracer, err + } + otelTracer.tracerProvider = tracerProvider + otel.SetTracerProvider(tracerProvider) + + // Set up propagator. + prop := newPropagator() + otelTracer.propagator = prop + otel.SetTextMapPropagator(prop) + + otelTracer.tracer = otel.Tracer(otelTracer.appName) + return &otelTracer, nil +} + +// IsNoOpTracing returns true if the provider is set to "noop" +func IsNoOpTracing(conf *configuration.Config) bool { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + return strings.EqualFold(providerName, "noop") +} + +// TracerProvider returns the tracer provider component. By default, the noop +// tracer provider is returned. +func (t otelTracing) TracerProvider() trace.TracerProvider { + if t.tracerProvider == nil { + return noop.NewTracerProvider() + } + return t.tracerProvider +} + +// Propagator returns the component that helps propagate trace context across +// API boundaries. By default, a W3C Trace Context format propagator is returned. +func (t otelTracing) Propagator() propagation.TextMapPropagator { + if t.propagator == nil { + return propagation.TraceContext{} + } + return t.propagator +} + +func newPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) +} + +// newTracerProvider creates the TracerProvider based on config setting +// If no config setting, a noop tracerProvider will be returned. +func newTracerProvider(conf *configuration.Config) (trace.TracerProvider, error) { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + if len(providerName) == 0 { + providerName = defaultTracerProvider + } + // Handling camelcase of provider. + providerName = strings.ToLower(providerName) + providerConfig := providersConfig[providerName] + if providerConfig == nil { + return nil, fmt.Errorf("%w for provider %s", ErrTracerProviderNotFound, providerName) + } + + traceProvider, err := providerConfig(conf) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + return traceProvider, nil +} + +func noopTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + return noop.NewTracerProvider(), nil +} + +func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + option := stdouttrace.WithPrettyPrint() + exporter, err := stdouttrace.New(option) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), + sdktrace.WithBatcher(exporter, + // Default is 5s. Set to 1s for demonstrative purposes. + sdktrace.WithBatchTimeout(time.Second)), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(otelTracer.appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), + ), + ), + ) + return tp, nil +} + +func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + // Send traces over HTTP + endpoint := conf.GetString("webconfig.opentelemetry.endpoint") + if endpoint == "" { + return nil, ErrTracerProviderBuildFailed + } + exporter, err := otlptracehttp.New(context.Background(), + otlptracehttp.WithEndpoint(endpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(otelTracer.appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), + ), + ), + ), nil +} + +func (s *WebconfigServer) OtelShutdown() { + sdkTraceProvider, ok := s.otelTracer.tracerProvider.(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} diff --git a/http/poke_handler.go b/http/poke_handler.go index 62c4092..ac50ba5 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -161,11 +161,9 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - status := http.StatusInternalServerError + status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 - } else if rherr.StatusCode > http.StatusInternalServerError { - status = rherr.StatusCode } // parse the core message diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 29c3364..ba63157 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -35,7 +35,8 @@ import ( ) var ( - mockedWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) + mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) ) func TestPokeHandler(t *testing.T) { @@ -47,7 +48,7 @@ func TestPokeHandler(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -286,7 +287,7 @@ func TestPokeHandlerInvalidMac(t *testing.T) { webpaMockServer := httptest.NewServer( http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) - _, _ = w.Write(mockedWebpaPokeResponse) + _, _ = w.Write(mockWebpaPokeResponse) })) defer webpaMockServer.Close() server.SetWebpaHost(webpaMockServer.URL) @@ -305,3 +306,32 @@ func TestPokeHandlerInvalidMac(t *testing.T) { assert.NilError(t, err) res.Body.Close() } + +func TestPokeHandlerWebpa403(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write(mockWebpaPoke403Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} diff --git a/http/refsubdocument_handler.go b/http/refsubdocument_handler.go new file mode 100644 index 0000000..6413ee4 --- /dev/null +++ b/http/refsubdocument_handler.go @@ -0,0 +1,125 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "errors" + "net/http" + + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" +) + +func (s *WebconfigServer) GetRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, _, _, err := s.ValidateRefData(w, r, false) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + refsubdoc, err := s.GetRefSubDocument(refId) + if err != nil { + if s.IsDbNotFound(err) { + Error(w, http.StatusNotFound, nil) + } else { + LogError(w, err) + Error(w, http.StatusInternalServerError, common.NewError(err)) + } + return + } + + w.Header().Set("Content-Type", "application/msgpack") + if refsubdoc.Version() != nil { + w.Header().Set(common.HeaderRefSubdocumentVersion, *refsubdoc.Version()) + } + w.WriteHeader(http.StatusOK) + w.Write(refsubdoc.Payload()) +} + +func (s *WebconfigServer) PostRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, bbytes, _, err := s.ValidateRefData(w, r, true) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + // handle version header + version := r.Header.Get(common.HeaderSubdocumentVersion) + if len(version) == 0 { + version = util.GetMurmur3Hash(bbytes) + } + + refsubdoc := common.NewRefSubDocument(bbytes, &version) + + err = s.SetRefSubDocument(refId, refsubdoc) + if err != nil { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + + WriteOkResponse(w, nil) +} + +func (s *WebconfigServer) DeleteRefSubDocumentHandler(w http.ResponseWriter, r *http.Request) { + refId, _, _, err := s.ValidateRefData(w, r, false) + if err != nil { + var status int + if errors.As(err, common.Http400ErrorType) { + status = http.StatusBadRequest + } else if errors.As(err, common.Http404ErrorType) { + status = http.StatusNotFound + } else if errors.As(err, common.Http500ErrorType) { + status = http.StatusInternalServerError + } else { + status = http.StatusInternalServerError + } + Error(w, status, common.NewError(err)) + return + } + + err = s.DeleteRefSubDocument(refId) + if err != nil { + if s.IsDbNotFound(err) { + Error(w, http.StatusNotFound, nil) + } else { + Error(w, http.StatusInternalServerError, common.NewError(err)) + } + return + } + WriteOkResponse(w, nil) +} diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go new file mode 100644 index 0000000..49171b0 --- /dev/null +++ b/http/refsubdocument_handler_test.go @@ -0,0 +1,76 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "bytes" + "fmt" + "io" + "net/http" + "testing" + + "github.com/rdkcentral/webconfig/util" + "github.com/google/uuid" + "gotest.tools/assert" +) + +func TestRefSubDocumentHandler(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + refId := uuid.New().String() + bbytes := util.RandomBytes(100, 150) + + // post + url := fmt.Sprintf("/api/v1/reference/%v/document", refId) + req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes) + + // delete + req, err = http.NewRequest("DELETE", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get but expect 404 + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) +} diff --git a/http/router.go b/http/router.go index 7645c51..c6ddaf1 100644 --- a/http/router.go +++ b/http/router.go @@ -91,6 +91,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() + sub2.Use(spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { @@ -127,5 +128,19 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { } sub4.HandleFunc("", s.DeleteDocumentHandler).Methods("DELETE") + sub5 := router.Path("/api/v1/reference/{ref}/document").Subrouter() + if testOnly { + sub5.Use(s.TestingMiddleware) + } else { + if s.ServerApiTokenAuthEnabled() { + sub5.Use(s.ApiMiddleware) + } else { + sub5.Use(s.NoAuthMiddleware) + } + } + sub5.HandleFunc("", s.GetRefSubDocumentHandler).Methods("GET") + sub5.HandleFunc("", s.PostRefSubDocumentHandler).Methods("POST") + sub5.HandleFunc("", s.DeleteRefSubDocumentHandler).Methods("DELETE") + return router } diff --git a/http/validator.go b/http/validator.go index 10683ec..d85921e 100644 --- a/http/validator.go +++ b/http/validator.go @@ -71,3 +71,40 @@ func (s *WebconfigServer) Validate(w http.ResponseWriter, r *http.Request, valid } return mac, subdocId, bodyBytes, fields, nil } + +func (s *WebconfigServer) ValidateRefData(w http.ResponseWriter, r *http.Request, validateContent bool) (string, []byte, log.Fields, error) { + var fields log.Fields + + // check mac + params := mux.Vars(r) + refId := params["ref"] + + // check for safety, but it should not fail + xw, ok := w.(*XResponseWriter) + if !ok { + err := *common.NewHttp500Error("responsewriter cast error") + return refId, nil, nil, common.NewError(err) + } + fields = xw.Audit() + + if !validateContent { + return refId, nil, fields, nil + } + + // ==== validate content ==== + // check content-type + contentType := r.Header.Get("Content-type") + if contentType != "application/msgpack" { + // TODO (1) if we should validate this header + // (2) if unexpected, return 400 or 415 + err := *common.NewHttp400Error("content-type not msgpack") + return refId, nil, nil, common.NewError(err) + } + + bodyBytes := xw.BodyBytes() + if len(bodyBytes) == 0 { + err := *common.NewHttp400Error("empty body") + return refId, nil, nil, common.NewError(err) + } + return refId, bodyBytes, fields, nil +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 0d82f31..3a6082b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -20,6 +20,7 @@ package http import ( "context" "crypto/tls" + "encoding/hex" "encoding/json" "fmt" "io" @@ -28,6 +29,9 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" @@ -73,6 +77,7 @@ var ( "privatessid", "homessid", } + ws *WebconfigServer ) type WebconfigServer struct { @@ -101,6 +106,7 @@ type WebconfigServer struct { traceparentParentID string tracestateVendorID string supplementaryAppendingEnabled bool + otelTracer *otelTracing // For OpenTelemetry Tracing } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -239,10 +245,15 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) + otelTracer, err := newOtel(conf) + if err != nil { + // Just log err and continue + log.Error("Could not initialize open telemetry for tracing, but continuing") + } supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) - return &WebconfigServer{ + ws = &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, @@ -271,8 +282,10 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer jwksEnabled: jwksEnabled, traceparentParentID: traceparentParentID, tracestateVendorID: tracestateVendorID, + otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } + return ws } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -626,12 +639,11 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques outTraceparent = traceparent[:36] + s.TraceparentParentID() + traceparent[52:55] } - // extrac tracestate from the header + // extract tracestate from the header tracestate := r.Header.Get(common.HeaderTracestate) if len(tracestate) > 0 { outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) } - // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { @@ -843,3 +855,61 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { } return itf, "" } + +func spanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + spanContext := trace.SpanContextFromContext(ctx) + remote := spanContext.IsRemote() + sc := trace.SpanContext{}.WithRemote(remote) + + traceIDStr, traceFlagsStr := parseTraceparent(r) + if traceIDStr != "" { + traceID, _ := trace.TraceIDFromHex(traceIDStr) + sc = sc.WithTraceID(traceID) + } + if traceFlagsStr != "" { + traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) + sc = sc.WithTraceFlags(traceFlags) + } + tracestateStr := getTracestate(r) + if tracestateStr != "" { + tracestate, _ := trace.ParseTraceState(tracestateStr) + sc = sc.WithTraceState(tracestate) + } + ctx = trace.ContextWithSpanContext(ctx, sc) + ctx, span := otelTracer.tracer.Start(ctx, "oswebconfig_poke_handler") + defer span.End() + + attr := attribute.String("env", otelTracer.envName) + span.SetAttributes(attr) + + // Pass the context with the span to the next handler + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +// extract traceparent from the header +func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { + inTraceparent := r.Header.Get(common.HeaderTraceparent) + if len(inTraceparent) == 55 { + traceID = inTraceparent[3:35] + traceFlags = inTraceparent[53:55] + } + return +} + +// extract tracestate from the header +func getTracestate(r *http.Request) string { + inTracestate := r.Header.Get(common.HeaderTracestate) + var outTracestate string + if len(inTracestate) > 0 { + outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, ws.TracestateVendorID(), ws.TraceparentParentID()) + } + return outTracestate +} + +func hexStringToBytes(hexString string) []byte { + bytes, _ := hex.DecodeString(hexString) + return bytes +} diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 021f049..d7681f4 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -251,18 +251,26 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field } func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { - var cont bool - + tfields := common.FilterLogFields(fields) + tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { cbytes := make([]byte, len(bbytes)) copy(cbytes, bbytes) if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont = c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, _, cont := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { + msg := fmt.Sprintf("finished success after 1 retry") + if i > 1 { + fmt.Sprintf("finished success after %v retries", i) + } + log.WithFields(tfields).Info(msg) break } + if i == c.retries { + log.WithFields(tfields).Infof("finished failure after %v retries", i) + } } <-c.queue } diff --git a/main.go b/main.go index 11c8893..d5684ef 100644 --- a/main.go +++ b/main.go @@ -60,6 +60,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) + defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From 468a7725a00489007dcb77ba532fcb3b10ac3b0e Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Mon, 17 Jun 2024 14:32:49 -0700 Subject: [PATCH 037/155] Feedback: Use the API instead of "oswebconfig_poke_handler" as the span name --- http/webconfig_server.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index e9800bb..7a5bf7b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -861,7 +861,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { remote := spanContext.IsRemote() sc := trace.SpanContext{}.WithRemote(remote) - traceIDStr, traceFlagsStr := parseTraceparent(r) + traceIDStr, traceFlagsStr := s.parseTraceparent(r) if traceIDStr != "" { traceID, _ := trace.TraceIDFromHex(traceIDStr) sc = sc.WithTraceID(traceID) @@ -876,7 +876,14 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { sc = sc.WithTraceState(tracestate) } ctx = trace.ContextWithSpanContext(ctx, sc) - ctx, span := otelTracer.tracer.Start(ctx, "oswebconfig_poke_handler") + + // Feedback: Better to use the "path"/API rather than a hard coded name + spanName := "oswebconfig_poke_handler" + pathTemplate, _ := mux.CurrentRoute(r).GetPathTemplate() + if pathTemplate != "" { + spanName = pathTemplate + } + ctx, span := otelTracer.tracer.Start(ctx, spanName) defer span.End() attr := attribute.String("env", otelTracer.envName) @@ -888,7 +895,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { } // extract traceparent from the header -func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { +func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { inTraceparent := r.Header.Get(common.HeaderTraceparent) if len(inTraceparent) == 55 { traceID = inTraceparent[3:35] From d3c83391ad24a74d4e36e64e1c85985cf5471834 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Tue, 18 Jun 2024 10:26:17 -0700 Subject: [PATCH 038/155] Ticket to be created Fine grained control for generating traces for post/patch from oswebconfig No traces for GET calls from oswebconfig --- config/sample_webconfig.conf | 5 +++++ http/http_client.go | 13 ++++++++----- http/mqtt_connector.go | 3 ++- http/upstream_connector.go | 3 ++- http/webpa_connector.go | 5 +++-- http/xconf_connector.go | 3 ++- 6 files changed, 22 insertions(+), 10 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index fbfc38b..c485ae4 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -14,6 +14,11 @@ webconfig { // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector provider = "noop" env_name = "dev" + // trace_post and trace_patch are flags that enable/disable instrumentation on child calls from oswebconfig + // trace_post is for requests to mqtt/upstream + trace_post = true + // trace_patch is for requests to webpa + trace_patch = true } // build info diff --git a/http/http_client.go b/http/http_client.go index 790d907..7bce843 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -63,7 +63,8 @@ type HttpClient struct { userAgent string } -func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config) *HttpClient { +// genTrace is a bool that indicates enabling otel or not +func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config, genTrace bool) *HttpClient { confKey := fmt.Sprintf("webconfig.%v.connect_timeout_in_secs", serviceName) connectTimeout := int(conf.GetInt32(confKey, defaultConnectTimeout)) @@ -95,10 +96,12 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl ExpectContinueTimeout: 1 * time.Second, TLSClientConfig: tlsConfig, } - transport = otelhttp.NewTransport(transport, - otelhttp.WithPropagators(otelTracer.propagator), - otelhttp.WithTracerProvider(otelTracer.tracerProvider), - ) + if genTrace { + transport = otelhttp.NewTransport(transport, + otelhttp.WithPropagators(otelTracer.propagator), + otelhttp.WithTracerProvider(otelTracer.tracerProvider), + ) + } return &HttpClient{ Client: &http.Client{ diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 07c086b..716aee6 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -45,8 +45,9 @@ func NewMqttConnector(conf *configuration.Config, tlsConfig *tls.Config) *MqttCo confKey := fmt.Sprintf("webconfig.%v.host", serviceName) host := conf.GetString(confKey, mqttHostDefault) + genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_post", false) return &MqttConnector{ - HttpClient: NewHttpClient(conf, serviceName, tlsConfig), + HttpClient: NewHttpClient(conf, serviceName, tlsConfig, genTrace), host: host, serviceName: serviceName, } diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 165aac6..4a460bc 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -47,8 +47,9 @@ func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *Up confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) upstreamUrlTemplate := conf.GetString(confKey, upstreamUrlTemplateDefault) + genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_post", false) return &UpstreamConnector{ - HttpClient: NewHttpClient(conf, serviceName, tlsConfig), + HttpClient: NewHttpClient(conf, serviceName, tlsConfig, genTrace), host: host, serviceName: serviceName, upstreamUrlTemplate: upstreamUrlTemplate, diff --git a/http/webpa_connector.go b/http/webpa_connector.go index d7681f4..b5258eb 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -130,9 +130,10 @@ func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *Webpa confKey = fmt.Sprintf("webconfig.%v.retry_in_msecs", webpaServiceName) retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) - syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig) + genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_patch", false) + syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig, genTrace) syncClient.SetStatusHandler(520, syncHandle520) - asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig) + asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig, genTrace) asyncClient.SetStatusHandler(520, asyncHandle520) confKey = fmt.Sprintf("webconfig.%v.api_version", webpaServiceName) diff --git a/http/xconf_connector.go b/http/xconf_connector.go index d40550a..e6cb630 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -44,7 +44,8 @@ func NewXconfConnector(conf *configuration.Config, tlsConfig *tls.Config) *Xconf host := conf.GetString(confKey, xconfHostDefault) return &XconfConnector{ - HttpClient: NewHttpClient(conf, serviceName, tlsConfig), + // last param indicates no traces to be generated + HttpClient: NewHttpClient(conf, serviceName, tlsConfig, false), host: host, serviceName: serviceName, } From ef2702a4534f56edfb3892e2e501f517d42f60d6 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Wed, 19 Jun 2024 15:57:17 -0700 Subject: [PATCH 039/155] Narrower Tracing changes: Add a child span only for webpa poke patch instead of instrumenting the http client --- config/sample_webconfig.conf | 5 ----- http/http_client.go | 10 +--------- http/mqtt_connector.go | 3 +-- http/poke_handler.go | 2 +- http/upstream_connector.go | 3 +-- http/webconfig_server.go | 26 ++++++++++++++++++++------ http/webpa_connector.go | 10 +++++++--- http/xconf_connector.go | 3 +-- 8 files changed, 32 insertions(+), 30 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index c485ae4..fbfc38b 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -14,11 +14,6 @@ webconfig { // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector provider = "noop" env_name = "dev" - // trace_post and trace_patch are flags that enable/disable instrumentation on child calls from oswebconfig - // trace_post is for requests to mqtt/upstream - trace_post = true - // trace_patch is for requests to webpa - trace_patch = true } // build info diff --git a/http/http_client.go b/http/http_client.go index 7bce843..e7e66bc 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -37,7 +37,6 @@ import ( "github.com/go-akka/configuration" "github.com/google/uuid" log "github.com/sirupsen/logrus" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) const ( @@ -63,8 +62,7 @@ type HttpClient struct { userAgent string } -// genTrace is a bool that indicates enabling otel or not -func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config, genTrace bool) *HttpClient { +func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config) *HttpClient { confKey := fmt.Sprintf("webconfig.%v.connect_timeout_in_secs", serviceName) connectTimeout := int(conf.GetInt32(confKey, defaultConnectTimeout)) @@ -96,12 +94,6 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl ExpectContinueTimeout: 1 * time.Second, TLSClientConfig: tlsConfig, } - if genTrace { - transport = otelhttp.NewTransport(transport, - otelhttp.WithPropagators(otelTracer.propagator), - otelhttp.WithTracerProvider(otelTracer.tracerProvider), - ) - } return &HttpClient{ Client: &http.Client{ diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 716aee6..07c086b 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -45,9 +45,8 @@ func NewMqttConnector(conf *configuration.Config, tlsConfig *tls.Config) *MqttCo confKey := fmt.Sprintf("webconfig.%v.host", serviceName) host := conf.GetString(confKey, mqttHostDefault) - genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_post", false) return &MqttConnector{ - HttpClient: NewHttpClient(conf, serviceName, tlsConfig, genTrace), + HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, } diff --git a/http/poke_handler.go b/http/poke_handler.go index ac50ba5..2192780 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -156,7 +156,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - transactionId, err := s.Poke(mac, token, pokeStr, fields) + transactionId, err := s.Poke(r.Context(), mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 4a460bc..165aac6 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -47,9 +47,8 @@ func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *Up confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) upstreamUrlTemplate := conf.GetString(confKey, upstreamUrlTemplateDefault) - genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_post", false) return &UpstreamConnector{ - HttpClient: NewHttpClient(conf, serviceName, tlsConfig, genTrace), + HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, upstreamUrlTemplate: upstreamUrlTemplate, diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 7a5bf7b..8a5fdc3 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -106,6 +106,7 @@ type WebconfigServer struct { tracestateVendorID string supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing + webpaPokeSpanName string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -252,7 +253,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) - return &WebconfigServer{ + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, @@ -284,6 +285,10 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } + // Init the child poke span name + ws.webpaPokeSpanName = ws.WebpaConnector.PokeSpanName() + + return ws } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -590,7 +595,10 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return fmt.Errorf("invalid partner") } -func (c *WebconfigServer) Poke(cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { +func (c *WebconfigServer) Poke(ctx context.Context, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { + ctx, span := newSpan(ctx, c.webpaPokeSpanName) + defer span.End() + body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) if err != nil { @@ -883,12 +891,9 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { if pathTemplate != "" { spanName = pathTemplate } - ctx, span := otelTracer.tracer.Start(ctx, spanName) + ctx, span := newSpan(ctx, spanName) defer span.End() - attr := attribute.String("env", otelTracer.envName) - span.SetAttributes(attr) - // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) }) @@ -914,6 +919,15 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } +func newSpan(ctx context.Context, spanName string) (context.Context, trace.Span) { + var span trace.Span + ctx, span = otelTracer.tracer.Start(ctx, spanName) + attr := attribute.String("env", otelTracer.envName) + span.SetAttributes(attr) + + return ctx, span +} + func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index b5258eb..4af41b4 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -130,10 +130,9 @@ func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *Webpa confKey = fmt.Sprintf("webconfig.%v.retry_in_msecs", webpaServiceName) retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) - genTrace := conf.GetBoolean("webconfig.opentelemetry.trace_patch", false) - syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig, genTrace) + syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig) syncClient.SetStatusHandler(520, syncHandle520) - asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig, genTrace) + asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig) asyncClient.SetStatusHandler(520, asyncHandle520) confKey = fmt.Sprintf("webconfig.%v.api_version", webpaServiceName) @@ -169,6 +168,11 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } +func (c *WebpaConnector) PokeSpanName() string { + // By convention, span name won't have the host, but only the base template + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion) +} + func (c *WebpaConnector) NewQueue(capacity int) error { if c.queue != nil { err := fmt.Errorf("queue is already initialized") diff --git a/http/xconf_connector.go b/http/xconf_connector.go index e6cb630..d40550a 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -44,8 +44,7 @@ func NewXconfConnector(conf *configuration.Config, tlsConfig *tls.Config) *Xconf host := conf.GetString(confKey, xconfHostDefault) return &XconfConnector{ - // last param indicates no traces to be generated - HttpClient: NewHttpClient(conf, serviceName, tlsConfig, false), + HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, } From e015833323b49eb3c101909ed1dab4f918dc360b Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Wed, 19 Jun 2024 21:39:51 -0700 Subject: [PATCH 040/155] span name err fixed and span name improved --- http/webpa_connector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 4af41b4..0f5a1f8 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -170,7 +170,7 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { func (c *WebpaConnector) PokeSpanName() string { // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion) + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") + " PATCH" } func (c *WebpaConnector) NewQueue(capacity int) error { From 9c1a0e0ddb26fae2178753025d542f52700bcc46 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Thu, 20 Jun 2024 16:37:40 -0700 Subject: [PATCH 041/155] otel: Add method and status code explicitly as attributese --- http/poke_handler.go | 14 +++++++++++++- http/router.go | 2 +- http/webconfig_server.go | 32 ++++++++++++++++++++++++-------- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index 2192780..83cea43 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -156,10 +156,21 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - transactionId, err := s.Poke(r.Context(), mac, token, pokeStr, fields) + statusCode := http.StatusOK + statusCodePtr := &statusCode + ctx, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") + + // TODO: endSpan should reflect the real status of the webpa patch call + // not the transformed custom status e.g 404 from webpa patch gets converted + // to 521. Clumsy way of doing this... + defer endSpanWithStatusCode(span, statusCodePtr) + + transactionId, err := s.Poke(ctx, mac, token, pokeStr, fields) + if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { + statusCode = rherr.StatusCode // webpa error handling status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { @@ -186,6 +197,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } + statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index bc7b0d6..f3e523f 100644 --- a/http/router.go +++ b/http/router.go @@ -91,7 +91,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() - sub2.Use(s.spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { @@ -101,6 +100,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } + sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 8a5fdc3..0dce2c2 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -596,9 +596,6 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { } func (c *WebconfigServer) Poke(ctx context.Context, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { - ctx, span := newSpan(ctx, c.webpaPokeSpanName) - defer span.End() - body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) if err != nil { @@ -891,8 +888,8 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { if pathTemplate != "" { spanName = pathTemplate } - ctx, span := newSpan(ctx, spanName) - defer span.End() + ctx, span := newSpan(ctx, spanName, "POST") + defer endSpanWithResponseWriter(span, w) // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) @@ -919,15 +916,34 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } -func newSpan(ctx context.Context, spanName string) (context.Context, trace.Span) { +func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { var span trace.Span ctx, span = otelTracer.tracer.Start(ctx, spanName) - attr := attribute.String("env", otelTracer.envName) - span.SetAttributes(attr) + envAttr := attribute.String("env", otelTracer.envName) + span.SetAttributes(envAttr) + + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) return ctx, span } +func endSpanWithStatusCode(span trace.Span, statusCodePtr *int) { + if statusCodePtr != nil { + statusAttr := attribute.Int("http.status_code", *statusCodePtr) + span.SetAttributes(statusAttr) + } + span.End() +} + +func endSpanWithResponseWriter(span trace.Span, w http.ResponseWriter) { + if xw, ok := w.(*XResponseWriter); ok { + statusAttr := attribute.Int("http.status_code", xw.Status()) + span.SetAttributes(statusAttr) + } + span.End() +} + func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes From c90b48d37f444c10e6bfa203f5f6d7d107653645 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Thu, 20 Jun 2024 19:26:31 -0700 Subject: [PATCH 042/155] Code review feedback --- http/poke_handler.go | 17 +++++++++++------ http/webconfig_server.go | 18 +++++------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index 83cea43..356874f 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,10 +25,12 @@ import ( "sort" "strings" + "github.com/gorilla/mux" + "go.opentelemetry.io/otel/attribute" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -157,13 +159,16 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } statusCode := http.StatusOK - statusCodePtr := &statusCode ctx, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") - // TODO: endSpan should reflect the real status of the webpa patch call - // not the transformed custom status e.g 404 from webpa patch gets converted - // to 521. Clumsy way of doing this... - defer endSpanWithStatusCode(span, statusCodePtr) + // endSpan should reflect the real status of the webpa patch call + // not the transformed custom status + // e.g 404 from webpa patch is converted to 521, but we want to show 404 + defer func() { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + span.End() + }() transactionId, err := s.Poke(ctx, mac, token, pokeStr, fields) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 0dce2c2..d354735 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,6 +29,9 @@ import ( "strings" "time" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -38,9 +41,6 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -889,7 +889,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { spanName = pathTemplate } ctx, span := newSpan(ctx, spanName, "POST") - defer endSpanWithResponseWriter(span, w) + defer endSpan(span, w) // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) @@ -928,15 +928,7 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte return ctx, span } -func endSpanWithStatusCode(span trace.Span, statusCodePtr *int) { - if statusCodePtr != nil { - statusAttr := attribute.Int("http.status_code", *statusCodePtr) - span.SetAttributes(statusAttr) - } - span.End() -} - -func endSpanWithResponseWriter(span trace.Span, w http.ResponseWriter) { +func endSpan(span trace.Span, w http.ResponseWriter) { if xw, ok := w.(*XResponseWriter); ok { statusAttr := attribute.Int("http.status_code", xw.Status()) span.SetAttributes(statusAttr) From b50ac1128155c7211acac4d25759aea08016bf1b Mon Sep 17 00:00:00 2001 From: RV Subramanyan Date: Fri, 21 Jun 2024 10:22:56 -0700 Subject: [PATCH 043/155] Revert "Otel: Explicity add method and status_code as attributes" --- http/poke_handler.go | 21 ++------------------- http/router.go | 2 +- http/webconfig_server.go | 30 +++++++++++------------------- 3 files changed, 14 insertions(+), 39 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index 356874f..2192780 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,12 +25,10 @@ import ( "sort" "strings" - "github.com/gorilla/mux" - "go.opentelemetry.io/otel/attribute" - "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" + "github.com/gorilla/mux" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -158,24 +156,10 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - statusCode := http.StatusOK - ctx, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") - - // endSpan should reflect the real status of the webpa patch call - // not the transformed custom status - // e.g 404 from webpa patch is converted to 521, but we want to show 404 - defer func() { - statusAttr := attribute.Int("http.status_code", statusCode) - span.SetAttributes(statusAttr) - span.End() - }() - - transactionId, err := s.Poke(ctx, mac, token, pokeStr, fields) - + transactionId, err := s.Poke(r.Context(), mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - statusCode = rherr.StatusCode // webpa error handling status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { @@ -202,7 +186,6 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } - statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index f3e523f..bc7b0d6 100644 --- a/http/router.go +++ b/http/router.go @@ -91,6 +91,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() + sub2.Use(s.spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { @@ -100,7 +101,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } - sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index d354735..8a5fdc3 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,9 +29,6 @@ import ( "strings" "time" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -41,6 +38,9 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -596,6 +596,9 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { } func (c *WebconfigServer) Poke(ctx context.Context, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { + ctx, span := newSpan(ctx, c.webpaPokeSpanName) + defer span.End() + body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) if err != nil { @@ -888,8 +891,8 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { if pathTemplate != "" { spanName = pathTemplate } - ctx, span := newSpan(ctx, spanName, "POST") - defer endSpan(span, w) + ctx, span := newSpan(ctx, spanName) + defer span.End() // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) @@ -916,26 +919,15 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } -func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { +func newSpan(ctx context.Context, spanName string) (context.Context, trace.Span) { var span trace.Span ctx, span = otelTracer.tracer.Start(ctx, spanName) + attr := attribute.String("env", otelTracer.envName) + span.SetAttributes(attr) - envAttr := attribute.String("env", otelTracer.envName) - span.SetAttributes(envAttr) - - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) return ctx, span } -func endSpan(span trace.Span, w http.ResponseWriter) { - if xw, ok := w.(*XResponseWriter); ok { - statusAttr := attribute.Int("http.status_code", xw.Status()) - span.SetAttributes(statusAttr) - } - span.End() -} - func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes From 61348f563d4a0f5b0e87c876d06f9372a3456c30 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Fri, 21 Jun 2024 11:41:26 -0700 Subject: [PATCH 044/155] otel cherrypick --- http/http_client.go | 12 ++------ http/otel.go | 10 +++---- http/poke_handler.go | 19 +++++++++++- http/router.go | 2 +- http/webconfig_server.go | 62 ++++++++++++++++++++++++++++++---------- http/webpa_connector.go | 5 ++++ 6 files changed, 79 insertions(+), 31 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index c6cc70b..e7e66bc 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -32,13 +32,11 @@ import ( "strings" "time" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" log "github.com/sirupsen/logrus" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" - - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" ) const ( @@ -96,15 +94,11 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl ExpectContinueTimeout: 1 * time.Second, TLSClientConfig: tlsConfig, } - transport = otelhttp.NewTransport(transport, - otelhttp.WithPropagators(otelTracer.propagator), - otelhttp.WithTracerProvider(otelTracer.tracerProvider), - ) return &HttpClient{ Client: &http.Client{ Transport: transport, - Timeout: time.Duration(readTimeout) * time.Second, + Timeout: time.Duration(readTimeout) * time.Second, }, retries: retries, retryInMsecs: retryInMsecs, diff --git a/http/otel.go b/http/otel.go index 1faa742..e0a806e 100644 --- a/http/otel.go +++ b/http/otel.go @@ -27,10 +27,10 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" @@ -45,7 +45,7 @@ type otelTracing struct { envName string appName string tracerProvider trace.TracerProvider - propagator propagation.TextMapPropagator + propagator propagation.TextMapPropagator tracer trace.Tracer } @@ -54,10 +54,10 @@ type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, var ( ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") - providersConfig = map[string]providerConstructor{ - "http": httpTraceProvider, + providersConfig = map[string]providerConstructor{ + "http": httpTraceProvider, "stdout": stdoutTraceProvider, - "noop": noopTraceProvider, + "noop": noopTraceProvider, } otelTracer otelTracing diff --git a/http/poke_handler.go b/http/poke_handler.go index ac50ba5..436fb2d 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,10 +25,12 @@ import ( "sort" "strings" + "github.com/gorilla/mux" + "go.opentelemetry.io/otel/attribute" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -156,10 +158,24 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } + statusCode := http.StatusOK + _, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") + + // endSpan should reflect the real status of the webpa patch call + // not the transformed custom status + // e.g 404 from webpa patch is converted to 521, but we want to show 404 + defer func() { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + span.End() + }() + transactionId, err := s.Poke(mac, token, pokeStr, fields) + if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { + statusCode = rherr.StatusCode // webpa error handling status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { @@ -186,6 +202,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } + statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index c6ddaf1..f3e523f 100644 --- a/http/router.go +++ b/http/router.go @@ -91,7 +91,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() - sub2.Use(spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { @@ -101,6 +100,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } + sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 3a6082b..717413a 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,6 +29,9 @@ import ( "strings" "time" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -38,9 +41,6 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -77,7 +77,6 @@ var ( "privatessid", "homessid", } - ws *WebconfigServer ) type WebconfigServer struct { @@ -107,6 +106,7 @@ type WebconfigServer struct { tracestateVendorID string supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing + webpaPokeSpanName string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -253,7 +253,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) - ws = &WebconfigServer{ + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), ReadTimeout: time.Duration(conf.GetInt32("webconfig.server.read_timeout_in_secs", 3)) * time.Second, @@ -285,6 +285,9 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, } + // Init the child poke span name + ws.webpaPokeSpanName = ws.WebpaConnector.PokeSpanName() + return ws } @@ -856,14 +859,14 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func spanMiddleware(next http.Handler) http.Handler { +func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() spanContext := trace.SpanContextFromContext(ctx) remote := spanContext.IsRemote() sc := trace.SpanContext{}.WithRemote(remote) - traceIDStr, traceFlagsStr := parseTraceparent(r) + traceIDStr, traceFlagsStr := s.parseTraceparent(r) if traceIDStr != "" { traceID, _ := trace.TraceIDFromHex(traceIDStr) sc = sc.WithTraceID(traceID) @@ -872,17 +875,21 @@ func spanMiddleware(next http.Handler) http.Handler { traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) sc = sc.WithTraceFlags(traceFlags) } - tracestateStr := getTracestate(r) + tracestateStr := s.getTracestate(r) if tracestateStr != "" { tracestate, _ := trace.ParseTraceState(tracestateStr) sc = sc.WithTraceState(tracestate) } ctx = trace.ContextWithSpanContext(ctx, sc) - ctx, span := otelTracer.tracer.Start(ctx, "oswebconfig_poke_handler") - defer span.End() - attr := attribute.String("env", otelTracer.envName) - span.SetAttributes(attr) + // Feedback: Better to use the "path"/API rather than a hard coded name + spanName := "oswebconfig_poke_handler" + pathTemplate, _ := mux.CurrentRoute(r).GetPathTemplate() + if pathTemplate != "" { + spanName = pathTemplate + } + ctx, span := newSpan(ctx, spanName, "POST") + defer endSpan(span, w) // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) @@ -890,7 +897,7 @@ func spanMiddleware(next http.Handler) http.Handler { } // extract traceparent from the header -func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { +func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { inTraceparent := r.Header.Get(common.HeaderTraceparent) if len(inTraceparent) == 55 { traceID = inTraceparent[3:35] @@ -900,15 +907,40 @@ func parseTraceparent(r *http.Request) (traceID string, traceFlags string) { } // extract tracestate from the header -func getTracestate(r *http.Request) string { +func (s *WebconfigServer) getTracestate(r *http.Request) string { inTracestate := r.Header.Get(common.HeaderTracestate) var outTracestate string if len(inTracestate) > 0 { - outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, ws.TracestateVendorID(), ws.TraceparentParentID()) + outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, s.TracestateVendorID(), s.TraceparentParentID()) } return outTracestate } +func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { + var span trace.Span + ctx, span = otelTracer.tracer.Start(ctx, spanName + " " + method) + + envAttr := attribute.String("env", otelTracer.envName) + span.SetAttributes(envAttr) + + /* + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) + */ + return ctx, span +} + +func endSpan(span trace.Span, w http.ResponseWriter) { + if xw, ok := w.(*XResponseWriter); ok { + fmt.Println("RV DEBUG endSpan status", xw.Status()) + statusAttr := attribute.Int("http.status_code", xw.Status()) + span.SetAttributes(statusAttr) + } else { + fmt.Println("RV DEBUG endSpan no status") + } + span.End() +} + func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index d7681f4..ca57dc1 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -168,6 +168,11 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } +func (c *WebpaConnector) PokeSpanName() string { + // By convention, span name won't have the host, but only the base template + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") +} + func (c *WebpaConnector) NewQueue(capacity int) error { if c.queue != nil { err := fmt.Errorf("queue is already initialized") From 4af72cf516539faa0c1bca355687dd35277144c6 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Fri, 21 Jun 2024 11:58:02 -0700 Subject: [PATCH 045/155] otel - http.method is part of span name instead of a separate attribute --- http/poke_handler.go | 21 +++++++++++++++++-- http/router.go | 2 +- http/webconfig_server.go | 44 ++++++++++++++++++++++++++++------------ http/webpa_connector.go | 2 +- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index 2192780..546f6ba 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,10 +25,12 @@ import ( "sort" "strings" + "github.com/gorilla/mux" + "go.opentelemetry.io/otel/attribute" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -156,11 +158,25 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - transactionId, err := s.Poke(r.Context(), mac, token, pokeStr, fields) + statusCode := http.StatusOK + _, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") + + // endSpan should reflect the real status of the webpa patch call + // not the transformed custom status + // e.g 404 from webpa patch is converted to 521, but we want to show 404 + defer func() { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + span.End() + }() + + transactionId, err := s.Poke(mac, token, pokeStr, fields) + if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling + statusCode = rherr.StatusCode status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 @@ -186,6 +202,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } + statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index bc7b0d6..f3e523f 100644 --- a/http/router.go +++ b/http/router.go @@ -91,7 +91,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub1.HandleFunc("", s.DeleteSubDocumentHandler).Methods("DELETE") sub2 := router.Path("/api/v1/device/{mac}/poke").Subrouter() - sub2.Use(s.spanMiddleware) if testOnly { sub2.Use(s.TestingMiddleware) } else { @@ -101,6 +100,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } + sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 8a5fdc3..b3a5d72 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,6 +29,9 @@ import ( "strings" "time" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -38,9 +41,6 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -595,10 +595,7 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return fmt.Errorf("invalid partner") } -func (c *WebconfigServer) Poke(ctx context.Context, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { - ctx, span := newSpan(ctx, c.webpaPokeSpanName) - defer span.End() - +func (c *WebconfigServer) Poke(cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) if err != nil { @@ -891,8 +888,8 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { if pathTemplate != "" { spanName = pathTemplate } - ctx, span := newSpan(ctx, spanName) - defer span.End() + ctx, span := newSpan(ctx, spanName, "POST") + defer endSpan(span, w) // Pass the context with the span to the next handler next.ServeHTTP(w, r.WithContext(ctx)) @@ -919,15 +916,36 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } -func newSpan(ctx context.Context, spanName string) (context.Context, trace.Span) { +func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { var span trace.Span - ctx, span = otelTracer.tracer.Start(ctx, spanName) - attr := attribute.String("env", otelTracer.envName) - span.SetAttributes(attr) + spanNameWithMethod := spanName + " " + method + ctx, span = otelTracer.tracer.Start(ctx, spanNameWithMethod) + + envAttr := attribute.String("env", otelTracer.envName) + span.SetAttributes(envAttr) + + resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) + span.SetAttributes(resourceNameAttr) + + /* + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) + */ return ctx, span } +func endSpan(span trace.Span, w http.ResponseWriter) { + if xw, ok := w.(*XResponseWriter); ok { + fmt.Println("RV DEBUG endSpan status", xw.Status()) + statusAttr := attribute.Int("http.status_code", xw.Status()) + span.SetAttributes(statusAttr) + } else { + fmt.Println("RV DEBUG endSpan no status") + } + span.End() +} + func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 0f5a1f8..ca57dc1 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -170,7 +170,7 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { func (c *WebpaConnector) PokeSpanName() string { // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") + " PATCH" + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") } func (c *WebpaConnector) NewQueue(capacity int) error { From d075d58ec152d525f848e44b8979610219e74c94 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Fri, 21 Jun 2024 12:35:09 -0700 Subject: [PATCH 046/155] otel: order of method + path template switched in span name --- http/webconfig_server.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index b3a5d72..10fb9f8 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -918,7 +918,9 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { var span trace.Span - spanNameWithMethod := spanName + " " + method + + // Convention: method followed by path template + spanNameWithMethod := method + " " + spanName ctx, span = otelTracer.tracer.Start(ctx, spanNameWithMethod) envAttr := attribute.String("env", otelTracer.envName) @@ -937,11 +939,8 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte func endSpan(span trace.Span, w http.ResponseWriter) { if xw, ok := w.(*XResponseWriter); ok { - fmt.Println("RV DEBUG endSpan status", xw.Status()) statusAttr := attribute.Int("http.status_code", xw.Status()) span.SetAttributes(statusAttr) - } else { - fmt.Println("RV DEBUG endSpan no status") } span.End() } From d75f359cefa4e73b54bcf1f52a62d130aef29ae1 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Fri, 21 Jun 2024 12:41:14 -0700 Subject: [PATCH 047/155] Cherrypick of latest otel changes --- http/webconfig_server.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 717413a..159c89f 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -918,7 +918,10 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { var span trace.Span - ctx, span = otelTracer.tracer.Start(ctx, spanName + " " + method) + + // Convention: method followed by path template + spanNameWithMethod := method + " " + spanName + ctx, span = otelTracer.tracer.Start(ctx, spanNameWithMethod) envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) @@ -932,11 +935,8 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte func endSpan(span trace.Span, w http.ResponseWriter) { if xw, ok := w.(*XResponseWriter); ok { - fmt.Println("RV DEBUG endSpan status", xw.Status()) statusAttr := attribute.Int("http.status_code", xw.Status()) span.SetAttributes(statusAttr) - } else { - fmt.Println("RV DEBUG endSpan no status") } span.End() } From 8f603a69585a4e206dbb32eeaae7612321b5ebcf Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 15 Jun 2024 12:00:33 -0700 Subject: [PATCH 048/155] cherrypick a fix for corrupted blob handling and resolve conflicts --- db/cassandra/document.go | 7 +- http/document_handler.go | 1 + http/multipart_test.go | 164 +++++++++++++++++++++++++++++ http/supplementary_handler_test.go | 3 - 4 files changed, 170 insertions(+), 5 deletions(-) diff --git a/db/cassandra/document.go b/db/cassandra/document.go index 2eaf58c..df580b8 100644 --- a/db/cassandra/document.go +++ b/db/cassandra/document.go @@ -29,7 +29,6 @@ import ( log "github.com/sirupsen/logrus" ) -// NOTE this func (c *CassandraClient) GetSubDocument(cpeMac string, groupId string) (*common.SubDocument, error) { var err error var payload []byte @@ -283,7 +282,11 @@ func (c *CassandraClient) GetDocument(cpeMac string, xargs ...interface{}) (fndo if c.IsEncryptedGroup(groupId) { payload, err = c.DecryptBytes(payload) if err != nil { - return nil, common.NewError(err) + tfields := common.FilterLogFields(fields) + tfields["logger"] = "subdoc" + tfields["subdoc_id"] = groupId + log.WithFields(tfields).Warn(err) + continue } } diff --git a/http/document_handler.go b/http/document_handler.go index f41f7ff..3f3e834 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -301,6 +301,7 @@ func (s *WebconfigServer) DeleteDocumentHandler(w http.ResponseWriter, r *http.R if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) } + WriteOkResponse(w, nil) } else { Error(w, http.StatusInternalServerError, common.NewError(err)) } diff --git a/http/multipart_test.go b/http/multipart_test.go index 30919f8..2aed59a 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -28,6 +28,7 @@ import ( "testing" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" "github.com/vmihailenco/msgpack/v4" @@ -1104,3 +1105,166 @@ func TestStateCorrectionEnabled(t *testing.T) { assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) } + +func TestCorruptedEncryptedDocumentHandler(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + tdbclient, ok := server.DatabaseClient.(*cassandra.CassandraClient) + if !ok { + t.Skip("Only test in cassandra env") + } + + cpeMac := util.GenerateRandomCpeMac() + encSubdocIds := []string{} + tdbclient.SetEncryptedSubdocIds(encSubdocIds) + readSubDocIds := tdbclient.EncryptedSubdocIds() + assert.DeepEqual(t, encSubdocIds, readSubDocIds) + assert.Assert(t, !tdbclient.IsEncryptedGroup("privatessid")) + + // ==== step 1 setup lan subdoc ==== + // post + subdocId := "lan" + lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + lanBytes := util.RandomBytes(100, 150) + req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + _ = rbytes + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", lanUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 2 setup wan subdoc ==== + // post + subdocId = "wan" + wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + wanBytes := util.RandomBytes(100, 150) + req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", wanUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 3 setup privatessid subdoc ==== + // post + subdocId = "privatessid" + privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + privatessidBytes := util.RandomBytes(100, 150) + req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", privatessidUrl, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidBytes) + + // ==== step 4 read the document ==== + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 3) + + // parse the actual data + mpart, ok := mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + mpart, ok = mpartMap["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidBytes) + + // ==== step 5 set privatessid as an encrypted subdoc ==== + encSubdocIds = []string{"privatessid"} + tdbclient.SetEncryptedSubdocIds(encSubdocIds) + readSubDocIds = tdbclient.EncryptedSubdocIds() + assert.DeepEqual(t, encSubdocIds, readSubDocIds) + assert.Assert(t, tdbclient.IsEncryptedGroup("privatessid")) + + // ==== step 6 read the document expect no error but 1 less subdoc ==== + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "0") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok = mpartMap["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mpartMap["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + _, ok = mpartMap["privatessid"] + assert.Assert(t, !ok) +} diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index b7007e5..942b0f8 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -732,9 +732,6 @@ func TestSupplementaryAppendingFlag(t *testing.T) { // ==== step 3 verify the query params ==== assert.Assert(t, strings.Contains(ss, "&stormReadyWifi=true")) - // okok := false - // assert.Assert(t, okok) - // ==== step 4 set append flag false ==== appendEnabled = false server.SetSupplementaryAppendingEnabled(appendEnabled) From dac69fee6ca173ed7c7422e2091b3087f8aae0f7 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 22 Jun 2024 00:48:34 -0700 Subject: [PATCH 049/155] handle non-ascii characters in headers from devices --- common/req_header.go | 51 +++++++++++++ common/req_header_test.go | 70 ++++++++++++++++++ common/root_document.go | 14 ++-- common/root_document_test.go | 10 +-- db/service.go | 62 ++++++++++++---- go.mod | 59 ++++++++-------- go.sum | 114 +++++------------------------- http/poke_handler.go | 5 +- http/rootdocument_handler_test.go | 74 +++++++++++++++++++ http/webconfig_server.go | 10 +-- 10 files changed, 307 insertions(+), 162 deletions(-) create mode 100644 common/req_header.go create mode 100644 common/req_header_test.go diff --git a/common/req_header.go b/common/req_header.go new file mode 100644 index 0000000..1a21f23 --- /dev/null +++ b/common/req_header.go @@ -0,0 +1,51 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "fmt" + "net/http" +) + +type ReqHeader struct { + http.Header +} + +func NewReqHeader(header http.Header) *ReqHeader { + return &ReqHeader{ + Header: header, + } +} + +func (h *ReqHeader) Get(k string) (string, error) { + v := h.Header.Get(k) + if !IsPrintable([]byte(v)) { + return "", fmt.Errorf("header %v invalid value %v discarded", k, v) + } + return v, nil +} + +func IsPrintable(bbytes []byte) bool { + for _, char := range bbytes { + // Check if the rune is outside the printable ASCII character range. + if char < 32 || char > 126 { + return false + } + } + return true +} diff --git a/common/req_header_test.go b/common/req_header_test.go new file mode 100644 index 0000000..829720c --- /dev/null +++ b/common/req_header_test.go @@ -0,0 +1,70 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "net/http" + "testing" + + "gotest.tools/assert" +) + +func TestIsPrintable(t *testing.T) { + b1 := []byte("hello world") + assert.Assert(t, IsPrintable(b1)) + b2 := []byte{0x00, 0x00, 0x00, 0x01} + assert.Assert(t, !IsPrintable(b2)) + b3 := append(b1, b2...) + assert.Assert(t, !IsPrintable(b3)) + + b4 := []byte("CGM4140COM_6.8p8s1_PROD_sey") + b5 := []byte{0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8} + b6 := append(b4, b5...) + assert.Assert(t, !IsPrintable(b6)) +} + +func TestReqHeader(t *testing.T) { + s1 := "hello world" + s2 := string([]byte{0x00, 0x00, 0x00, 0x01}) + s3 := s1 + s2 + + header := make(http.Header) + k1 := "maroon" + header.Set(k1, "helloworld") + k2 := "auburn" + header.Set(k2, s2) + k3 := "amber" + header.Set(k3, s3) + reqHeader := NewReqHeader(header) + + v1, err := reqHeader.Get(k1) + assert.NilError(t, err) + assert.Equal(t, v1, "helloworld") + + v2, err := reqHeader.Get(k2) + assert.Assert(t, err != nil) + assert.Equal(t, v2, "") + + v3, err := reqHeader.Get(k3) + assert.Assert(t, err != nil) + assert.Equal(t, v3, "") + + v4, err := reqHeader.Get("viridian") + assert.NilError(t, err) + assert.Equal(t, v4, "") +} diff --git a/common/root_document.go b/common/root_document.go index dc21075..a3542fd 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -113,23 +113,23 @@ func (d *RootDocument) Compare(r *RootDocument) int { return RootDocumentEquals } -func (d *RootDocument) IsDifferent(r *RootDocument) bool { +func (d *RootDocument) Equals(r *RootDocument) bool { if d.Bitmap != r.Bitmap { - return true + return false } if d.FirmwareVersion != r.FirmwareVersion { - return true + return false } if d.ModelName != r.ModelName { - return true + return false } if d.PartnerId != r.PartnerId { - return true + return false } if d.SchemaVersion != r.SchemaVersion { - return true + return false } - return false + return true } // update in place diff --git a/common/root_document_test.go b/common/root_document_test.go index a3c2d88..c2ed5e2 100644 --- a/common/root_document_test.go +++ b/common/root_document_test.go @@ -80,7 +80,7 @@ func TestRootDocumentUpdate(t *testing.T) { assert.Assert(t, !strings.Contains(line, "map[")) } -func TestRootDocumentIsDifferent(t *testing.T) { +func TestRootDocumentEquals(t *testing.T) { bitmap := 123 version := "foo" schemaVersion := "33554433-1.3,33554434-1.3" @@ -89,11 +89,11 @@ func TestRootDocumentIsDifferent(t *testing.T) { firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") rootdoc2 := rootdoc1.Clone() - isDiff := rootdoc1.IsDifferent(rootdoc2) - assert.Assert(t, !isDiff) + ok := rootdoc1.Equals(rootdoc2) + assert.Assert(t, ok) firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "") - isDiff = rootdoc1.IsDifferent(rootdoc3) - assert.Assert(t, isDiff) + ok = rootdoc1.Equals(rootdoc3) + assert.Assert(t, !ok) } diff --git a/db/service.go b/db/service.go index 882d0c4..7f438cd 100644 --- a/db/service.go +++ b/db/service.go @@ -45,14 +45,23 @@ var ( // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream // (3) return a new variable to indicate goUpstream -func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { +func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { fieldsDict := make(util.Dict) fieldsDict.Update(fields) + tfields := common.FilterLogFields(fields) + tfields["logger"] = "request" + + // XPC-21583 Validate all headers + rHeader := common.NewReqHeader(inHeader) // ==== deviceRootDocument should always be created from request header ==== var bitmap int var err error - supportedDocs := rHeader.Get(common.HeaderSupportedDocs) + supportedDocs, err := rHeader.Get(common.HeaderSupportedDocs) + if err != nil { + log.WithFields(tfields).Warn(err) + } + if len(supportedDocs) > 0 { bitmap, err = util.GetCpeBitmap(supportedDocs) if err != nil { @@ -60,26 +69,43 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field } } - schemaVersion := strings.ToLower(rHeader.Get(common.HeaderSchemaVersion)) - modelName := rHeader.Get(common.HeaderModelName) + schemaVersion, err := rHeader.Get(common.HeaderSchemaVersion) + if err != nil { + log.WithFields(tfields).Warn(err) + } + schemaVersion = strings.ToLower(schemaVersion) + + modelName, err := rHeader.Get(common.HeaderModelName) + if err != nil { + log.WithFields(tfields).Warn(err) + } - partnerId := rHeader.Get(common.HeaderPartnerID) + partnerId, err := rHeader.Get(common.HeaderPartnerID) + if err != nil { + log.WithFields(tfields).Warn(err) + } if len(partnerId) == 0 { partnerId = fieldsDict.GetString("partner") } - firmwareVersion := rHeader.Get(common.HeaderFirmwareVersion) + firmwareVersion, err := rHeader.Get(common.HeaderFirmwareVersion) + if err != nil { + log.WithFields(tfields).Warn(err) + } // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "") // ==== parse mac ==== - mac := rHeader.Get(common.HeaderDeviceId) + mac, err := rHeader.Get(common.HeaderDeviceId) + if err != nil { + log.WithFields(tfields).Warn(err) + } var document *common.Document // get version map - deviceVersionMap, versions, err := parseVersionMap(rHeader) + deviceVersionMap, versions, err := parseVersionMap(rHeader, tfields) if err != nil { var gvmErr common.GroupVersionMismatchError if errors.As(err, &gvmErr) { @@ -126,13 +152,17 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== var rootCmpEnum int // mget fakes no meta change so that meta are not updated - if rHeader.Get("User-Agent") == "mget" { + userAgent, err := rHeader.Get("User-Agent") + if err != nil { + log.WithFields(tfields).Warn(err) + } + if userAgent == "mget" { rootCmpEnum = common.RootDocumentVersionOnlyChanged } else { rootCmpEnum = cloudRootDocument.Compare(deviceRootDocument) } - if isDiff := cloudRootDocument.IsDifferent(deviceRootDocument); isDiff { + if isEqual := cloudRootDocument.Equals(deviceRootDocument); !isEqual { // need to update rootDoc meta // NOTE need to clone the deviceRootDocument and set the version "" to avoid device root update was set back to cloud clonedRootDoc := deviceRootDocument.Clone() @@ -250,16 +280,22 @@ func GetSetColumnsStr(columns []string) string { } // deviceVersionMap := parseVersionMap(rHeader, d) -func parseVersionMap(rHeader http.Header) (map[string]string, []string, error) { +func parseVersionMap(rHeader *common.ReqHeader, tfields log.Fields) (map[string]string, []string, error) { deviceVersionMap := make(map[string]string) - queryStr := rHeader.Get(common.HeaderDocName) + queryStr, err := rHeader.Get(common.HeaderDocName) + if err != nil { + log.WithFields(tfields).Warn(err) + } subdocIds := strings.Split(queryStr, ",") if len(queryStr) == 0 { return deviceVersionMap, nil, nil } - ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) + ifNoneMatch, err := rHeader.Get(common.HeaderIfNoneMatch) + if err != nil { + log.WithFields(tfields).Warn(err) + } versions := strings.Split(ifNoneMatch, ",") if len(subdocIds) != len(versions) { diff --git a/go.mod b/go.mod index 188e4df..95108f9 100644 --- a/go.mod +++ b/go.mod @@ -3,51 +3,48 @@ module github.com/rdkcentral/webconfig go 1.21 require ( - github.com/IBM/sarama v1.43.2 + github.com/IBM/sarama v1.42.1 github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 github.com/gocql/gocql v1.6.0 - github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/uuid v1.6.0 - github.com/gorilla/mux v1.8.1 - github.com/mattn/go-sqlite3 v1.14.22 - github.com/prometheus/client_golang v1.19.1 - github.com/prometheus/client_model v0.6.1 - github.com/sirupsen/logrus v1.9.3 - github.com/twmb/murmur3 v1.1.8 + github.com/gorilla/mux v1.8.0 + github.com/mattn/go-sqlite3 v1.14.15 + github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_model v0.2.0 + github.com/sirupsen/logrus v1.9.0 + github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 go.opentelemetry.io/otel/sdk v1.27.0 go.opentelemetry.io/otel/trace v1.27.0 - go.uber.org/automaxprocs v1.5.3 - go.uber.org/ratelimit v0.3.1 - golang.org/x/sync v0.7.0 + go.uber.org/automaxprocs v1.5.1 + go.uber.org/ratelimit v0.2.0 + golang.org/x/sync v0.6.0 gotest.tools v2.2.0+incompatible ) require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect - github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.6.0 // indirect + github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -55,25 +52,25 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/common v0.54.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/proto/otlp v1.3.1 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect google.golang.org/grpc v1.64.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index 9e2917d..74c2010 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= -github.com/IBM/sarama v1.43.2 h1:HABeEqRUh32z8yzY2hGB/j8mHSzC/HA9zlEjqFNCzSw= -github.com/IBM/sarama v1.43.2/go.mod h1:Kyo4WkF24Z+1nz7xeVUFWIuKVV8RS3wM8mkvPKMdXFQ= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -46,8 +44,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= -github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -63,8 +59,6 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -75,8 +69,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= -github.com/eapache/go-resiliency v1.6.0 h1:CqGDTLtpwuWKn6Nj3uNUdflaq+/kIPsg0gfNzHton30= -github.com/eapache/go-resiliency v1.6.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= @@ -85,8 +77,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= @@ -105,8 +95,6 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -115,8 +103,6 @@ github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJr github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -179,20 +165,14 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1/go.mod h1:5SN9VR2LTsRFsrEC6FHgRbTWrTHu6tqPeKxEQv15giM= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -225,8 +205,6 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -239,12 +217,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -254,8 +228,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -271,23 +243,17 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.54.0 h1:ZlZy0BgJhTwVZUn7dLOkwCZHUkrAqd3WYtcFCWnM1D8= -github.com/prometheus/common v0.54.0/go.mod h1:/TQgMJP5CuVYveyT7n/0Ix8yLNNXy9yRSkhnLTHPDIQ= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -295,20 +261,16 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -325,8 +287,6 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= -github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= -github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= @@ -343,50 +303,28 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 h1:1u/AyyOqAWzy+SkPxDpahCNZParHV8Vid1RnI2clyDE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0 h1:0W5o9SzoR15ocYHEQfvfipzcNog1lBxOLfnex91Hk6s= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.26.0/go.mod h1:zVZ8nz+VSggWmnh6tTsJqXQ7rU4xLwRtna1M4x5jq58= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= -go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= -go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= -go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= -go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= -go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= -go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= -go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= -go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= -go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -395,10 +333,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -465,10 +401,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -488,8 +422,6 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -532,10 +464,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -548,10 +478,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -653,16 +581,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de h1:jFNzHPIeuzhdRwVhbZdiym9q0ory/xY3sA+v2wPg8I0= -google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= -google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3 h1:QW9+G6Fir4VcRXVH8x3LilNAb6cxBGLa6+GM4hRwexE= -google.golang.org/genproto/googleapis/api v0.0.0-20240610135401-a8a62080eff3/go.mod h1:kdrSS/OiLkPrNUpzD4aHgCq2rVuC/YRxok32HXZ4vRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda h1:LI5DOvAxUPMv/50agcLLoo+AdWc1irS9Rzz4vPuD1V4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3 h1:9Xyg6I9IWQZhRVfCWjKK+l6kI0jHcPesVlMnT//aHNo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240610135401-a8a62080eff3/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -675,8 +597,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -691,10 +611,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http/poke_handler.go b/http/poke_handler.go index 546f6ba..4258031 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,12 +25,11 @@ import ( "sort" "strings" - "github.com/gorilla/mux" - "go.opentelemetry.io/otel/attribute" - "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" + "github.com/gorilla/mux" + "go.opentelemetry.io/otel/attribute" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 7bb66c6..d329258 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -233,3 +233,77 @@ func TestPostRootDocumentHandler(t *testing.T) { assert.Equal(t, getResp.Data, *srcDoc1) } + +func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 GET /rootdocument and expect 404 ==== + rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) + req, err := http.NewRequest("GET", rootdocUrl, nil) + req.Header.Set("Content-Type", "application/json") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 2 GET /config device ==== + // boots up but with out data in db + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + assert.NilError(t, err) + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db to compare version + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + + expectedBitmap1, err := util.GetCpeBitmap(supportedDocs1) + assert.NilError(t, err) + expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") + assert.DeepEqual(t, rootdoc, expectedRootdoc) + + // ==== step 2 ==== + supportedDocs2 := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + string(util.RandomBytes(10, 15)) + firmwareVersion2 := "CGM4331COM_4.11p7s1_PROD_sey" + string(util.RandomBytes(10, 15)) + modelName2 := "CGM4331COM" + string(util.RandomBytes(10, 15)) + partner2 := "comcast" + string(util.RandomBytes(10, 15)) + schemaVersion2 := "33554433-1.3,33554434-1.3" + string(util.RandomBytes(10, 15)) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs2) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion2) + req.Header.Set(common.HeaderModelName, modelName2) + req.Header.Set(common.HeaderPartnerID, partner2) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + res = ExecuteRequest(req, router).Result() + assert.NilError(t, err) + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db to compare version + rootdoc2, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rootdoc.Equals(rootdoc2)) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 10fb9f8..da028a5 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,9 +29,6 @@ import ( "strings" "time" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -41,6 +38,9 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -930,8 +930,8 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte span.SetAttributes(resourceNameAttr) /* - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) */ return ctx, span From 1e089b6d3022fef2b7fb99d85185688ced951b37 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sun, 23 Jun 2024 17:43:29 -0700 Subject: [PATCH 050/155] fix a bug that updated_time was changed after calling /upstream even though the subdoc is not changed --- db/service.go | 10 +- http/upstream_test.go | 311 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+), 4 deletions(-) diff --git a/db/service.go b/db/service.go index 7f438cd..6f25653 100644 --- a/db/service.go +++ b/db/service.go @@ -457,18 +457,20 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old } labels["client"] = "default" - updatedTime := int(time.Now().UnixNano() / 1000000) - newSubdoc.SetUpdatedTime(&updatedTime) - newState := common.InDeployment if oldVersion, ok := versionMap[subdocId]; ok { if newSubdoc.Version() != nil { if oldVersion == *newSubdoc.Version() { - newState = common.Deployed + return nil } } } + updatedTime := int(time.Now().UnixNano() / 1000000) + newSubdoc.SetUpdatedTime(&updatedTime) + + newState := common.InDeployment newSubdoc.SetState(&newState) + err = c.SetSubDocument(cpeMac, subdocId, newSubdoc, oldState, labels, fields) if err != nil { return common.NewError(err) diff --git a/http/upstream_test.go b/http/upstream_test.go index f587467..b672d44 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -545,3 +545,314 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { assert.Equal(t, state, common.Deployed) } } + +func TestUpstreamUpdatedTime(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + privatessidV14Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + mparts, err := util.ParseMultipartAsList(r.Header, reqBytes) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // modify the payload + newMparts := []common.Multipart{} + for _, mpart := range mparts { + if mpart.Name == "privatessid" { + version := util.GetMurmur3Hash(privatessidV14Bytes) + newMpart := common.Multipart{ + Name: mpart.Name, + Version: version, + Bytes: privatessidV14Bytes, + } + newMparts = append(newMparts, newMpart) + } else { + newMparts = append(newMparts, mpart) + } + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["privatessid"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + srcbytesMap["privatessid"] = privatessidV14Bytes + expectedStateMap := map[string]int{ + "privatessid": common.InDeployment, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + + switch subdocId { + case "privatessid": + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) != subdocUpdatedTimeMap[subdocId]) + case "lan", "wan": + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } + } +} From e29945fb9b2fe13190317b60e42276682daeefeb Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Tue, 25 Jun 2024 23:47:20 -0700 Subject: [PATCH 051/155] XPC-21774: Populate method and route attributes in otel --- http/webconfig_server.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index da028a5..41ba1f3 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -929,10 +929,11 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) span.SetAttributes(resourceNameAttr) - /* - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) - */ + routeAttr := attribute.String("http.route", spanName) + span.SetAttributes(routeAttr) + + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) return ctx, span } From ffab4a12cb6d2da1fd6eea1262cc63ff612b9f8e Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 22 Jun 2024 00:48:34 -0700 Subject: [PATCH 052/155] cherrypick and resolve conflicts --- common/req_header.go | 51 +++++++++++++++++++++ common/req_header_test.go | 70 +++++++++++++++++++++++++++++ common/root_document.go | 14 +++--- common/root_document_test.go | 10 ++--- db/service.go | 62 ++++++++++++++++++++------ go.mod | 2 - go.sum | 4 -- http/poke_handler.go | 5 +-- http/rootdocument_handler_test.go | 74 +++++++++++++++++++++++++++++++ http/webconfig_server.go | 10 ++--- 10 files changed, 263 insertions(+), 39 deletions(-) create mode 100644 common/req_header.go create mode 100644 common/req_header_test.go diff --git a/common/req_header.go b/common/req_header.go new file mode 100644 index 0000000..1a21f23 --- /dev/null +++ b/common/req_header.go @@ -0,0 +1,51 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "fmt" + "net/http" +) + +type ReqHeader struct { + http.Header +} + +func NewReqHeader(header http.Header) *ReqHeader { + return &ReqHeader{ + Header: header, + } +} + +func (h *ReqHeader) Get(k string) (string, error) { + v := h.Header.Get(k) + if !IsPrintable([]byte(v)) { + return "", fmt.Errorf("header %v invalid value %v discarded", k, v) + } + return v, nil +} + +func IsPrintable(bbytes []byte) bool { + for _, char := range bbytes { + // Check if the rune is outside the printable ASCII character range. + if char < 32 || char > 126 { + return false + } + } + return true +} diff --git a/common/req_header_test.go b/common/req_header_test.go new file mode 100644 index 0000000..829720c --- /dev/null +++ b/common/req_header_test.go @@ -0,0 +1,70 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "net/http" + "testing" + + "gotest.tools/assert" +) + +func TestIsPrintable(t *testing.T) { + b1 := []byte("hello world") + assert.Assert(t, IsPrintable(b1)) + b2 := []byte{0x00, 0x00, 0x00, 0x01} + assert.Assert(t, !IsPrintable(b2)) + b3 := append(b1, b2...) + assert.Assert(t, !IsPrintable(b3)) + + b4 := []byte("CGM4140COM_6.8p8s1_PROD_sey") + b5 := []byte{0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8} + b6 := append(b4, b5...) + assert.Assert(t, !IsPrintable(b6)) +} + +func TestReqHeader(t *testing.T) { + s1 := "hello world" + s2 := string([]byte{0x00, 0x00, 0x00, 0x01}) + s3 := s1 + s2 + + header := make(http.Header) + k1 := "maroon" + header.Set(k1, "helloworld") + k2 := "auburn" + header.Set(k2, s2) + k3 := "amber" + header.Set(k3, s3) + reqHeader := NewReqHeader(header) + + v1, err := reqHeader.Get(k1) + assert.NilError(t, err) + assert.Equal(t, v1, "helloworld") + + v2, err := reqHeader.Get(k2) + assert.Assert(t, err != nil) + assert.Equal(t, v2, "") + + v3, err := reqHeader.Get(k3) + assert.Assert(t, err != nil) + assert.Equal(t, v3, "") + + v4, err := reqHeader.Get("viridian") + assert.NilError(t, err) + assert.Equal(t, v4, "") +} diff --git a/common/root_document.go b/common/root_document.go index dc21075..a3542fd 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -113,23 +113,23 @@ func (d *RootDocument) Compare(r *RootDocument) int { return RootDocumentEquals } -func (d *RootDocument) IsDifferent(r *RootDocument) bool { +func (d *RootDocument) Equals(r *RootDocument) bool { if d.Bitmap != r.Bitmap { - return true + return false } if d.FirmwareVersion != r.FirmwareVersion { - return true + return false } if d.ModelName != r.ModelName { - return true + return false } if d.PartnerId != r.PartnerId { - return true + return false } if d.SchemaVersion != r.SchemaVersion { - return true + return false } - return false + return true } // update in place diff --git a/common/root_document_test.go b/common/root_document_test.go index a3c2d88..c2ed5e2 100644 --- a/common/root_document_test.go +++ b/common/root_document_test.go @@ -80,7 +80,7 @@ func TestRootDocumentUpdate(t *testing.T) { assert.Assert(t, !strings.Contains(line, "map[")) } -func TestRootDocumentIsDifferent(t *testing.T) { +func TestRootDocumentEquals(t *testing.T) { bitmap := 123 version := "foo" schemaVersion := "33554433-1.3,33554434-1.3" @@ -89,11 +89,11 @@ func TestRootDocumentIsDifferent(t *testing.T) { firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" rootdoc1 := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") rootdoc2 := rootdoc1.Clone() - isDiff := rootdoc1.IsDifferent(rootdoc2) - assert.Assert(t, !isDiff) + ok := rootdoc1.Equals(rootdoc2) + assert.Assert(t, ok) firmwareVersion3 := "TG4482PC2_4.14p7s3_PROD_sey" rootdoc3 := NewRootDocument(bitmap, firmwareVersion3, modelName, partnerId, schemaVersion, version, "") - isDiff = rootdoc1.IsDifferent(rootdoc3) - assert.Assert(t, isDiff) + ok = rootdoc1.Equals(rootdoc3) + assert.Assert(t, !ok) } diff --git a/db/service.go b/db/service.go index 882d0c4..7f438cd 100644 --- a/db/service.go +++ b/db/service.go @@ -45,14 +45,23 @@ var ( // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream // (3) return a new variable to indicate goUpstream -func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { +func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { fieldsDict := make(util.Dict) fieldsDict.Update(fields) + tfields := common.FilterLogFields(fields) + tfields["logger"] = "request" + + // XPC-21583 Validate all headers + rHeader := common.NewReqHeader(inHeader) // ==== deviceRootDocument should always be created from request header ==== var bitmap int var err error - supportedDocs := rHeader.Get(common.HeaderSupportedDocs) + supportedDocs, err := rHeader.Get(common.HeaderSupportedDocs) + if err != nil { + log.WithFields(tfields).Warn(err) + } + if len(supportedDocs) > 0 { bitmap, err = util.GetCpeBitmap(supportedDocs) if err != nil { @@ -60,26 +69,43 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field } } - schemaVersion := strings.ToLower(rHeader.Get(common.HeaderSchemaVersion)) - modelName := rHeader.Get(common.HeaderModelName) + schemaVersion, err := rHeader.Get(common.HeaderSchemaVersion) + if err != nil { + log.WithFields(tfields).Warn(err) + } + schemaVersion = strings.ToLower(schemaVersion) + + modelName, err := rHeader.Get(common.HeaderModelName) + if err != nil { + log.WithFields(tfields).Warn(err) + } - partnerId := rHeader.Get(common.HeaderPartnerID) + partnerId, err := rHeader.Get(common.HeaderPartnerID) + if err != nil { + log.WithFields(tfields).Warn(err) + } if len(partnerId) == 0 { partnerId = fieldsDict.GetString("partner") } - firmwareVersion := rHeader.Get(common.HeaderFirmwareVersion) + firmwareVersion, err := rHeader.Get(common.HeaderFirmwareVersion) + if err != nil { + log.WithFields(tfields).Warn(err) + } // start with an empty rootDocument.Version, just in case there are errors in parsing the version from headers deviceRootDocument := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, "", "") // ==== parse mac ==== - mac := rHeader.Get(common.HeaderDeviceId) + mac, err := rHeader.Get(common.HeaderDeviceId) + if err != nil { + log.WithFields(tfields).Warn(err) + } var document *common.Document // get version map - deviceVersionMap, versions, err := parseVersionMap(rHeader) + deviceVersionMap, versions, err := parseVersionMap(rHeader, tfields) if err != nil { var gvmErr common.GroupVersionMismatchError if errors.As(err, &gvmErr) { @@ -126,13 +152,17 @@ func BuildGetDocument(c DatabaseClient, rHeader http.Header, route string, field // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== var rootCmpEnum int // mget fakes no meta change so that meta are not updated - if rHeader.Get("User-Agent") == "mget" { + userAgent, err := rHeader.Get("User-Agent") + if err != nil { + log.WithFields(tfields).Warn(err) + } + if userAgent == "mget" { rootCmpEnum = common.RootDocumentVersionOnlyChanged } else { rootCmpEnum = cloudRootDocument.Compare(deviceRootDocument) } - if isDiff := cloudRootDocument.IsDifferent(deviceRootDocument); isDiff { + if isEqual := cloudRootDocument.Equals(deviceRootDocument); !isEqual { // need to update rootDoc meta // NOTE need to clone the deviceRootDocument and set the version "" to avoid device root update was set back to cloud clonedRootDoc := deviceRootDocument.Clone() @@ -250,16 +280,22 @@ func GetSetColumnsStr(columns []string) string { } // deviceVersionMap := parseVersionMap(rHeader, d) -func parseVersionMap(rHeader http.Header) (map[string]string, []string, error) { +func parseVersionMap(rHeader *common.ReqHeader, tfields log.Fields) (map[string]string, []string, error) { deviceVersionMap := make(map[string]string) - queryStr := rHeader.Get(common.HeaderDocName) + queryStr, err := rHeader.Get(common.HeaderDocName) + if err != nil { + log.WithFields(tfields).Warn(err) + } subdocIds := strings.Split(queryStr, ",") if len(queryStr) == 0 { return deviceVersionMap, nil, nil } - ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) + ifNoneMatch, err := rHeader.Get(common.HeaderIfNoneMatch) + if err != nil { + log.WithFields(tfields).Warn(err) + } versions := strings.Split(ifNoneMatch, ",") if len(subdocIds) != len(versions) { diff --git a/go.mod b/go.mod index 2c15107..3b3e677 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,6 @@ require ( github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 go.opentelemetry.io/otel v1.27.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 @@ -38,7 +37,6 @@ require ( github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect diff --git a/go.sum b/go.sum index b6af8c0..7cda15d 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= @@ -305,8 +303,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 h1:9l89oX4ba9kHbBol3Xin3leYJ+252h0zszDtBwyKe2A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0/go.mod h1:XLZfZboOJWHNKUv7eH0inh0E9VV6eWDFB/9yJyTLPp0= go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= diff --git a/http/poke_handler.go b/http/poke_handler.go index 436fb2d..bdccecd 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -25,12 +25,11 @@ import ( "sort" "strings" - "github.com/gorilla/mux" - "go.opentelemetry.io/otel/attribute" - "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" + "github.com/gorilla/mux" + "go.opentelemetry.io/otel/attribute" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 7bb66c6..d329258 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -233,3 +233,77 @@ func TestPostRootDocumentHandler(t *testing.T) { assert.Equal(t, getResp.Data, *srcDoc1) } + +func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 GET /rootdocument and expect 404 ==== + rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) + req, err := http.NewRequest("GET", rootdocUrl, nil) + req.Header.Set("Content-Type", "application/json") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 2 GET /config device ==== + // boots up but with out data in db + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + assert.NilError(t, err) + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db to compare version + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + + expectedBitmap1, err := util.GetCpeBitmap(supportedDocs1) + assert.NilError(t, err) + expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") + assert.DeepEqual(t, rootdoc, expectedRootdoc) + + // ==== step 2 ==== + supportedDocs2 := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + string(util.RandomBytes(10, 15)) + firmwareVersion2 := "CGM4331COM_4.11p7s1_PROD_sey" + string(util.RandomBytes(10, 15)) + modelName2 := "CGM4331COM" + string(util.RandomBytes(10, 15)) + partner2 := "comcast" + string(util.RandomBytes(10, 15)) + schemaVersion2 := "33554433-1.3,33554434-1.3" + string(util.RandomBytes(10, 15)) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs2) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion2) + req.Header.Set(common.HeaderModelName, modelName2) + req.Header.Set(common.HeaderPartnerID, partner2) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + res = ExecuteRequest(req, router).Result() + assert.NilError(t, err) + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + _ = rbytes + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // read from db to compare version + rootdoc2, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.Assert(t, rootdoc.Equals(rootdoc2)) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 159c89f..abd66ee 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -29,9 +29,6 @@ import ( "strings" "time" - "github.com/go-akka/configuration" - "github.com/gorilla/mux" - log "github.com/sirupsen/logrus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -41,6 +38,9 @@ import ( "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" + "github.com/go-akka/configuration" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" ) // TODO enum, probably no need @@ -927,8 +927,8 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte span.SetAttributes(envAttr) /* - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) */ return ctx, span } From 0a778c714a116d1bf7b6df9a145d658d312905ee Mon Sep 17 00:00:00 2001 From: James Chao Date: Sun, 23 Jun 2024 17:43:29 -0700 Subject: [PATCH 053/155] fix a bug that updated_time was changed after calling /upstream even though the subdoc is not changed --- db/service.go | 10 +- http/upstream_test.go | 311 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+), 4 deletions(-) diff --git a/db/service.go b/db/service.go index 7f438cd..6f25653 100644 --- a/db/service.go +++ b/db/service.go @@ -457,18 +457,20 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old } labels["client"] = "default" - updatedTime := int(time.Now().UnixNano() / 1000000) - newSubdoc.SetUpdatedTime(&updatedTime) - newState := common.InDeployment if oldVersion, ok := versionMap[subdocId]; ok { if newSubdoc.Version() != nil { if oldVersion == *newSubdoc.Version() { - newState = common.Deployed + return nil } } } + updatedTime := int(time.Now().UnixNano() / 1000000) + newSubdoc.SetUpdatedTime(&updatedTime) + + newState := common.InDeployment newSubdoc.SetState(&newState) + err = c.SetSubDocument(cpeMac, subdocId, newSubdoc, oldState, labels, fields) if err != nil { return common.NewError(err) diff --git a/http/upstream_test.go b/http/upstream_test.go index f587467..b672d44 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -545,3 +545,314 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { assert.Equal(t, state, common.Deployed) } } + +func TestUpstreamUpdatedTime(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + privatessidV14Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + mparts, err := util.ParseMultipartAsList(r.Header, reqBytes) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // modify the payload + newMparts := []common.Multipart{} + for _, mpart := range mparts { + if mpart.Name == "privatessid" { + version := util.GetMurmur3Hash(privatessidV14Bytes) + newMpart := common.Multipart{ + Name: mpart.Name, + Version: version, + Bytes: privatessidV14Bytes, + } + newMparts = append(newMparts, newMpart) + } else { + newMparts = append(newMparts, mpart) + } + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["privatessid"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + srcbytesMap["privatessid"] = privatessidV14Bytes + expectedStateMap := map[string]int{ + "privatessid": common.InDeployment, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + + switch subdocId { + case "privatessid": + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) != subdocUpdatedTimeMap[subdocId]) + case "lan", "wan": + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } + } +} From cf18789ec01e48bc10496a9c815a36a3e78aa69c Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Wed, 26 Jun 2024 12:23:57 -0700 Subject: [PATCH 054/155] otel cherrypick for method name population --- http/webconfig_server.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index abd66ee..cf8b0bf 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -925,11 +925,13 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) + + resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) + span.SetAttributes(resourceNameAttr) + + routeAttr := attribute.String("http.route", spanName) + span.SetAttributes(routeAttr) - /* - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) - */ return ctx, span } From c2ead08af58c88e4b7e54601900a87beec942160 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Wed, 26 Jun 2024 14:17:36 -0700 Subject: [PATCH 055/155] Git merge err, some lines were missed --- http/webconfig_server.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index cf8b0bf..41ba1f3 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -925,13 +925,16 @@ func newSpan(ctx context.Context, spanName string, method string) (context.Conte envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) - + resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) span.SetAttributes(resourceNameAttr) routeAttr := attribute.String("http.route", spanName) span.SetAttributes(routeAttr) + methodAttr := attribute.String("http.method", method) + span.SetAttributes(methodAttr) + return ctx, span } From 205d67ab98cea9486dd2eb22cfcaabb150d9fe72 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 27 Jun 2024 18:39:28 -0700 Subject: [PATCH 056/155] Skip updating subdocs if instructed by upstream response --- common/const_var.go | 5 + http/multipart.go | 22 +- http/upstream_test.go | 579 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 598 insertions(+), 8 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 2fcf2af..6d57e4a 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -107,6 +107,11 @@ const ( HeaderTracestate = "Tracestate" HeaderContentLength = "Content-Length" HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" + HeaderUpstreamResponse = "X-Upstream-Response" +) + +const ( + SkipDbUpdate = "skip-db-update" ) // header X-System-Supported-Docs diff --git a/http/multipart.go b/http/multipart.go index b2860af..c1f29d4 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -285,10 +285,13 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) - // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) - if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + // there are special use cases when we do not want to update subdocuments + if upstreamRespHeader.Get(common.HeaderUpstreamResponse) != common.SkipDbUpdate { + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } } finalFilteredDocument := finalDocument.FilterForGet(deviceVersionMap) @@ -406,10 +409,13 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l finalDocument.DeleteSubDocument(subdocId) } - // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) - if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + // there are special use cases when we do not want to update subdocuments + if upstreamRespHeader.Get(common.HeaderUpstreamResponse) != common.SkipDbUpdate { + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } } if finalDocument.Length() == 0 { diff --git a/http/upstream_test.go b/http/upstream_test.go index b672d44..9a7807b 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -856,3 +856,582 @@ func TestUpstreamUpdatedTime(t *testing.T) { } } } + +func TestUpstreamResponseSkipDbUpdate(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + // create a new document + pfBytes := util.RandomBytes(m, n) + version := util.GetMurmur3Hash(pfBytes) + newMparts := []common.Multipart{ + { + Name: "portforwarding", + Version: version, + Bytes: pfBytes, + }, + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["portforwarding"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + expectedStateMap := map[string]int{ + "privatessid": common.Deployed, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } +} + +func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + // create a new document + pfBytes := util.RandomBytes(m, n) + version := util.GetMurmur3Hash(pfBytes) + newMparts := []common.Multipart{ + { + Name: "portforwarding", + Version: version, + Bytes: pfBytes, + }, + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["portforwarding"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + expectedStateMap := map[string]int{ + "privatessid": common.Deployed, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } +} From 72353ceae3dc33ace936da9194dbf8b5b5369145 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 27 Jun 2024 18:39:28 -0700 Subject: [PATCH 057/155] Skip updating subdocs if instructed by upstream response --- common/const_var.go | 5 + http/multipart.go | 22 +- http/upstream_test.go | 579 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 598 insertions(+), 8 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 2fcf2af..6d57e4a 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -107,6 +107,11 @@ const ( HeaderTracestate = "Tracestate" HeaderContentLength = "Content-Length" HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" + HeaderUpstreamResponse = "X-Upstream-Response" +) + +const ( + SkipDbUpdate = "skip-db-update" ) // header X-System-Supported-Docs diff --git a/http/multipart.go b/http/multipart.go index b2860af..c1f29d4 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -285,10 +285,13 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) - // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) - if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + // there are special use cases when we do not want to update subdocuments + if upstreamRespHeader.Get(common.HeaderUpstreamResponse) != common.SkipDbUpdate { + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, false, deviceVersionMap, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } } finalFilteredDocument := finalDocument.FilterForGet(deviceVersionMap) @@ -406,10 +409,13 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l finalDocument.DeleteSubDocument(subdocId) } - // update states based on the final document - err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) - if err != nil { - return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + // there are special use cases when we do not want to update subdocuments + if upstreamRespHeader.Get(common.HeaderUpstreamResponse) != common.SkipDbUpdate { + // update states based on the final document + err = db.WriteDocumentFromUpstream(c, mac, upstreamRespEtag, finalDocument, document, true, nil, fields) + if err != nil { + return http.StatusInternalServerError, upstreamRespHeader, upstreamRespBytes, common.NewError(err) + } } if finalDocument.Length() == 0 { diff --git a/http/upstream_test.go b/http/upstream_test.go index b672d44..9a7807b 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -856,3 +856,582 @@ func TestUpstreamUpdatedTime(t *testing.T) { } } } + +func TestUpstreamResponseSkipDbUpdate(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + // create a new document + pfBytes := util.RandomBytes(m, n) + version := util.GetMurmur3Hash(pfBytes) + newMparts := []common.Multipart{ + { + Name: "portforwarding", + Version: version, + Bytes: pfBytes, + }, + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["portforwarding"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + expectedStateMap := map[string]int{ + "privatessid": common.Deployed, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } +} + +func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidV13Bytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidV13Bytes, + "lan": lanBytes, + "wan": wanBytes, + } + + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // parse request + reqBytes, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // not necessarily always the case but we return 404 if the input is empty + if len(reqBytes) == 0 { + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, "") + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusNotFound) + return + } + + // create a new document + pfBytes := util.RandomBytes(m, n) + version := util.GetMurmur3Hash(pfBytes) + newMparts := []common.Multipart{ + { + Name: "portforwarding", + Version: version, + Bytes: pfBytes, + }, + } + newRootVersion := db.HashRootVersion(newMparts) + + respBytes, err := common.WriteMultipartBytes(newMparts) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(err.Error())) + return + } + + // generate response + w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderEtag, newRootVersion) + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(respBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidV13Bytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidV13Bytes) + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 update the states ==== + for _, subdocId := range subdocIds { + notifBody := fmt.Sprintf(`{"namespace": "%v", "application_status": "success", "updated_time": 1635976420, "cpe_doc_version": "984628970", "transaction_uuid": "6ef948f6-cbfa-4620-bde7-8acca1f95ba3_____005CFE970DE53C1"}`, subdocId) + var m common.EventMessage + err := json.Unmarshal([]byte(notifBody), &m) + assert.NilError(t, err) + fields := make(log.Fields) + err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + assert.NilError(t, err) + } + + // ==== step 9 verify all states deployed ==== + subdocUpdatedTimeMap := make(map[string]string) + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.Deployed) + + updatedTimeText := res.Header.Get(common.HeaderSubdocumentUpdatedTime) + if len(updatedTimeText) > 0 { + subdocUpdatedTimeMap[subdocId] = updatedTimeText + } + } + + // ==== step 10 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderIfNoneMatch, "NONE") + schemaVersion2 := "33554433-1.4,33554434-1.4" + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion2) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 1) + _, ok = mpartMap["portforwarding"] + assert.Assert(t, ok) + + // ==== step 11 verify all states deployed ==== + // srcbytesMap changed + expectedStateMap := map[string]int{ + "privatessid": common.Deployed, + "lan": common.Deployed, + "wan": common.Deployed, + } + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set("Content-Type", "application/msgpack") + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, expectedStateMap[subdocId]) + assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) + } +} From de3eee0c2058a44c3420ff0278af4633cdbc48e8 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Sat, 29 Jun 2024 10:07:32 -0700 Subject: [PATCH 058/155] XPC-22084: Add operation-name, http.url, http.url_details attributes in otel --- config/sample_webconfig.conf | 1 + http/otel.go | 9 ++++--- http/poke_handler.go | 5 +++- http/webconfig_server.go | 51 ++++++++++++++++++++++-------------- http/webpa_connector.go | 9 +++++-- 5 files changed, 49 insertions(+), 26 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index fbfc38b..0ac059b 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -14,6 +14,7 @@ webconfig { // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector provider = "noop" env_name = "dev" + operation_name = "http.request" } // build info diff --git a/http/otel.go b/http/otel.go index e0a806e..0637e2f 100644 --- a/http/otel.go +++ b/http/otel.go @@ -35,7 +35,6 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/go-akka/configuration" - log "github.com/sirupsen/logrus" ) @@ -44,6 +43,7 @@ type otelTracing struct { providerName string envName string appName string + opName string tracerProvider trace.TracerProvider propagator propagation.TextMapPropagator tracer trace.Tracer @@ -71,7 +71,7 @@ const defaultTracerProvider = "noop" // newOtel creates a structure with components that apps can use to initialize OpenTelemetry // tracing instrumentation code. func newOtel(conf *configuration.Config) (*otelTracing, error) { - if IsNoOpTracing(conf) { + if IsNoopTracing(conf) { log.Debug("open telemetry tracing disabled (noop)") } else { log.Debug("opentelemetry tracing enabled") @@ -80,6 +80,7 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { otelTracer.appName = conf.GetString("webconfig.app_name") otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") + otelTracer.opName = conf.GetString("webconfig.opentelemetryl.operation_name", "http.request") tracerProvider, err := newTracerProvider(conf) if err != nil { return &otelTracer, err @@ -96,8 +97,8 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { return &otelTracer, nil } -// IsNoOpTracing returns true if the provider is set to "noop" -func IsNoOpTracing(conf *configuration.Config) bool { +// IsNoopTracing returns true if the provider is set to "noop" +func IsNoopTracing(conf *configuration.Config) bool { providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) return strings.EqualFold(providerName, "noop") } diff --git a/http/poke_handler.go b/http/poke_handler.go index 4258031..4552769 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -158,7 +158,10 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } statusCode := http.StatusOK - _, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") + pokeSpanPath := s.WebpaConnector.PokeSpanPath(mac) + // We are not passing any qparams to webpa, so fullPath = path + fullPath := pokeSpanPath + _, span := s.newChildPokeSpan(r.Context(), s.webpaPokeSpanTemplate, pokeSpanPath, fullPath, "PATCH") // endSpan should reflect the real status of the webpa patch call // not the transformed custom status diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 41ba1f3..a4b8ab3 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -106,7 +106,7 @@ type WebconfigServer struct { tracestateVendorID string supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing - webpaPokeSpanName string + webpaPokeSpanTemplate string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -286,7 +286,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, } // Init the child poke span name - ws.webpaPokeSpanName = ws.WebpaConnector.PokeSpanName() + ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() return ws } @@ -882,13 +882,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { } ctx = trace.ContextWithSpanContext(ctx, sc) - // Feedback: Better to use the "path"/API rather than a hard coded name - spanName := "oswebconfig_poke_handler" - pathTemplate, _ := mux.CurrentRoute(r).GetPathTemplate() - if pathTemplate != "" { - spanName = pathTemplate - } - ctx, span := newSpan(ctx, spanName, "POST") + ctx, span := s.newParentPokeSpan(ctx, r) defer endSpan(span, w) // Pass the context with the span to the next handler @@ -916,26 +910,45 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } -func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { +func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { var span trace.Span - // Convention: method followed by path template - spanNameWithMethod := method + " " + spanName - ctx, span = otelTracer.tracer.Start(ctx, spanNameWithMethod) + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindServer)) + + // Feedback: Better to use the "path"/API rather than a hard coded name + route := "oswebconfig_poke_handler" + if mux.CurrentRoute(r) != nil { // This can be nil in unit tests + route, _ = mux.CurrentRoute(r).GetPathTemplate() + } + s.addAttributes(span, route, r.URL.Path, r.URL.String(), r.Method) + return ctx, span +} +func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, path string, fullPath string, method string) (context.Context, trace.Span) { + var span trace.Span + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) + + s.addAttributes(span, route, path, fullPath, method) + return ctx, span + } + +func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) - resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) - span.SetAttributes(resourceNameAttr) - - routeAttr := attribute.String("http.route", spanName) + routeAttr := attribute.String("http.route", route) span.SetAttributes(routeAttr) + pathAttr := attribute.String("http.url_details.path", path) + span.SetAttributes(pathAttr) + + fullPathAttr := attribute.String("http.url", fullPath) + span.SetAttributes(fullPathAttr) + methodAttr := attribute.String("http.method", method) span.SetAttributes(methodAttr) - - return ctx, span } func endSpan(span trace.Span, w http.ResponseWriter) { diff --git a/http/webpa_connector.go b/http/webpa_connector.go index ca57dc1..f6aadbe 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -168,9 +168,14 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } -func (c *WebpaConnector) PokeSpanName() string { +func (c *WebpaConnector) PokeSpanTemplate() string { // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "{mac}") +} + +// Base URL with the cpemac populated +func (c *WebpaConnector) PokeSpanPath(mac string) string { + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, mac) } func (c *WebpaConnector) NewQueue(capacity int) error { From df09776bc00ad6ca66259d6a55ff0eb1238908fa Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 29 Jun 2024 13:11:36 -0700 Subject: [PATCH 059/155] Add the capability to forward kafka messages --- common/log_fields.go | 16 ++++++++++ common/log_fields_test.go | 25 +++++++++++++++ http/otel.go | 3 +- http/webconfig_server.go | 65 ++++++++++++++++++++++++++++++++++++--- kafka/consumer.go | 4 +++ 5 files changed, 108 insertions(+), 5 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index 06289e6..8ec4f19 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -28,6 +28,12 @@ var ( "out_traceparent", "out_tracestate", } + coreFields = []string{ + "app_name", + "audit_id", + "body", + "cpe_mac", + } ) func FilterLogFields(src log.Fields, excludes ...string) log.Fields { @@ -53,3 +59,13 @@ func UpdateLogFields(fields, newfields log.Fields) { fields[k] = v } } + +func CopyCoreLogFields(src log.Fields) log.Fields { + fields := log.Fields{} + for _, k := range coreFields { + if itf, ok := src[k]; ok { + fields[k] = itf + } + } + return fields +} diff --git a/common/log_fields_test.go b/common/log_fields_test.go index e357855..4a95b9a 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -95,3 +95,28 @@ func TestUpdateLogFields(t *testing.T) { assert.DeepEqual(t, src, expected) } + +func TestCopyCoreLogFields(t *testing.T) { + body := map[string]interface{}{ + "device_id": "mac:29cf4fe3914e", + "http_status_code": 304, + "transaction_uuid": "f160f5f2-c899-4652-b066-c9b68328d74f", + "version": "1719689278", + } + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "audit_id": "3787b860bdf64d0d87929ac8fc46b54e", + "cpe_mac": "29CF4FE3914E", + "body": body, + } + expected := log.Fields{ + "audit_id": "3787b860bdf64d0d87929ac8fc46b54e", + "cpe_mac": "29CF4FE3914E", + "body": body, + } + copied := CopyCoreLogFields(src) + assert.DeepEqual(t, copied, expected) +} diff --git a/http/otel.go b/http/otel.go index 0637e2f..69ca64f 100644 --- a/http/otel.go +++ b/http/otel.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -35,6 +35,7 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" ) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index a4b8ab3..1cfd799 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -32,6 +32,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" @@ -89,6 +90,7 @@ type WebconfigServer struct { *XconfConnector *MqttConnector *UpstreamConnector + sarama.AsyncProducer tlsConfig *tls.Config notLoggedHeaders []string metricsEnabled bool @@ -107,6 +109,8 @@ type WebconfigServer struct { supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing webpaPokeSpanTemplate string + kafkaProducerEnabled bool + kafkaProducerTopic string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -253,6 +257,25 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) + // kafka producer + var kafkaProducer sarama.AsyncProducer + kafkaProducerEnabled := conf.GetBoolean("webconfig.kafka_producer.enabled") + var kafkaProducerTopic string + if kafkaProducerEnabled { + brokersStr := conf.GetString("webconfig.kafka_producer.brokers") + if len(brokersStr) == 0 { + panic(fmt.Errorf("webconfig.kafka_producer.brokers is empty")) + } + brokers := strings.Split(brokersStr, ",") + kafkaProducerTopic = conf.GetString("webconfig.kafka_producer.topic") + + saramaConfig := sarama.NewConfig() + kafkaProducer, err = sarama.NewAsyncProducer(brokers, saramaConfig) + if err != nil { + panic(err) + } + } + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), @@ -267,6 +290,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer XconfConnector: NewXconfConnector(conf, tlsConfig), MqttConnector: NewMqttConnector(conf, tlsConfig), UpstreamConnector: NewUpstreamConnector(conf, tlsConfig), + AsyncProducer: kafkaProducer, tlsConfig: tlsConfig, notLoggedHeaders: notLoggedHeaders, metricsEnabled: metricsEnabled, @@ -284,6 +308,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer tracestateVendorID: tracestateVendorID, otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, + kafkaProducerEnabled: kafkaProducerEnabled, + kafkaProducerTopic: kafkaProducerTopic, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -580,6 +606,22 @@ func (s *WebconfigServer) SetSupplementaryAppendingEnabled(enabled bool) { s.supplementaryAppendingEnabled = enabled } +func (s *WebconfigServer) KafkaProducerEnabled() bool { + return s.kafkaProducerEnabled +} + +func (s *WebconfigServer) SetKafkaProducerEnabled(enabled bool) { + s.kafkaProducerEnabled = enabled +} + +func (s *WebconfigServer) KafkaProducerTopic() string { + return s.kafkaProducerTopic +} + +func (s *WebconfigServer) SetKafkaProducerTopic(x string) { + s.kafkaProducerTopic = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { @@ -929,11 +971,11 @@ func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, pa var span trace.Span ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) - + s.addAttributes(span, route, path, fullPath, method) - return ctx, span - } - + return ctx, span +} + func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) @@ -963,3 +1005,18 @@ func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes } + +func (s *WebconfigServer) ForwardKafkaMessage(message *sarama.ConsumerMessage, fields log.Fields) { + outMessage := &sarama.ProducerMessage{ + Topic: s.KafkaProducerTopic(), + Key: sarama.ByteEncoder(message.Key), + Value: sarama.ByteEncoder(message.Value), + } + s.Input() <- outMessage + + tfields := common.CopyCoreLogFields(fields) + tfields["logger"] = "kafkaproducer" + tfields["kafka_topic"] = outMessage.Topic + tfields["kafka_key"] = string(message.Key) + log.WithFields(tfields).Info("send") +} diff --git a/kafka/consumer.go b/kafka/consumer.go index 1135d5d..83263ac 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -253,6 +253,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } metrics.CountKafkaEvents(eventName, status, message.Partition) } + + if c.KafkaProducerEnabled() { + c.ForwardKafkaMessage(message, fields) + } case <-session.Context().Done(): return nil } From 2a744ae380e46488df81efb266ff2fb7d99bf091 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Mon, 1 Jul 2024 09:09:01 -0700 Subject: [PATCH 060/155] otel Missed a line --- http/otel.go | 6 +++--- http/webconfig_server.go | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/http/otel.go b/http/otel.go index 69ca64f..1c56b63 100644 --- a/http/otel.go +++ b/http/otel.go @@ -14,7 +14,8 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ + package http import ( @@ -35,7 +36,6 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/go-akka/configuration" - log "github.com/sirupsen/logrus" ) @@ -81,7 +81,7 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { otelTracer.appName = conf.GetString("webconfig.app_name") otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") - otelTracer.opName = conf.GetString("webconfig.opentelemetryl.operation_name", "http.request") + otelTracer.opName = conf.GetString("webconfig.opentelemetry.operation_name", "http.request") tracerProvider, err := newTracerProvider(conf) if err != nil { return &otelTracer, err diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 1cfd799..88d84b5 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -980,6 +980,9 @@ func (s *WebconfigServer) addAttributes(span trace.Span, route string, path stri envAttr := attribute.String("env", otelTracer.envName) span.SetAttributes(envAttr) + operationNameAttr := attribute.String("operation.name", otelTracer.opName) + span.SetAttributes(operationNameAttr) + routeAttr := attribute.String("http.route", route) span.SetAttributes(routeAttr) From 7b1c5fe180aa98d2c9c8baa8817b21945b86ac0c Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 00:20:06 -0700 Subject: [PATCH 061/155] forward notifications on conditions --- config/sample_webconfig.conf | 7 ++++++ db/cassandra/state_update_test.go | 12 ++++++---- db/service.go | 30 ++++++++++++------------ http/otel.go | 1 - http/upstream_test.go | 15 ++++++++---- http/webconfig_server.go | 17 ++++++++++---- kafka/consumer.go | 38 +++++++++++++++++++++++-------- 7 files changed, 81 insertions(+), 39 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 0ac059b..60d9364 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -245,4 +245,11 @@ webconfig { // correct subdoc states if versions match but not "deployed" state_correction_enabled= false + + // forward kafka messages if needed + kafka_producer { + enabled = false + brokers = "localhost:9092" + topics = "webconfig_downstream" + } } diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 0857770..2d9bbb0 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -58,8 +58,9 @@ func TestStateUpdate1(t *testing.T) { var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -72,8 +73,9 @@ func TestStateUpdate1(t *testing.T) { m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) @@ -109,8 +111,9 @@ func TestStateUpdate2(t *testing.T) { var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -123,8 +126,9 @@ func TestStateUpdate2(t *testing.T) { m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, ok) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) diff --git a/db/service.go b/db/service.go index 6f25653..9493527 100644 --- a/db/service.go +++ b/db/service.go @@ -335,10 +335,11 @@ func HashRootVersion(itf interface{}) string { return util.GetMurmur3Hash(buffer.Bytes()) } -func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) error { +func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) (bool, error) { + var updatedBy304 bool // TODO: original config-version-report for ble, NO-OP for now if len(m.Reports) > 0 { - return nil + return updatedBy304, nil } updatedTime := int(time.Now().UnixNano() / 1000000) @@ -350,7 +351,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage } labels, err := c.GetRootDocumentLabels(cpeMac) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } labels["client"] = metricsAgent @@ -359,14 +360,14 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage if m.HttpStatusCode != nil { // all non-304 got discarded if *m.HttpStatusCode != http.StatusNotModified { - return nil + return updatedBy304, nil } // process 304 fields["src_caller"] = common.GetCaller() doc, err := c.GetDocument(cpeMac, fields) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } newState := common.Deployed @@ -375,20 +376,21 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage for groupId, oldSubdoc := range doc.Items() { // fix the bad condition when updated_time is negative if oldSubdoc.NeedsUpdateForHttp304() { + updatedBy304 = true newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) oldState := *oldSubdoc.State() if err := c.SetSubDocument(cpeMac, groupId, newSubdoc, oldState, labels, fields); err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } } } - return nil + return updatedBy304, nil } // subdoc-report, should have some validation already if m.ApplicationStatus == nil || m.Namespace == nil { - return common.NewError(fmt.Errorf("ill-formatted event")) + return updatedBy304, common.NewError(fmt.Errorf("ill-formatted event")) } state := common.Failure @@ -401,13 +403,13 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return nil + return updatedBy304, nil } targetGroupId := *m.Namespace subdoc, err := c.GetSubDocument(cpeMac, *m.Namespace) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } var oldState int @@ -417,7 +419,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), } - return common.NewError(err) + return updatedBy304, common.NewError(err) } } @@ -427,7 +429,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid updated_time(%v) in db", docUpdatedTime), } - return common.NewError(err) + return updatedBy304, common.NewError(err) } } @@ -440,9 +442,9 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err = c.SetSubDocument(cpeMac, targetGroupId, newSubdoc, oldState, labels, fields) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } - return nil + return updatedBy304, nil } func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { diff --git a/http/otel.go b/http/otel.go index 1c56b63..468035f 100644 --- a/http/otel.go +++ b/http/otel.go @@ -15,7 +15,6 @@ * * SPDX-License-Identifier: Apache-2.0 */ - package http import ( diff --git a/http/upstream_test.go b/http/upstream_test.go index 9a7807b..127dcc1 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -263,8 +263,9 @@ func TestUpstream(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -525,8 +526,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -775,8 +777,9 @@ func TestUpstreamUpdatedTime(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -1074,8 +1077,9 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -1362,8 +1366,9 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 88d84b5..ae63adf 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1009,17 +1009,24 @@ func hexStringToBytes(hexString string) []byte { return bytes } -func (s *WebconfigServer) ForwardKafkaMessage(message *sarama.ConsumerMessage, fields log.Fields) { +func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMessage, fields log.Fields) { + tfields := common.CopyCoreLogFields(fields) + + bbytes, err := json.Marshal(m) + if err != nil { + tfields["logger"] = "error" + log.WithFields(tfields).Error(common.NewError(err)) + return + } outMessage := &sarama.ProducerMessage{ Topic: s.KafkaProducerTopic(), - Key: sarama.ByteEncoder(message.Key), - Value: sarama.ByteEncoder(message.Value), + Key: sarama.ByteEncoder(kbytes), + Value: sarama.ByteEncoder(bbytes), } s.Input() <- outMessage - tfields := common.CopyCoreLogFields(fields) tfields["logger"] = "kafkaproducer" tfields["kafka_topic"] = outMessage.Topic - tfields["kafka_key"] = string(message.Key) + tfields["kafka_key"] = string(kbytes) log.WithFields(tfields).Info("send") } diff --git a/kafka/consumer.go b/kafka/consumer.go index 83263ac..52b22fb 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net/http" "strings" "time" @@ -75,31 +76,31 @@ func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { return nil } -func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, error) { +func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, bool, error) { var m common.EventMessage err := json.Unmarshal(bbytes, &m) if err != nil { - return nil, common.NewError(err) + return nil, false, common.NewError(err) } fields["body"] = m cpeMac, err := m.Validate(true) if err != nil { - return nil, common.NewError(err) + return nil, false, common.NewError(err) } if m.ErrorDetails != nil && *m.ErrorDetails == "max_retry_reached" { - return &m, nil + return &m, false, nil } fields["cpemac"] = cpeMac fields["cpe_mac"] = cpeMac - err = db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) + updatedBy304, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) if err != nil { // NOTE return the *eventMessage - return &m, common.NewError(err) + return &m, updatedBy304, common.NewError(err) } - return &m, nil + return &m, updatedBy304, nil } // NOTE we choose to return an EventMessage object just to pass along the metricsAgent @@ -203,6 +204,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram var err error logMessage := "discarded" var m *common.EventMessage + var updatedBy304 bool eventName, rptHeaderValue := getEventName(message) switch eventName { @@ -212,10 +214,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") - m, err = c.handleNotification(bbytes, fields) + m, updatedBy304, err = c.handleNotification(bbytes, fields) logMessage = "ok" case "webpa-state": - m, err = c.handleNotification(message.Value, fields) + m, updatedBy304, err = c.handleNotification(message.Value, fields) logMessage = "ok" } @@ -255,7 +257,23 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } if c.KafkaProducerEnabled() { - c.ForwardKafkaMessage(message, fields) + if len(m.Reports) == 0 { + if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { + // build a root/success message + namespace := "root" + applicationStatus := "success" + em := &common.EventMessage{ + Namespace: &namespace, + ApplicationStatus: &applicationStatus, + DeviceId: m.DeviceId, + TransactionUuid: m.TransactionUuid, + Version: m.Version, + } + c.ForwardKafkaMessage(message.Key, em, fields) + } else if m.ApplicationStatus != nil && *m.ApplicationStatus == "success" && m.Namespace != nil && *m.Namespace != "telemetry" { + c.ForwardKafkaMessage(message.Key, m, fields) + } + } } case <-session.Context().Done(): return nil From 96ae446e7a7241c35067814bc9332b098e8c7070 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 00:26:55 -0700 Subject: [PATCH 062/155] fix a typo in sample config --- config/sample_webconfig.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 60d9364..4b67a55 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -250,6 +250,6 @@ webconfig { kafka_producer { enabled = false brokers = "localhost:9092" - topics = "webconfig_downstream" + topic = "webconfig_downstream" } } From 3fc8f95a0a7ebea63474f8fa6f51aa47542f2f63 Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Tue, 2 Jul 2024 17:24:25 -0700 Subject: [PATCH 063/155] populate error graph in DD --- http/webconfig_server.go | 44 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index ae63adf..27da9c7 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -30,7 +30,9 @@ import ( "time" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" @@ -977,29 +979,35 @@ func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, pa } func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { - envAttr := attribute.String("env", otelTracer.envName) - span.SetAttributes(envAttr) - - operationNameAttr := attribute.String("operation.name", otelTracer.opName) - span.SetAttributes(operationNameAttr) - - routeAttr := attribute.String("http.route", route) - span.SetAttributes(routeAttr) - - pathAttr := attribute.String("http.url_details.path", path) - span.SetAttributes(pathAttr) - - fullPathAttr := attribute.String("http.url", fullPath) - span.SetAttributes(fullPathAttr) - - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) + span.SetAttributes( + attribute.String("env", otelTracer.envName), + attribute.String("operation.name", otelTracer.opName), + attribute.String("http.url_details.path", path), + semconv.HTTPMethodKey.String(method), + semconv.HTTPRouteKey.String(route), + semconv.HTTPURLKey.String(fullPath), + ) + + log.Debug(fmt.Sprintf("added span attribute key=env, value=%s", otelTracer.envName)) + log.Debug(fmt.Sprintf("added span attribute key=operation.name, value=%s", otelTracer.opName)) + log.Debug(fmt.Sprintf("added span attribute key=http.url_details.path, value=%s", path)) + log.Debug(fmt.Sprintf("added span attribute key=http.method, value=%s", method)) + log.Debug(fmt.Sprintf("added span attribute key=http.route, value=%s", route)) + log.Debug(fmt.Sprintf("added span attribute key=http.url, value=%s", fullPath)) } func endSpan(span trace.Span, w http.ResponseWriter) { if xw, ok := w.(*XResponseWriter); ok { - statusAttr := attribute.Int("http.status_code", xw.Status()) + statusCode := xw.Status() + statusAttr := attribute.Int("http.status_code", statusCode) span.SetAttributes(statusAttr) + log.Debug(fmt.Sprintf("added span attribute key=http.status_code, value=%d", statusCode)) + if statusCode >= http.StatusInternalServerError { + statusText := http.StatusText(statusCode) + span.SetStatus(codes.Error, statusText) + span.SetAttributes(attribute.String("http.response.error", statusText)) + log.Debug(fmt.Sprintf("added span attribute key=http.response.error, value=%s", statusText)) + } } span.End() } From 13fe5c5fc61193daa8af7ceb6cc341aa89a86fab Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 14:09:43 -0700 Subject: [PATCH 064/155] handle ill-formatted kafka messages in forwarding --- kafka/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/consumer.go b/kafka/consumer.go index 52b22fb..5d94e00 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -256,7 +256,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram metrics.CountKafkaEvents(eventName, status, message.Partition) } - if c.KafkaProducerEnabled() { + if c.KafkaProducerEnabled() && m != nil { if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { // build a root/success message From 84c399e27fe2d178811d6a87a3b1e873ca16630b Mon Sep 17 00:00:00 2001 From: "rv.subbu" Date: Wed, 3 Jul 2024 16:18:46 -0700 Subject: [PATCH 065/155] otel cherrypick for populating err graphs in DD --- config/sample_webconfig.conf | 1 + http/otel.go | 9 ++--- http/poke_handler.go | 7 ++-- http/webconfig_server.go | 70 ++++++++++++++++++++++++------------ http/webpa_connector.go | 9 +++-- 5 files changed, 65 insertions(+), 31 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index fbfc38b..0ac059b 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -14,6 +14,7 @@ webconfig { // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector provider = "noop" env_name = "dev" + operation_name = "http.request" } // build info diff --git a/http/otel.go b/http/otel.go index e0a806e..468035f 100644 --- a/http/otel.go +++ b/http/otel.go @@ -35,7 +35,6 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/go-akka/configuration" - log "github.com/sirupsen/logrus" ) @@ -44,6 +43,7 @@ type otelTracing struct { providerName string envName string appName string + opName string tracerProvider trace.TracerProvider propagator propagation.TextMapPropagator tracer trace.Tracer @@ -71,7 +71,7 @@ const defaultTracerProvider = "noop" // newOtel creates a structure with components that apps can use to initialize OpenTelemetry // tracing instrumentation code. func newOtel(conf *configuration.Config) (*otelTracing, error) { - if IsNoOpTracing(conf) { + if IsNoopTracing(conf) { log.Debug("open telemetry tracing disabled (noop)") } else { log.Debug("opentelemetry tracing enabled") @@ -80,6 +80,7 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { otelTracer.appName = conf.GetString("webconfig.app_name") otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") + otelTracer.opName = conf.GetString("webconfig.opentelemetry.operation_name", "http.request") tracerProvider, err := newTracerProvider(conf) if err != nil { return &otelTracer, err @@ -96,8 +97,8 @@ func newOtel(conf *configuration.Config) (*otelTracing, error) { return &otelTracer, nil } -// IsNoOpTracing returns true if the provider is set to "noop" -func IsNoOpTracing(conf *configuration.Config) bool { +// IsNoopTracing returns true if the provider is set to "noop" +func IsNoopTracing(conf *configuration.Config) bool { providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) return strings.EqualFold(providerName, "noop") } diff --git a/http/poke_handler.go b/http/poke_handler.go index bdccecd..4552769 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -158,7 +158,10 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } statusCode := http.StatusOK - _, span := newSpan(r.Context(), s.webpaPokeSpanName, "PATCH") + pokeSpanPath := s.WebpaConnector.PokeSpanPath(mac) + // We are not passing any qparams to webpa, so fullPath = path + fullPath := pokeSpanPath + _, span := s.newChildPokeSpan(r.Context(), s.webpaPokeSpanTemplate, pokeSpanPath, fullPath, "PATCH") // endSpan should reflect the real status of the webpa patch call // not the transformed custom status @@ -174,8 +177,8 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - statusCode = rherr.StatusCode // webpa error handling + statusCode = rherr.StatusCode status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 41ba1f3..570246b 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -30,7 +30,9 @@ import ( "time" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" @@ -106,7 +108,7 @@ type WebconfigServer struct { tracestateVendorID string supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing - webpaPokeSpanName string + webpaPokeSpanTemplate string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -286,7 +288,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, } // Init the child poke span name - ws.webpaPokeSpanName = ws.WebpaConnector.PokeSpanName() + ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() return ws } @@ -882,13 +884,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { } ctx = trace.ContextWithSpanContext(ctx, sc) - // Feedback: Better to use the "path"/API rather than a hard coded name - spanName := "oswebconfig_poke_handler" - pathTemplate, _ := mux.CurrentRoute(r).GetPathTemplate() - if pathTemplate != "" { - spanName = pathTemplate - } - ctx, span := newSpan(ctx, spanName, "POST") + ctx, span := s.newParentPokeSpan(ctx, r) defer endSpan(span, w) // Pass the context with the span to the next handler @@ -916,32 +912,60 @@ func (s *WebconfigServer) getTracestate(r *http.Request) string { return outTracestate } -func newSpan(ctx context.Context, spanName string, method string) (context.Context, trace.Span) { +func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { var span trace.Span - // Convention: method followed by path template - spanNameWithMethod := method + " " + spanName - ctx, span = otelTracer.tracer.Start(ctx, spanNameWithMethod) + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindServer)) - envAttr := attribute.String("env", otelTracer.envName) - span.SetAttributes(envAttr) + // Feedback: Better to use the "path"/API rather than a hard coded name + route := "oswebconfig_poke_handler" + if mux.CurrentRoute(r) != nil { // This can be nil in unit tests + route, _ = mux.CurrentRoute(r).GetPathTemplate() + } + s.addAttributes(span, route, r.URL.Path, r.URL.String(), r.Method) + return ctx, span +} - resourceNameAttr := attribute.String("resource.name", spanNameWithMethod) - span.SetAttributes(resourceNameAttr) +func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, path string, fullPath string, method string) (context.Context, trace.Span) { + var span trace.Span + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) - routeAttr := attribute.String("http.route", spanName) - span.SetAttributes(routeAttr) + s.addAttributes(span, route, path, fullPath, method) + return ctx, span +} - methodAttr := attribute.String("http.method", method) - span.SetAttributes(methodAttr) +func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { + span.SetAttributes( + attribute.String("env", otelTracer.envName), + attribute.String("operation.name", otelTracer.opName), + attribute.String("http.url_details.path", path), + semconv.HTTPMethodKey.String(method), + semconv.HTTPRouteKey.String(route), + semconv.HTTPURLKey.String(fullPath), + ) - return ctx, span + log.Debug(fmt.Sprintf("added span attribute key=env, value=%s", otelTracer.envName)) + log.Debug(fmt.Sprintf("added span attribute key=operation.name, value=%s", otelTracer.opName)) + log.Debug(fmt.Sprintf("added span attribute key=http.url_details.path, value=%s", path)) + log.Debug(fmt.Sprintf("added span attribute key=http.method, value=%s", method)) + log.Debug(fmt.Sprintf("added span attribute key=http.route, value=%s", route)) + log.Debug(fmt.Sprintf("added span attribute key=http.url, value=%s", fullPath)) } func endSpan(span trace.Span, w http.ResponseWriter) { if xw, ok := w.(*XResponseWriter); ok { - statusAttr := attribute.Int("http.status_code", xw.Status()) + statusCode := xw.Status() + statusAttr := attribute.Int("http.status_code", statusCode) span.SetAttributes(statusAttr) + log.Debug(fmt.Sprintf("added span attribute key=http.status_code, value=%d", statusCode)) + if statusCode >= http.StatusInternalServerError { + statusText := http.StatusText(statusCode) + span.SetStatus(codes.Error, statusText) + span.SetAttributes(attribute.String("http.response.error", statusText)) + log.Debug(fmt.Sprintf("added span attribute key=http.response.error, value=%s", statusText)) + } } span.End() } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index ca57dc1..f6aadbe 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -168,9 +168,14 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } -func (c *WebpaConnector) PokeSpanName() string { +func (c *WebpaConnector) PokeSpanTemplate() string { // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "") + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "{mac}") +} + +// Base URL with the cpemac populated +func (c *WebpaConnector) PokeSpanPath(mac string) string { + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, mac) } func (c *WebpaConnector) NewQueue(capacity int) error { From c7a9e82a74bcdc6793af4db4af8248d6eb5ab07a Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 29 Jun 2024 13:11:36 -0700 Subject: [PATCH 066/155] Add the capability to forward kafka messages --- common/log_fields.go | 16 +++++++++++ common/log_fields_test.go | 25 +++++++++++++++++ http/otel.go | 3 ++- http/webconfig_server.go | 57 +++++++++++++++++++++++++++++++++++++++ kafka/consumer.go | 4 +++ 5 files changed, 104 insertions(+), 1 deletion(-) diff --git a/common/log_fields.go b/common/log_fields.go index 06289e6..8ec4f19 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -28,6 +28,12 @@ var ( "out_traceparent", "out_tracestate", } + coreFields = []string{ + "app_name", + "audit_id", + "body", + "cpe_mac", + } ) func FilterLogFields(src log.Fields, excludes ...string) log.Fields { @@ -53,3 +59,13 @@ func UpdateLogFields(fields, newfields log.Fields) { fields[k] = v } } + +func CopyCoreLogFields(src log.Fields) log.Fields { + fields := log.Fields{} + for _, k := range coreFields { + if itf, ok := src[k]; ok { + fields[k] = itf + } + } + return fields +} diff --git a/common/log_fields_test.go b/common/log_fields_test.go index e357855..4a95b9a 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -95,3 +95,28 @@ func TestUpdateLogFields(t *testing.T) { assert.DeepEqual(t, src, expected) } + +func TestCopyCoreLogFields(t *testing.T) { + body := map[string]interface{}{ + "device_id": "mac:29cf4fe3914e", + "http_status_code": 304, + "transaction_uuid": "f160f5f2-c899-4652-b066-c9b68328d74f", + "version": "1719689278", + } + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "audit_id": "3787b860bdf64d0d87929ac8fc46b54e", + "cpe_mac": "29CF4FE3914E", + "body": body, + } + expected := log.Fields{ + "audit_id": "3787b860bdf64d0d87929ac8fc46b54e", + "cpe_mac": "29CF4FE3914E", + "body": body, + } + copied := CopyCoreLogFields(src) + assert.DeepEqual(t, copied, expected) +} diff --git a/http/otel.go b/http/otel.go index 468035f..f36c0a1 100644 --- a/http/otel.go +++ b/http/otel.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -35,6 +35,7 @@ import ( "go.opentelemetry.io/otel/trace/noop" "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" ) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 570246b..d3311a0 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -34,6 +34,7 @@ import ( "go.opentelemetry.io/otel/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" @@ -91,6 +92,7 @@ type WebconfigServer struct { *XconfConnector *MqttConnector *UpstreamConnector + sarama.AsyncProducer tlsConfig *tls.Config notLoggedHeaders []string metricsEnabled bool @@ -109,6 +111,8 @@ type WebconfigServer struct { supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing webpaPokeSpanTemplate string + kafkaProducerEnabled bool + kafkaProducerTopic string } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -255,6 +259,25 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) + // kafka producer + var kafkaProducer sarama.AsyncProducer + kafkaProducerEnabled := conf.GetBoolean("webconfig.kafka_producer.enabled") + var kafkaProducerTopic string + if kafkaProducerEnabled { + brokersStr := conf.GetString("webconfig.kafka_producer.brokers") + if len(brokersStr) == 0 { + panic(fmt.Errorf("webconfig.kafka_producer.brokers is empty")) + } + brokers := strings.Split(brokersStr, ",") + kafkaProducerTopic = conf.GetString("webconfig.kafka_producer.topic") + + saramaConfig := sarama.NewConfig() + kafkaProducer, err = sarama.NewAsyncProducer(brokers, saramaConfig) + if err != nil { + panic(err) + } + } + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), @@ -269,6 +292,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer XconfConnector: NewXconfConnector(conf, tlsConfig), MqttConnector: NewMqttConnector(conf, tlsConfig), UpstreamConnector: NewUpstreamConnector(conf, tlsConfig), + AsyncProducer: kafkaProducer, tlsConfig: tlsConfig, notLoggedHeaders: notLoggedHeaders, metricsEnabled: metricsEnabled, @@ -286,6 +310,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer tracestateVendorID: tracestateVendorID, otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, + kafkaProducerEnabled: kafkaProducerEnabled, + kafkaProducerTopic: kafkaProducerTopic, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -582,6 +608,22 @@ func (s *WebconfigServer) SetSupplementaryAppendingEnabled(enabled bool) { s.supplementaryAppendingEnabled = enabled } +func (s *WebconfigServer) KafkaProducerEnabled() bool { + return s.kafkaProducerEnabled +} + +func (s *WebconfigServer) SetKafkaProducerEnabled(enabled bool) { + s.kafkaProducerEnabled = enabled +} + +func (s *WebconfigServer) KafkaProducerTopic() string { + return s.kafkaProducerTopic +} + +func (s *WebconfigServer) SetKafkaProducerTopic(x string) { + s.kafkaProducerTopic = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { @@ -974,3 +1016,18 @@ func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes } + +func (s *WebconfigServer) ForwardKafkaMessage(message *sarama.ConsumerMessage, fields log.Fields) { + outMessage := &sarama.ProducerMessage{ + Topic: s.KafkaProducerTopic(), + Key: sarama.ByteEncoder(message.Key), + Value: sarama.ByteEncoder(message.Value), + } + s.Input() <- outMessage + + tfields := common.CopyCoreLogFields(fields) + tfields["logger"] = "kafkaproducer" + tfields["kafka_topic"] = outMessage.Topic + tfields["kafka_key"] = string(message.Key) + log.WithFields(tfields).Info("send") +} diff --git a/kafka/consumer.go b/kafka/consumer.go index 1135d5d..83263ac 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -253,6 +253,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } metrics.CountKafkaEvents(eventName, status, message.Partition) } + + if c.KafkaProducerEnabled() { + c.ForwardKafkaMessage(message, fields) + } case <-session.Context().Done(): return nil } From 74652a9f668462fc6aa858332f857b7a85300bb2 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 00:20:06 -0700 Subject: [PATCH 067/155] cherrypick and resolve conflicts --- config/sample_webconfig.conf | 7 ++++++ db/cassandra/state_update_test.go | 12 ++++++---- db/service.go | 30 ++++++++++++------------ http/otel.go | 2 +- http/upstream_test.go | 15 ++++++++---- http/webconfig_server.go | 17 ++++++++++---- kafka/consumer.go | 38 +++++++++++++++++++++++-------- 7 files changed, 82 insertions(+), 39 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 0ac059b..60d9364 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -245,4 +245,11 @@ webconfig { // correct subdoc states if versions match but not "deployed" state_correction_enabled= false + + // forward kafka messages if needed + kafka_producer { + enabled = false + brokers = "localhost:9092" + topics = "webconfig_downstream" + } } diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 0857770..2d9bbb0 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -58,8 +58,9 @@ func TestStateUpdate1(t *testing.T) { var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -72,8 +73,9 @@ func TestStateUpdate1(t *testing.T) { m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) @@ -109,8 +111,9 @@ func TestStateUpdate2(t *testing.T) { var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -123,8 +126,9 @@ func TestStateUpdate2(t *testing.T) { m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, ok) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) diff --git a/db/service.go b/db/service.go index 6f25653..9493527 100644 --- a/db/service.go +++ b/db/service.go @@ -335,10 +335,11 @@ func HashRootVersion(itf interface{}) string { return util.GetMurmur3Hash(buffer.Bytes()) } -func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) error { +func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) (bool, error) { + var updatedBy304 bool // TODO: original config-version-report for ble, NO-OP for now if len(m.Reports) > 0 { - return nil + return updatedBy304, nil } updatedTime := int(time.Now().UnixNano() / 1000000) @@ -350,7 +351,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage } labels, err := c.GetRootDocumentLabels(cpeMac) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } labels["client"] = metricsAgent @@ -359,14 +360,14 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage if m.HttpStatusCode != nil { // all non-304 got discarded if *m.HttpStatusCode != http.StatusNotModified { - return nil + return updatedBy304, nil } // process 304 fields["src_caller"] = common.GetCaller() doc, err := c.GetDocument(cpeMac, fields) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } newState := common.Deployed @@ -375,20 +376,21 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage for groupId, oldSubdoc := range doc.Items() { // fix the bad condition when updated_time is negative if oldSubdoc.NeedsUpdateForHttp304() { + updatedBy304 = true newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) oldState := *oldSubdoc.State() if err := c.SetSubDocument(cpeMac, groupId, newSubdoc, oldState, labels, fields); err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } } } - return nil + return updatedBy304, nil } // subdoc-report, should have some validation already if m.ApplicationStatus == nil || m.Namespace == nil { - return common.NewError(fmt.Errorf("ill-formatted event")) + return updatedBy304, common.NewError(fmt.Errorf("ill-formatted event")) } state := common.Failure @@ -401,13 +403,13 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return nil + return updatedBy304, nil } targetGroupId := *m.Namespace subdoc, err := c.GetSubDocument(cpeMac, *m.Namespace) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } var oldState int @@ -417,7 +419,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), } - return common.NewError(err) + return updatedBy304, common.NewError(err) } } @@ -427,7 +429,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid updated_time(%v) in db", docUpdatedTime), } - return common.NewError(err) + return updatedBy304, common.NewError(err) } } @@ -440,9 +442,9 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err = c.SetSubDocument(cpeMac, targetGroupId, newSubdoc, oldState, labels, fields) if err != nil { - return common.NewError(err) + return updatedBy304, common.NewError(err) } - return nil + return updatedBy304, nil } func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { diff --git a/http/otel.go b/http/otel.go index f36c0a1..8c82c39 100644 --- a/http/otel.go +++ b/http/otel.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( diff --git a/http/upstream_test.go b/http/upstream_test.go index 9a7807b..127dcc1 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -263,8 +263,9 @@ func TestUpstream(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -525,8 +526,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -775,8 +777,9 @@ func TestUpstreamUpdatedTime(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -1074,8 +1077,9 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== @@ -1362,8 +1366,9 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) + assert.Assert(t, !ok) } // ==== step 9 verify all states deployed ==== diff --git a/http/webconfig_server.go b/http/webconfig_server.go index d3311a0..27da9c7 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1017,17 +1017,24 @@ func hexStringToBytes(hexString string) []byte { return bytes } -func (s *WebconfigServer) ForwardKafkaMessage(message *sarama.ConsumerMessage, fields log.Fields) { +func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMessage, fields log.Fields) { + tfields := common.CopyCoreLogFields(fields) + + bbytes, err := json.Marshal(m) + if err != nil { + tfields["logger"] = "error" + log.WithFields(tfields).Error(common.NewError(err)) + return + } outMessage := &sarama.ProducerMessage{ Topic: s.KafkaProducerTopic(), - Key: sarama.ByteEncoder(message.Key), - Value: sarama.ByteEncoder(message.Value), + Key: sarama.ByteEncoder(kbytes), + Value: sarama.ByteEncoder(bbytes), } s.Input() <- outMessage - tfields := common.CopyCoreLogFields(fields) tfields["logger"] = "kafkaproducer" tfields["kafka_topic"] = outMessage.Topic - tfields["kafka_key"] = string(message.Key) + tfields["kafka_key"] = string(kbytes) log.WithFields(tfields).Info("send") } diff --git a/kafka/consumer.go b/kafka/consumer.go index 83263ac..52b22fb 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "net/http" "strings" "time" @@ -75,31 +76,31 @@ func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { return nil } -func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, error) { +func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, bool, error) { var m common.EventMessage err := json.Unmarshal(bbytes, &m) if err != nil { - return nil, common.NewError(err) + return nil, false, common.NewError(err) } fields["body"] = m cpeMac, err := m.Validate(true) if err != nil { - return nil, common.NewError(err) + return nil, false, common.NewError(err) } if m.ErrorDetails != nil && *m.ErrorDetails == "max_retry_reached" { - return &m, nil + return &m, false, nil } fields["cpemac"] = cpeMac fields["cpe_mac"] = cpeMac - err = db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) + updatedBy304, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) if err != nil { // NOTE return the *eventMessage - return &m, common.NewError(err) + return &m, updatedBy304, common.NewError(err) } - return &m, nil + return &m, updatedBy304, nil } // NOTE we choose to return an EventMessage object just to pass along the metricsAgent @@ -203,6 +204,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram var err error logMessage := "discarded" var m *common.EventMessage + var updatedBy304 bool eventName, rptHeaderValue := getEventName(message) switch eventName { @@ -212,10 +214,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") - m, err = c.handleNotification(bbytes, fields) + m, updatedBy304, err = c.handleNotification(bbytes, fields) logMessage = "ok" case "webpa-state": - m, err = c.handleNotification(message.Value, fields) + m, updatedBy304, err = c.handleNotification(message.Value, fields) logMessage = "ok" } @@ -255,7 +257,23 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } if c.KafkaProducerEnabled() { - c.ForwardKafkaMessage(message, fields) + if len(m.Reports) == 0 { + if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { + // build a root/success message + namespace := "root" + applicationStatus := "success" + em := &common.EventMessage{ + Namespace: &namespace, + ApplicationStatus: &applicationStatus, + DeviceId: m.DeviceId, + TransactionUuid: m.TransactionUuid, + Version: m.Version, + } + c.ForwardKafkaMessage(message.Key, em, fields) + } else if m.ApplicationStatus != nil && *m.ApplicationStatus == "success" && m.Namespace != nil && *m.Namespace != "telemetry" { + c.ForwardKafkaMessage(message.Key, m, fields) + } + } } case <-session.Context().Done(): return nil From e57980525b7e8fb085f1660d61e5f81a52145994 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 00:26:55 -0700 Subject: [PATCH 068/155] fix a typo in sample config --- config/sample_webconfig.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 60d9364..4b67a55 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -250,6 +250,6 @@ webconfig { kafka_producer { enabled = false brokers = "localhost:9092" - topics = "webconfig_downstream" + topic = "webconfig_downstream" } } From 2ee6174f26391faf08c7f435a54a4f53312c7538 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 3 Jul 2024 14:09:43 -0700 Subject: [PATCH 069/155] handle ill-formatted kafka messages in forwarding --- kafka/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/consumer.go b/kafka/consumer.go index 52b22fb..5d94e00 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -256,7 +256,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram metrics.CountKafkaEvents(eventName, status, message.Partition) } - if c.KafkaProducerEnabled() { + if c.KafkaProducerEnabled() && m != nil { if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { // build a root/success message From c741d32cdca2180baa7b481ce4569990858c5bc0 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 16 Jul 2024 14:12:21 -0700 Subject: [PATCH 070/155] fix a bug that forwarded kafka messages were not logged properly --- http/webconfig_server.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 27da9c7..8deb28e 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -31,8 +31,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/trace" "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" @@ -1034,7 +1034,8 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess s.Input() <- outMessage tfields["logger"] = "kafkaproducer" - tfields["kafka_topic"] = outMessage.Topic - tfields["kafka_key"] = string(kbytes) + tfields["output_topic"] = outMessage.Topic + tfields["output_key"] = string(kbytes) + tfields["output_body"] = m log.WithFields(tfields).Info("send") } From 1f6c9e3c900b725017d96af76b367cba381e6cf9 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 16 Jul 2024 14:12:21 -0700 Subject: [PATCH 071/155] fix a bug that forwarded kafka messages were not logged properly --- http/webconfig_server.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 27da9c7..8deb28e 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -31,8 +31,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/trace" "github.com/IBM/sarama" "github.com/rdkcentral/webconfig/common" @@ -1034,7 +1034,8 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess s.Input() <- outMessage tfields["logger"] = "kafkaproducer" - tfields["kafka_topic"] = outMessage.Topic - tfields["kafka_key"] = string(kbytes) + tfields["output_topic"] = outMessage.Topic + tfields["output_key"] = string(kbytes) + tfields["output_body"] = m log.WithFields(tfields).Info("send") } From 37e8626cb3d84309fbc5a95453cc40010b142065 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 19 Jul 2024 15:34:31 -0700 Subject: [PATCH 072/155] Forward webconfig notifications without filtering --- kafka/consumer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kafka/consumer.go b/kafka/consumer.go index 5d94e00..85b6fd9 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -257,6 +257,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } if c.KafkaProducerEnabled() && m != nil { + c.ForwardKafkaMessage(message.Key, m, fields) if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { // build a root/success message @@ -270,8 +271,6 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram Version: m.Version, } c.ForwardKafkaMessage(message.Key, em, fields) - } else if m.ApplicationStatus != nil && *m.ApplicationStatus == "success" && m.Namespace != nil && *m.Namespace != "telemetry" { - c.ForwardKafkaMessage(message.Key, m, fields) } } } From 510eecada3b84ca94ef9e6805a1182f073680412 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 19 Jul 2024 15:34:31 -0700 Subject: [PATCH 073/155] Forward webconfig notifications without filtering --- kafka/consumer.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kafka/consumer.go b/kafka/consumer.go index 5d94e00..85b6fd9 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -257,6 +257,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram } if c.KafkaProducerEnabled() && m != nil { + c.ForwardKafkaMessage(message.Key, m, fields) if len(m.Reports) == 0 { if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { // build a root/success message @@ -270,8 +271,6 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram Version: m.Version, } c.ForwardKafkaMessage(message.Key, em, fields) - } else if m.ApplicationStatus != nil && *m.ApplicationStatus == "success" && m.Namespace != nil && *m.Namespace != "telemetry" { - c.ForwardKafkaMessage(message.Key, m, fields) } } } From 40ae4a14f420be789144b3be952e65d820b24c9e Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 30 Jul 2024 19:16:43 -0700 Subject: [PATCH 074/155] generate notifications if states are corrected by versions from devices --- db/cassandra/document_test.go | 2 +- db/service.go | 42 ++++++++++++++++++++++------------- http/multipart.go | 5 ++++- http/otel.go | 2 +- http/webconfig_server.go | 34 ++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 19 deletions(-) diff --git a/db/cassandra/document_test.go b/db/cassandra/document_test.go index b314690..8e63844 100644 --- a/db/cassandra/document_test.go +++ b/db/cassandra/document_test.go @@ -206,7 +206,7 @@ func TestBlockedSubdocIds(t *testing.T) { rHeader.Set(common.HeaderSupportedDocs, rdkSupportedDocsHeaderStr) - document, _, _, _, _, err := db.BuildGetDocument(tdbclient, rHeader, common.RouteHttp, fields) + document, _, _, _, _, _, err := db.BuildGetDocument(tdbclient, rHeader, common.RouteHttp, fields) assert.NilError(t, err) assert.Assert(t, document.Length() == 3) versionMap := document.VersionMap() diff --git a/db/service.go b/db/service.go index 9493527..bce94fa 100644 --- a/db/service.go +++ b/db/service.go @@ -45,7 +45,7 @@ var ( // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream // (3) return a new variable to indicate goUpstream -func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { +func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, []common.EventMessage, error) { fieldsDict := make(util.Dict) fieldsDict.Update(fields) tfields := common.FilterLogFields(fields) @@ -57,6 +57,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // ==== deviceRootDocument should always be created from request header ==== var bitmap int var err error + messages := []common.EventMessage{} supportedDocs, err := rHeader.Get(common.HeaderSupportedDocs) if err != nil { log.WithFields(tfields).Warn(err) @@ -115,11 +116,11 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel document, err = c.GetDocument(mac, fields) if err != nil { // TODO what about 404 should be included here - return nil, nil, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, nil, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } deviceVersionMap = RebuildDeviceVersionMap(versions, document.VersionMap()) } else { - return nil, nil, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, nil, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } @@ -130,7 +131,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel cloudRootDocument, err := c.GetRootDocument(mac) if err != nil { if !c.IsDbNotFound(err) { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // no root doc in db, create a new one // NOTE need to clone the deviceRootDocument and set the version "" to avoid device root update was set back to cloud @@ -143,10 +144,10 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel log.WithFields(tfields).Info(line) } if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // the returned err is dbNotFound - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== @@ -174,7 +175,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel log.WithFields(tfields).Info(line) } if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } @@ -182,7 +183,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } for subdocId, subdocument := range document.Items() { @@ -201,11 +202,20 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel newState := common.Deployed subdocument.SetState(&newState) if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } + applicationStatus := "success" + namespace := subdocId + version := cloudVersion + m := common.EventMessage{ + DeviceId: "mac:" + mac, + Namespace: &namespace, + ApplicationStatus: &applicationStatus, + Version: &version, + } + messages = append(messages, m) } } - } switch rootCmpEnum { @@ -213,7 +223,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // create an empty "document" document := common.NewDocument(cloudRootDocument) // no need to update root doc - return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil case common.RootDocumentVersionOnlyChanged, common.RootDocumentMissing: // meta unchanged but subdoc versions change ==> new configs // getDoc, then filter @@ -221,7 +231,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel document, err = c.GetDocument(mac, fields) if err != nil { // 404 should be included here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(err) } } document.SetRootDocument(cloudRootDocument) @@ -229,23 +239,23 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel for _, subdocId := range c.BlockedSubdocIds() { filteredDocument.DeleteSubDocument(subdocId) } - return filteredDocument, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return filteredDocument, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil case common.RootDocumentMetaChanged: // getDoc, send it upstream if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { // 404 should be included here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(err) } } document.SetRootDocument(cloudRootDocument) - return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, true, nil + return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, true, messages, nil } // default, should not come here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil } func GetValuesStr(length int) string { diff --git a/http/multipart.go b/http/multipart.go index c1f29d4..a8676a9 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -128,7 +128,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return status, respHeader, rbytes, nil } - document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, err := db.BuildGetDocument(c, rHeader, route, fields) + document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, messages, err := db.BuildGetDocument(c, rHeader, route, fields) + if s.KafkaProducerEnabled() && s.StateCorrectionEnabled() && len(messages) > 0 { + s.ForwardSuccessKafkaMessages(messages, fields) + } if uconn == nil { if err != nil { if !s.IsDbNotFound(err) { diff --git a/http/otel.go b/http/otel.go index 4c649cc..468035f 100644 --- a/http/otel.go +++ b/http/otel.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 8deb28e..59da474 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -42,6 +42,7 @@ import ( "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" + "github.com/google/uuid" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -1039,3 +1040,36 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess tfields["output_body"] = m log.WithFields(tfields).Info("send") } + +func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMessage, fields log.Fields) { + tfields := common.CopyCoreLogFields(fields) + tfields["logger"] = "kafkaproducer" + tfields["output_topic"] = s.KafkaProducerTopic() + + for _, m := range messages { + if len(m.DeviceId) != 16 { + log.WithFields(tfields).Warn("invalid device_id " + m.DeviceId) + continue + } + mac := m.DeviceId[4:] + transactionUuid := s.AppName() + "_____" + uuid.New().String() + m.TransactionUuid = &transactionUuid + + bbytes, err := json.Marshal(m) + if err != nil { + tfields["logger"] = "error" + log.WithFields(tfields).Error(common.NewError(err)) + return + } + outMessage := &sarama.ProducerMessage{ + Topic: s.KafkaProducerTopic(), + Key: sarama.ByteEncoder(mac), + Value: sarama.ByteEncoder(bbytes), + } + s.Input() <- outMessage + + tfields["output_key"] = mac + tfields["output_body"] = m + log.WithFields(tfields).Info("send") + } +} From 658c0ade33698bf517557a3c67628a30431e72cf Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 3 Aug 2024 10:52:13 -0700 Subject: [PATCH 075/155] revise bitmap test codes --- common/bitmap.go | 169 ++++++ common/bitmap_test.go | 47 ++ common/const_var.go | 127 ----- http/supported_groups_handler_test.go | 184 +++---- util/firmware_bitmap_test.go | 734 +++++++++----------------- 5 files changed, 533 insertions(+), 728 deletions(-) create mode 100644 common/bitmap.go create mode 100644 common/bitmap_test.go diff --git a/common/bitmap.go b/common/bitmap.go new file mode 100644 index 0000000..dd9c777 --- /dev/null +++ b/common/bitmap.go @@ -0,0 +1,169 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +// header X-System-Supported-Docs +type BitMaskTuple struct { + GroupBit int + CpeBit int +} + +// The group based bitmaps will be merged into 1 cpe bitmap +// 1: []BitMaskTuple{ // meta_group_id: defined by RDK +// +// BitMaskTuple{1, 1}, // {"index_of_bit_from_lsb" for a group bitmap, "index_of_bit_from_lsb" for the cpe bitmap +var ( + SupportedDocsBitMaskMap = map[int][]BitMaskTuple{ + 1: { + {1, 1}, + {2, 2}, + {3, 3}, + {4, 4}, + {5, 5}, + {6, 6}, + {7, 29}, // connectedbuilding + {8, 35}, // xmspeedboost + }, + 2: { + {1, 7}, + {2, 8}, + {3, 9}, + }, + 3: { + {1, 10}, + }, + 4: { + {1, 11}, + }, + 5: { + {1, 12}, + }, + 6: { + {1, 13}, // mesh + {2, 31}, // clienttosteeringprofile + {3, 36}, // meshsteeringprofiles + {4, 37}, // wifistatsconfig + {5, 38}, // mwoconfigs + {6, 39}, // interference + {7, 34}, // wifimotionsettings + }, + 7: { + {1, 14}, + }, + 8: { + {1, 15}, + {2, 32}, + {3, 33}, + }, + 9: { + {1, 16}, + {2, 17}, + }, + 10: { + {1, 18}, + {2, 19}, + }, + 11: { + {1, 20}, + {2, 25}, + }, + 12: { + {1, 21}, + {2, 23}, + }, + 13: { + {1, 22}, + }, + 14: { + {1, 24}, + }, + 15: { + {1, 26}, + {2, 27}, + }, + 16: { + {1, 28}, + }, + 17: { + {1, 30}, + }, + } +) + +var ( + SubdocBitIndexMap = map[string]int{ + "portforwarding": 1, + "lan": 2, + "wan": 3, + "macbinding": 4, + "hotspot": 5, + "bridge": 6, + "privatessid": 7, + "homessid": 8, + "radio": 9, + "moca": 10, + "xdns": 11, + "advsecurity": 12, + "mesh": 13, + "aker": 14, + "telemetry": 15, + "statusreport": 16, + "trafficreport": 17, + "interfacereport": 18, + "radioreport": 19, + "telcovoip": 20, + "wanmanager": 21, + "voiceservice": 22, + "wanfailover": 23, + "cellularconfig": 24, + "telcovoice": 25, + "gwfailover": 26, + "gwrestore": 27, + "prioritizedmacs": 28, + "connectedbuilding": 29, + "lldqoscontrol": 30, + "clienttosteeringprofile": 31, + "defaultrfc": 32, + "rfc": 33, + "wifimotionsettings": 34, + "xmspeedboost": 35, + "meshsteeringprofiles": 36, + "wifistatsconfig": 37, + "mwoconfigs": 38, + "interference": 39, + } +) + +func GetDefaultSupportedSubdocMap() map[string]bool { + m := make(map[string]bool) + for k := range SubdocBitIndexMap { + m[k] = false + } + return m +} + +func BuildSupportedSubdocMapWithDefaults(supportedSubdocIds []string) map[string]bool { + m := make(map[string]bool) + for k := range SubdocBitIndexMap { + m[k] = false + } + for _, s := range supportedSubdocIds { + m[s] = true + } + return m +} diff --git a/common/bitmap_test.go b/common/bitmap_test.go new file mode 100644 index 0000000..9c75139 --- /dev/null +++ b/common/bitmap_test.go @@ -0,0 +1,47 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "slices" + "testing" + + "gotest.tools/assert" +) + +func TestGetDefaultSupportedSubdocMap(t *testing.T) { + m := GetDefaultSupportedSubdocMap() + assert.Equal(t, len(m), len(SubdocBitIndexMap)) +} + +func TestBuildSupportedSubdocMapWithDefaults(t *testing.T) { + supportedSubdocIds := []string{ + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + } + m := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.Equal(t, len(m), len(SubdocBitIndexMap)) + for k := range m { + if m[k] { + assert.Assert(t, slices.Contains(supportedSubdocIds, k)) + } + } +} diff --git a/common/const_var.go b/common/const_var.go index 6d57e4a..513807e 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -114,133 +114,6 @@ const ( SkipDbUpdate = "skip-db-update" ) -// header X-System-Supported-Docs -type BitMaskTuple struct { - GroupBit int - CpeBit int -} - -// The group based bitmaps will be merged into 1 cpe bitmap -// 1: []BitMaskTuple{ // meta_group_id: defined by RDK -// -// BitMaskTuple{1, 1}, // {"index_of_bit_from_lsb" for a group bitmap, "index_of_bit_from_lsb" for the cpe bitmap -var ( - SupportedDocsBitMaskMap = map[int][]BitMaskTuple{ - 1: { - {1, 1}, - {2, 2}, - {3, 3}, - {4, 4}, - {5, 5}, - {6, 6}, - {7, 29}, // connectedbuilding - {8, 35}, // xmspeedboost - }, - 2: { - {1, 7}, - {2, 8}, - {3, 9}, - }, - 3: { - {1, 10}, - }, - 4: { - {1, 11}, - }, - 5: { - {1, 12}, - }, - 6: { - {1, 13}, // mesh - {2, 31}, // clienttosteeringprofile - {3, 36}, // meshsteeringprofiles - {4, 37}, // wifistatsconfig - {5, 38}, // mwoconfigs - {6, 39}, // interference - {7, 34}, // wifimotionsettings - }, - 7: { - {1, 14}, - }, - 8: { - {1, 15}, - {2, 32}, - {3, 33}, - }, - 9: { - {1, 16}, - {2, 17}, - }, - 10: { - {1, 18}, - {2, 19}, - }, - 11: { - {1, 20}, - {2, 25}, - }, - 12: { - {1, 21}, - {2, 23}, - }, - 13: { - {1, 22}, - }, - 14: { - {1, 24}, - }, - 15: { - {1, 26}, - {2, 27}, - }, - 16: { - {1, 28}, - }, - 17: { - {1, 30}, - }, - } -) - -var ( - SubdocBitIndexMap = map[string]int{ - "portforwarding": 1, - "lan": 2, - "wan": 3, - "macbinding": 4, - "hotspot": 5, - "bridge": 6, - "privatessid": 7, - "homessid": 8, - "radio": 9, - "moca": 10, - "xdns": 11, - "advsecurity": 12, - "mesh": 13, - "aker": 14, - "telemetry": 15, - "statusreport": 16, - "trafficreport": 17, - "interfacereport": 18, - "radioreport": 19, - "telcovoip": 20, - "wanmanager": 21, - "voiceservice": 22, - "wanfailover": 23, - "cellularconfig": 24, - "telcovoice": 25, - "gwfailover": 26, - "gwrestore": 27, - "prioritizedmacs": 28, - "connectedbuilding": 29, - "lldqoscontrol": 30, - "clienttosteeringprofile": 31, - "defaultrfc": 32, - "rfc": 33, - "wifimotionsettings": 34, - "xmspeedboost": 35, - } -) var ( SupportedPokeDocs = []string{"primary", "telemetry"} diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 13ffbe9..833d656 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -46,6 +46,7 @@ func TestSupportedGroupsHandler(t *testing.T) { res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusNotFound) // call GET /config to add supported-doc header @@ -66,43 +67,21 @@ func TestSupportedGroupsHandler(t *testing.T) { assert.Equal(t, bitmap, rdoc.Bitmap) // call GET /supported_groups - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, - "wanfailover": false, - "cellularconfig": false, - "gwfailover": false, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, - "clienttosteeringprofile": false, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) // call GET /supported_groups to verify response req, err = http.NewRequest("GET", sgUrl, nil) @@ -115,7 +94,7 @@ func TestSupportedGroupsHandler(t *testing.T) { var supportedGroupsGetResponse common.SupportedGroupsGetResponse err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) // ==== step 2 add lan data ==== subdocId := "lan" @@ -132,6 +111,7 @@ func TestSupportedGroupsHandler(t *testing.T) { res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusOK) // get @@ -170,7 +150,7 @@ func TestSupportedGroupsHandler(t *testing.T) { err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) // ==== step 5 setup supported docs for fw version 2 ===== sids := strings.Split(rdkSupportedDocsHeaderStr, ",") @@ -179,17 +159,17 @@ func TestSupportedGroupsHandler(t *testing.T) { group1Bitmap, err := util.BitarrayToBitmap(newGroup1Bitarray) assert.NilError(t, err) sids[0] = fmt.Sprintf("%v", group1Bitmap) - expectedEnabled["wan"] = false - expectedEnabled["macbinding"] = false - expectedEnabled["hotspot"] = true - expectedEnabled["bridge"] = true + supportedSubdocMap["wan"] = false + supportedSubdocMap["macbinding"] = false + supportedSubdocMap["hotspot"] = true + supportedSubdocMap["bridge"] = true newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" group2Bitmap, err := util.BitarrayToBitmap(newGroup2Bitarray) assert.NilError(t, err) sids[1] = fmt.Sprintf("%v", group2Bitmap) - expectedEnabled["privatessid"] = false - expectedEnabled["radio"] = true + supportedSubdocMap["privatessid"] = false + supportedSubdocMap["radio"] = true rdkSupportedDocsHeaderStr = strings.Join(sids, ",") @@ -218,7 +198,7 @@ func TestSupportedGroupsHandler(t *testing.T) { err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) } func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { @@ -235,6 +215,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusNotFound) // call GET /config to add supported-doc header @@ -255,43 +236,22 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { assert.Equal(t, bitmap, rdoc.Bitmap) // call GET /supported_groups - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": false, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": true, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, - "gwfailover": false, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, - "clienttosteeringprofile": false, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telcovoice", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) // call GET /supported_groups to verify response req, err = http.NewRequest("GET", sgUrl, nil) @@ -304,7 +264,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { var supportedGroupsGetResponse common.SupportedGroupsGetResponse err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) } func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testing.T) { @@ -321,6 +281,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) assert.NilError(t, err) + _ = rbytes assert.Equal(t, res.StatusCode, http.StatusNotFound) // call GET /config to add supported-doc header @@ -341,43 +302,28 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin assert.Equal(t, bitmap, rdoc.Bitmap) // call GET /supported_groups - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwfailover": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, - "clienttosteeringprofile": false, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "gwfailover", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) // call GET /supported_groups to verify response req, err = http.NewRequest("GET", sgUrl, nil) @@ -390,5 +336,5 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin var supportedGroupsGetResponse common.SupportedGroupsGetResponse err = json.Unmarshal(rbytes, &supportedGroupsGetResponse) assert.NilError(t, err) - assert.DeepEqual(t, expectedEnabled, supportedGroupsGetResponse.Data.Groups) + assert.DeepEqual(t, supportedSubdocMap, supportedGroupsGetResponse.Data.Groups) } diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 28c3159..8572f68 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -22,6 +22,7 @@ import ( "strings" "testing" + "github.com/rdkcentral/webconfig/common" "gotest.tools/assert" ) @@ -155,20 +156,15 @@ func TestParseCustomizedGroupBitarray(t *testing.T) { } parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocIds := []string{} + for k, v := range expectedEnabled { + if v { + supportedSubdocIds = append(supportedSubdocIds, k) + } + } + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseTelcovoipAndWanmanager(t *testing.T) { @@ -176,55 +172,30 @@ func TestParseTelcovoipAndWanmanager(t *testing.T) { sids := strings.Split(rdkSupportedDocsHeaderStr, ",") // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": false, - "radio": false, - "moca": false, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": true, - "telcovoice": false, - "wanmanager": true, - "voiceservice": false, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "privatessid", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "telcovoip", + "wanmanager", } rdkSupportedDocsHeaderStr = strings.Join(sids, ",") cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) + for _, subdocId := range supportedSubdocIds { + assert.Assert(t, IsSubdocSupported(cpeBitmap, subdocId)) } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestBitmapParsing(t *testing.T) { @@ -233,110 +204,52 @@ func TestBitmapParsing(t *testing.T) { rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,33554435,50331649,67108865,83886081,100663297,117440513,134217729", newBitmap) // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": false, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "macbinding", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseVoiceService(t *testing.T) { rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809" - // sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": true, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": true, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "voiceservice", } - // rdkSupportedDocsHeaderStr = strings.Join(sids, ",") cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["cellularconfig"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestManualBitmap(t *testing.T) { @@ -355,425 +268,282 @@ func TestParseSupportedDocsWithNewGroups(t *testing.T) { rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", ss, xBitValue) // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": true, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": true, - "cellularconfig": true, + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "voiceservice", + "cellularconfig", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["wanfailover"] = false - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { rdkSupportedDocsHeaderStr := "16777231,33554435,67108865,100663297,117440513,134217729,201326595,234881025" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": false, - "aker": true, - "bridge": false, - "cellularconfig": true, - "homessid": true, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, + supportedSubdocIds := []string{ + "aker", + "cellularconfig", + "homessid", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549378,201326595" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "homessid": false, - "hotspot": false, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": false, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoip": false, - "telcovoice": true, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": true, - "xdns": true, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telcovoice", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - expectedEnabled["gwfailover"] = false - expectedEnabled["gwrestore"] = false - expectedEnabled["prioritizedmacs"] = false - expectedEnabled["connectedbuilding"] = false - expectedEnabled["lldqoscontrol"] = false - expectedEnabled["clienttosteeringprofile"] = false - expectedEnabled["defaultrfc"] = false - expectedEnabled["rfc"] = false - expectedEnabled["wifimotionsettings"] = false - expectedEnabled["xmspeedboost"] = false - - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": false, - "connectedbuilding": false, - "lldqoscontrol": false, - "clienttosteeringprofile": false, - "rfc": false, - "defaultrfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,251658241,268435457" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": false, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": false, - "lldqoscontrol": false, - "clienttosteeringprofile": false, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *testing.T) { rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241,268435457,285212673" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, - "clienttosteeringprofile": false, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217729,201326594,218103809,251658241,268435457,285212673" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, - "clienttosteeringprofile": true, - "defaultrfc": false, - "rfc": false, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } func TestParseSupportedDocsHeaderRfc(t *testing.T) { rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" // build expected - expectedEnabled := map[string]bool{ - "advsecurity": true, - "aker": true, - "bridge": false, - "cellularconfig": false, - "gwfailover": true, - "homessid": true, - "hotspot": true, - "interfacereport": false, - "lan": true, - "macbinding": true, - "mesh": true, - "moca": true, - "portforwarding": true, - "privatessid": true, - "radio": false, - "radioreport": false, - "statusreport": false, - "telcovoice": false, - "telcovoip": false, - "telemetry": true, - "trafficreport": false, - "voiceservice": true, - "wan": true, - "wanfailover": true, - "wanmanager": false, - "xdns": true, - "gwrestore": false, - "prioritizedmacs": true, - "connectedbuilding": true, - "lldqoscontrol": true, - "clienttosteeringprofile": true, - "defaultrfc": true, - "rfc": true, - "wifimotionsettings": false, - "xmspeedboost": false, + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "defaultrfc", + "rfc", } cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderHcm(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", } + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) parsedSupportedMap := GetSupportedMap(cpeBitmap) - assert.DeepEqual(t, parsedSupportedMap, expectedEnabled) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } From 3c32f125f99f381c070ada060952c45cf25dfb45 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 30 Jul 2024 19:16:43 -0700 Subject: [PATCH 076/155] generate notifications if states are corrected by versions from devices --- db/cassandra/document_test.go | 2 +- db/service.go | 42 ++++++++++++++++++++++------------- http/multipart.go | 5 ++++- http/webconfig_server.go | 34 ++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 18 deletions(-) diff --git a/db/cassandra/document_test.go b/db/cassandra/document_test.go index b314690..8e63844 100644 --- a/db/cassandra/document_test.go +++ b/db/cassandra/document_test.go @@ -206,7 +206,7 @@ func TestBlockedSubdocIds(t *testing.T) { rHeader.Set(common.HeaderSupportedDocs, rdkSupportedDocsHeaderStr) - document, _, _, _, _, err := db.BuildGetDocument(tdbclient, rHeader, common.RouteHttp, fields) + document, _, _, _, _, _, err := db.BuildGetDocument(tdbclient, rHeader, common.RouteHttp, fields) assert.NilError(t, err) assert.Assert(t, document.Length() == 3) versionMap := document.VersionMap() diff --git a/db/service.go b/db/service.go index 9493527..bce94fa 100644 --- a/db/service.go +++ b/db/service.go @@ -45,7 +45,7 @@ var ( // (1) need to have a dedicate function update states AFTER this function is executed // (2) read from the existing "root_document" table and build those into the header for upstream // (3) return a new variable to indicate goUpstream -func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, error) { +func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fields log.Fields) (*common.Document, *common.RootDocument, *common.RootDocument, map[string]string, bool, []common.EventMessage, error) { fieldsDict := make(util.Dict) fieldsDict.Update(fields) tfields := common.FilterLogFields(fields) @@ -57,6 +57,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // ==== deviceRootDocument should always be created from request header ==== var bitmap int var err error + messages := []common.EventMessage{} supportedDocs, err := rHeader.Get(common.HeaderSupportedDocs) if err != nil { log.WithFields(tfields).Warn(err) @@ -115,11 +116,11 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel document, err = c.GetDocument(mac, fields) if err != nil { // TODO what about 404 should be included here - return nil, nil, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, nil, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } deviceVersionMap = RebuildDeviceVersionMap(versions, document.VersionMap()) } else { - return nil, nil, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, nil, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } @@ -130,7 +131,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel cloudRootDocument, err := c.GetRootDocument(mac) if err != nil { if !c.IsDbNotFound(err) { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // no root doc in db, create a new one // NOTE need to clone the deviceRootDocument and set the version "" to avoid device root update was set back to cloud @@ -143,10 +144,10 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel log.WithFields(tfields).Info(line) } if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // the returned err is dbNotFound - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== @@ -174,7 +175,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel log.WithFields(tfields).Info(line) } if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } @@ -182,7 +183,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } } for subdocId, subdocument := range document.Items() { @@ -201,11 +202,20 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel newState := common.Deployed subdocument.SetState(&newState) if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } + applicationStatus := "success" + namespace := subdocId + version := cloudVersion + m := common.EventMessage{ + DeviceId: "mac:" + mac, + Namespace: &namespace, + ApplicationStatus: &applicationStatus, + Version: &version, + } + messages = append(messages, m) } } - } switch rootCmpEnum { @@ -213,7 +223,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // create an empty "document" document := common.NewDocument(cloudRootDocument) // no need to update root doc - return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil case common.RootDocumentVersionOnlyChanged, common.RootDocumentMissing: // meta unchanged but subdoc versions change ==> new configs // getDoc, then filter @@ -221,7 +231,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel document, err = c.GetDocument(mac, fields) if err != nil { // 404 should be included here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(err) } } document.SetRootDocument(cloudRootDocument) @@ -229,23 +239,23 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel for _, subdocId := range c.BlockedSubdocIds() { filteredDocument.DeleteSubDocument(subdocId) } - return filteredDocument, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return filteredDocument, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil case common.RootDocumentMetaChanged: // getDoc, send it upstream if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { // 404 should be included here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, common.NewError(err) + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(err) } } document.SetRootDocument(cloudRootDocument) - return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, true, nil + return document, cloudRootDocument, deviceRootDocument, deviceVersionMap, true, messages, nil } // default, should not come here - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, nil } func GetValuesStr(length int) string { diff --git a/http/multipart.go b/http/multipart.go index c1f29d4..a8676a9 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -128,7 +128,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return status, respHeader, rbytes, nil } - document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, err := db.BuildGetDocument(c, rHeader, route, fields) + document, oldRootDocument, newRootDocument, deviceVersionMap, postUpstream, messages, err := db.BuildGetDocument(c, rHeader, route, fields) + if s.KafkaProducerEnabled() && s.StateCorrectionEnabled() && len(messages) > 0 { + s.ForwardSuccessKafkaMessages(messages, fields) + } if uconn == nil { if err != nil { if !s.IsDbNotFound(err) { diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 8deb28e..59da474 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -42,6 +42,7 @@ import ( "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" + "github.com/google/uuid" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -1039,3 +1040,36 @@ func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMess tfields["output_body"] = m log.WithFields(tfields).Info("send") } + +func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMessage, fields log.Fields) { + tfields := common.CopyCoreLogFields(fields) + tfields["logger"] = "kafkaproducer" + tfields["output_topic"] = s.KafkaProducerTopic() + + for _, m := range messages { + if len(m.DeviceId) != 16 { + log.WithFields(tfields).Warn("invalid device_id " + m.DeviceId) + continue + } + mac := m.DeviceId[4:] + transactionUuid := s.AppName() + "_____" + uuid.New().String() + m.TransactionUuid = &transactionUuid + + bbytes, err := json.Marshal(m) + if err != nil { + tfields["logger"] = "error" + log.WithFields(tfields).Error(common.NewError(err)) + return + } + outMessage := &sarama.ProducerMessage{ + Topic: s.KafkaProducerTopic(), + Key: sarama.ByteEncoder(mac), + Value: sarama.ByteEncoder(bbytes), + } + s.Input() <- outMessage + + tfields["output_key"] = mac + tfields["output_body"] = m + log.WithFields(tfields).Info("send") + } +} From 423ac08a47233b9fa44cc250a1c5397b2cf9d1a0 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 7 Aug 2024 14:58:20 -0700 Subject: [PATCH 077/155] Return 504 if xconf GET returns timeout/context_cancellation --- http/http_client.go | 15 ++++- http/supplementary_handler_test.go | 95 +++++++++++++++++++++++++++++- 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index e7e66bc..63d989c 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -32,10 +32,10 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) @@ -263,6 +263,15 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] fields[fmt.Sprintf("%v_status", loggerName)] = res.StatusCode rbytes, err := io.ReadAll(res.Body) if err != nil { + // XPC-23206 catch the timeout/context_cancellation error + // ex: context deadline exceeded (Client.Timeout or context cancellation while reading body) + lowerErrText := strings.ToLower(err.Error()) + if strings.Contains(lowerErrText, "timeout") { + err = common.RemoteHttpError{ + Message: err.Error(), + StatusCode: http.StatusGatewayTimeout, + } + } fields[errorKey] = err.Error() if userAgent != "mget" { log.WithFields(fields).Info(endMessage) diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index 942b0f8..ff6f445 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -797,3 +797,96 @@ func TestSupplementaryApiBadRequest(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusBadRequest) } + +func TestSupplementaryXconfTimeout(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== setup mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(time.Duration(1) * time.Second) + w.WriteHeader(http.StatusOK) + w.Write([]byte(mockProfileResponse)) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + server.XconfConnector.SetXconfHost(mockServer.URL) + server.XconfConnector.HttpClient.Client.Timeout = time.Duration(500) * time.Millisecond + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) +} + +type errorRoundTripper struct{} + +func (ert *errorRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // Return a response with an error-inducing reader + return &http.Response{ + Body: io.NopCloser(&errorReader{}), + }, nil +} + +// errorReader simulates an error when reading +type errorReader struct{} + +func (er *errorReader) Read(p []byte) (int, error) { + return 0, fmt.Errorf("context deadline exceeded (Client.Timeout or context cancellation while reading body)") +} + +func TestSupplementaryXconfReadAllErr(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== setup mock client ==== + mockedClient := &http.Client{ + Transport: &errorRoundTripper{}, + } + + server.XconfConnector.HttpClient.Client = mockedClient + + // ==== setup data ==== + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 verify /config expect 200 with 1 mpart ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + + // add headers + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, "TG1682G") + req.Header.Set(common.HeaderPartnerID, "comcast") + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, "TG1682_3.14p9s6_PROD_sey") + + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) +} From 1ad7969441a11e97bb5f3fa8981630af554e41af Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Sep 2024 15:16:27 -0700 Subject: [PATCH 078/155] Add debug loggings for 403 error analysis --- http/http_client.go | 6 +++--- http/supplementary_handler_test.go | 2 +- http/webconfig_server.go | 31 ++++++++++++++++++++++++++++-- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index 63d989c..aad3085 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( @@ -32,10 +32,10 @@ import ( "strings" "time" - "github.com/go-akka/configuration" - "github.com/google/uuid" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" + "github.com/go-akka/configuration" + "github.com/google/uuid" log "github.com/sirupsen/logrus" ) diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index ff6f445..54e2642 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 59da474..b3884af 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -22,6 +22,7 @@ import ( "crypto/tls" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -81,6 +82,7 @@ var ( "privatessid", "homessid", } + codec *security.AesCodec ) type WebconfigServer struct { @@ -368,6 +370,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + var tokenErr error if len(token) > 0 { params := mux.Vars(r) mac, ok := params["mac"] @@ -382,18 +385,20 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { fields := xw.Audit() fields["src_partner"] = partnerId partnerId = "unknown" + tokenErr = common.NewError(err) } xw.SetPartnerId(partnerId) } else { - xw.LogDebug(r, "token", fmt.Sprintf("CpeMiddleware() VerifyCpeToken()=false, err=%v", err)) + tokenErr = common.NewError(err) } } else { - xw.LogDebug(r, "token", "CpeMiddleware() error no token") + tokenErr = common.NewError(errors.New("CpeMiddleware() error no token")) } if isValid { next.ServeHTTP(xw, r) } else { + s.LogToken(xw, token, tokenErr) Error(xw, http.StatusForbidden, nil) } } @@ -1073,3 +1078,25 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes log.WithFields(tfields).Info("send") } } + +func (s *WebconfigServer) LogToken(xw *XResponseWriter, token string, tokenErr error) { + fields := xw.Audit() + fields["logger"] = "token" + tfields := common.FilterLogFields(fields) + + if codec == nil { + codec, _ = security.NewAesCodec(s.Config) + } + + if codec == nil { + tfields["plaintoken"] = token + } else { + var encToken string + if encryptedB64, err := codec.Encrypt(token); err == nil { + encToken = encryptedB64 + } + tfields["enctoken"] = encToken + } + + log.WithFields(tfields).Debug(tokenErr) +} From 367450ddf0c83b24dfb4a23c92fbfbe01849ec43 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Sep 2024 16:37:06 -0700 Subject: [PATCH 079/155] Add debug loggings for 403 error analysis --- http/webconfig_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index b3884af..da6fe3a 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -398,6 +398,8 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if isValid { next.ServeHTTP(xw, r) } else { + fields := xw.Audit() + fields["full_header"] = util.HeaderToMap(getFilteredHeader(r, nil)) s.LogToken(xw, token, tokenErr) Error(xw, http.StatusForbidden, nil) } From 7e2b44c04238a061da125870e7dcbc3fd7d6f5ea Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 5 Sep 2024 11:54:46 -0700 Subject: [PATCH 080/155] Add debug loggings for 403 error analysis --- common/log_fields.go | 11 ++++++- common/log_fields_test.go | 69 +++++++++++++++++++++++++++++++++++++++ http/webconfig_server.go | 43 ++++++++++++++++-------- 3 files changed, 108 insertions(+), 15 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index 8ec4f19..5ce267b 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -18,6 +18,8 @@ package common import ( + "maps" + log "github.com/sirupsen/logrus" ) @@ -39,7 +41,14 @@ var ( func FilterLogFields(src log.Fields, excludes ...string) log.Fields { fields := log.Fields{} for k, v := range src { - fields[k] = v + switch ty := v.(type) { + case map[string]string: + fields[k] = maps.Clone(ty) + case map[string]interface{}: + fields[k] = maps.Clone(ty) + default: + fields[k] = ty + } } for _, x := range unloggedFields { diff --git a/common/log_fields_test.go b/common/log_fields_test.go index 4a95b9a..960af77 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -119,4 +119,73 @@ func TestCopyCoreLogFields(t *testing.T) { } copied := CopyCoreLogFields(src) assert.DeepEqual(t, copied, expected) + + body["violet"] = "purple" + +} + +func TestFilterLogFieldsWithItfMap(t *testing.T) { + weekday := map[string]interface{}{ + "mon": 1, + "tue": 2, + "wed": 3, + "thu": 4, + } + + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "weekday": weekday, + } + + filtered := FilterLogFields(src) + assert.DeepEqual(t, src, filtered) + + itf, ok := filtered["weekday"] + assert.Assert(t, ok) + fw := itf.(map[string]interface{}) + fw["fri"] = 5 + + itf, ok = src["weekday"] + assert.Assert(t, ok) + sw := itf.(map[string]interface{}) + assert.Assert(t, len(sw) == 4) +} + +func TestFilterLogFieldsWithStrMap(t *testing.T) { + weekday := map[string]string{ + "mon": "1", + "tue": "2", + "wed": "3", + "thu": "4", + } + + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "weekday": weekday, + } + + filtered := FilterLogFields(src) + assert.DeepEqual(t, src, filtered) + + itf, ok := filtered["weekday"] + assert.Assert(t, ok) + fw := itf.(map[string]string) + fw["fri"] = "5" + + itf, ok = src["weekday"] + assert.Assert(t, ok) + sw := itf.(map[string]string) + assert.Assert(t, len(sw) == 4) } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index da6fe3a..05d4ea2 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -66,6 +66,7 @@ const ( defaultTraceparentParentID = "0000000000000001" defaultTracestateVendorID = "webconfig" defaultSupplementaryAppendingEnabled = true + authPrefixLength = 60 ) var ( @@ -370,6 +371,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + authorization := r.Header.Get("Authorization") var tokenErr error if len(token) > 0 { params := mux.Vars(r) @@ -398,9 +400,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if isValid { next.ServeHTTP(xw, r) } else { - fields := xw.Audit() - fields["full_header"] = util.HeaderToMap(getFilteredHeader(r, nil)) - s.LogToken(xw, token, tokenErr) + s.LogToken(xw, authorization, token, tokenErr) Error(xw, http.StatusForbidden, nil) } } @@ -1081,23 +1081,38 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes } } -func (s *WebconfigServer) LogToken(xw *XResponseWriter, token string, tokenErr error) { +func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token string, tokenErr error) { fields := xw.Audit() fields["logger"] = "token" tfields := common.FilterLogFields(fields) - - if codec == nil { - codec, _ = security.NewAesCodec(s.Config) + var headerMap map[string]string + var isObfuscated bool + if itf, ok := tfields["header"]; ok { + headerMap = itf.(map[string]string) + if len(headerMap) > 0 { + ss := authorization + if len(ss) > authPrefixLength { + ss = authorization[:authPrefixLength] + "****" + isObfuscated = true + } + headerMap["Authorization"] = ss + } } - if codec == nil { - tfields["plaintoken"] = token - } else { - var encToken string - if encryptedB64, err := codec.Encrypt(token); err == nil { - encToken = encryptedB64 + if isObfuscated { + if codec == nil { + codec, _ = security.NewAesCodec(s.Config) + } + + if codec == nil { + tfields["plaintoken"] = token + } else { + var encToken string + if encryptedB64, err := codec.Encrypt(token); err == nil { + encToken = encryptedB64 + } + tfields["enctoken"] = encToken } - tfields["enctoken"] = encToken } log.WithFields(tfields).Debug(tokenErr) From 3a24698ce19636b7a20965af8affbd112104e294 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Sep 2024 15:16:27 -0700 Subject: [PATCH 081/155] Add debug loggings for 403 error analysis --- http/http_client.go | 6 +++--- http/supplementary_handler_test.go | 2 +- http/webconfig_server.go | 31 ++++++++++++++++++++++++++++-- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index 63d989c..aad3085 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( @@ -32,10 +32,10 @@ import ( "strings" "time" - "github.com/go-akka/configuration" - "github.com/google/uuid" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" + "github.com/go-akka/configuration" + "github.com/google/uuid" log "github.com/sirupsen/logrus" ) diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index ff6f445..54e2642 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 - */ +*/ package http import ( diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 59da474..b3884af 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -22,6 +22,7 @@ import ( "crypto/tls" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -81,6 +82,7 @@ var ( "privatessid", "homessid", } + codec *security.AesCodec ) type WebconfigServer struct { @@ -368,6 +370,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + var tokenErr error if len(token) > 0 { params := mux.Vars(r) mac, ok := params["mac"] @@ -382,18 +385,20 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { fields := xw.Audit() fields["src_partner"] = partnerId partnerId = "unknown" + tokenErr = common.NewError(err) } xw.SetPartnerId(partnerId) } else { - xw.LogDebug(r, "token", fmt.Sprintf("CpeMiddleware() VerifyCpeToken()=false, err=%v", err)) + tokenErr = common.NewError(err) } } else { - xw.LogDebug(r, "token", "CpeMiddleware() error no token") + tokenErr = common.NewError(errors.New("CpeMiddleware() error no token")) } if isValid { next.ServeHTTP(xw, r) } else { + s.LogToken(xw, token, tokenErr) Error(xw, http.StatusForbidden, nil) } } @@ -1073,3 +1078,25 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes log.WithFields(tfields).Info("send") } } + +func (s *WebconfigServer) LogToken(xw *XResponseWriter, token string, tokenErr error) { + fields := xw.Audit() + fields["logger"] = "token" + tfields := common.FilterLogFields(fields) + + if codec == nil { + codec, _ = security.NewAesCodec(s.Config) + } + + if codec == nil { + tfields["plaintoken"] = token + } else { + var encToken string + if encryptedB64, err := codec.Encrypt(token); err == nil { + encToken = encryptedB64 + } + tfields["enctoken"] = encToken + } + + log.WithFields(tfields).Debug(tokenErr) +} From 820b6780c038e0f205b9b40eb38ec375027e277f Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Sep 2024 16:37:06 -0700 Subject: [PATCH 082/155] Add debug loggings for 403 error analysis --- http/webconfig_server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index b3884af..da6fe3a 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -398,6 +398,8 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if isValid { next.ServeHTTP(xw, r) } else { + fields := xw.Audit() + fields["full_header"] = util.HeaderToMap(getFilteredHeader(r, nil)) s.LogToken(xw, token, tokenErr) Error(xw, http.StatusForbidden, nil) } From 98e9e7dfc3edf77c9fe1fcfbc9c43f5e36af72ff Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 5 Sep 2024 11:54:46 -0700 Subject: [PATCH 083/155] Add debug loggings for 403 error analysis --- common/log_fields.go | 11 ++++++- common/log_fields_test.go | 69 +++++++++++++++++++++++++++++++++++++++ http/webconfig_server.go | 43 ++++++++++++++++-------- 3 files changed, 108 insertions(+), 15 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index 8ec4f19..5ce267b 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -18,6 +18,8 @@ package common import ( + "maps" + log "github.com/sirupsen/logrus" ) @@ -39,7 +41,14 @@ var ( func FilterLogFields(src log.Fields, excludes ...string) log.Fields { fields := log.Fields{} for k, v := range src { - fields[k] = v + switch ty := v.(type) { + case map[string]string: + fields[k] = maps.Clone(ty) + case map[string]interface{}: + fields[k] = maps.Clone(ty) + default: + fields[k] = ty + } } for _, x := range unloggedFields { diff --git a/common/log_fields_test.go b/common/log_fields_test.go index 4a95b9a..960af77 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -119,4 +119,73 @@ func TestCopyCoreLogFields(t *testing.T) { } copied := CopyCoreLogFields(src) assert.DeepEqual(t, copied, expected) + + body["violet"] = "purple" + +} + +func TestFilterLogFieldsWithItfMap(t *testing.T) { + weekday := map[string]interface{}{ + "mon": 1, + "tue": 2, + "wed": 3, + "thu": 4, + } + + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "weekday": weekday, + } + + filtered := FilterLogFields(src) + assert.DeepEqual(t, src, filtered) + + itf, ok := filtered["weekday"] + assert.Assert(t, ok) + fw := itf.(map[string]interface{}) + fw["fri"] = 5 + + itf, ok = src["weekday"] + assert.Assert(t, ok) + sw := itf.(map[string]interface{}) + assert.Assert(t, len(sw) == 4) +} + +func TestFilterLogFieldsWithStrMap(t *testing.T) { + weekday := map[string]string{ + "mon": "1", + "tue": "2", + "wed": "3", + "thu": "4", + } + + src := log.Fields{ + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "weekday": weekday, + } + + filtered := FilterLogFields(src) + assert.DeepEqual(t, src, filtered) + + itf, ok := filtered["weekday"] + assert.Assert(t, ok) + fw := itf.(map[string]string) + fw["fri"] = "5" + + itf, ok = src["weekday"] + assert.Assert(t, ok) + sw := itf.(map[string]string) + assert.Assert(t, len(sw) == 4) } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index da6fe3a..05d4ea2 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -66,6 +66,7 @@ const ( defaultTraceparentParentID = "0000000000000001" defaultTracestateVendorID = "webconfig" defaultSupplementaryAppendingEnabled = true + authPrefixLength = 60 ) var ( @@ -370,6 +371,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + authorization := r.Header.Get("Authorization") var tokenErr error if len(token) > 0 { params := mux.Vars(r) @@ -398,9 +400,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if isValid { next.ServeHTTP(xw, r) } else { - fields := xw.Audit() - fields["full_header"] = util.HeaderToMap(getFilteredHeader(r, nil)) - s.LogToken(xw, token, tokenErr) + s.LogToken(xw, authorization, token, tokenErr) Error(xw, http.StatusForbidden, nil) } } @@ -1081,23 +1081,38 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes } } -func (s *WebconfigServer) LogToken(xw *XResponseWriter, token string, tokenErr error) { +func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token string, tokenErr error) { fields := xw.Audit() fields["logger"] = "token" tfields := common.FilterLogFields(fields) - - if codec == nil { - codec, _ = security.NewAesCodec(s.Config) + var headerMap map[string]string + var isObfuscated bool + if itf, ok := tfields["header"]; ok { + headerMap = itf.(map[string]string) + if len(headerMap) > 0 { + ss := authorization + if len(ss) > authPrefixLength { + ss = authorization[:authPrefixLength] + "****" + isObfuscated = true + } + headerMap["Authorization"] = ss + } } - if codec == nil { - tfields["plaintoken"] = token - } else { - var encToken string - if encryptedB64, err := codec.Encrypt(token); err == nil { - encToken = encryptedB64 + if isObfuscated { + if codec == nil { + codec, _ = security.NewAesCodec(s.Config) + } + + if codec == nil { + tfields["plaintoken"] = token + } else { + var encToken string + if encryptedB64, err := codec.Encrypt(token); err == nil { + encToken = encryptedB64 + } + tfields["enctoken"] = encToken } - tfields["enctoken"] = encToken } log.WithFields(tfields).Debug(tokenErr) From bdd12bb835fca64b8a000f66963224f578db3ad2 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 10 Sep 2024 16:11:41 -0700 Subject: [PATCH 084/155] return 404 for NONE-REBOOT without subdocs --- http/factory_reset_upstream_test.go | 2 +- http/multipart.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index f6bbe0c..3240ea6 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -382,7 +382,7 @@ func TestFactoryResetUpstreamAddData(t *testing.T) { _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() - assert.Equal(t, res.StatusCode, http.StatusOK) + assert.Equal(t, res.StatusCode, http.StatusNotFound) rootDocument, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) diff --git a/http/multipart.go b/http/multipart.go index a8676a9..8b9414e 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -339,7 +339,9 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { - if !s.IsDbNotFound(err) { + if s.IsDbNotFound(err) { + return http.StatusNotFound, respHeader, nil, nil + } else { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } } From ec1f5b7bbf4b5a44d1e3c0362f2a1c13cc0cec52 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 10 Sep 2024 16:11:41 -0700 Subject: [PATCH 085/155] return 404 for NONE-REBOOT without subdocs --- http/factory_reset_upstream_test.go | 2 +- http/multipart.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index f6bbe0c..3240ea6 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -382,7 +382,7 @@ func TestFactoryResetUpstreamAddData(t *testing.T) { _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() - assert.Equal(t, res.StatusCode, http.StatusOK) + assert.Equal(t, res.StatusCode, http.StatusNotFound) rootDocument, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) diff --git a/http/multipart.go b/http/multipart.go index a8676a9..8b9414e 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -339,7 +339,9 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l document, err := c.GetDocument(mac, fields) if err != nil { - if !s.IsDbNotFound(err) { + if s.IsDbNotFound(err) { + return http.StatusNotFound, respHeader, nil, nil + } else { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } } From d165d6df1f2912a413cc1f1265b2cb88bb96c9df Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 17 Sep 2024 13:36:18 -0700 Subject: [PATCH 086/155] Reduce unnecessary upstream calls when no meta headers change --- db/service.go | 8 +++++--- http/multipart.go | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/db/service.go b/db/service.go index bce94fa..617ffc8 100644 --- a/db/service.go +++ b/db/service.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( @@ -27,9 +27,9 @@ import ( "strings" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) @@ -183,7 +183,9 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if document == nil { document, err = c.GetDocument(mac, fields) if err != nil { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + if !c.IsDbNotFound(err) { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + } } } for subdocId, subdocument := range document.Items() { diff --git a/http/multipart.go b/http/multipart.go index 8b9414e..d46f665 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,10 +24,10 @@ import ( "strconv" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -204,6 +204,8 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } respStatus = http.StatusOK + } else if len(document.RootVersion()) == 0 { + respStatus = http.StatusNotFound } // mget ==> no upstream From ad941aabdfab98f5bddf431b8549d0c826926c0b Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 18 Sep 2024 13:51:15 -0700 Subject: [PATCH 087/155] add a flag to control the enabling of sarama logger --- config/sample_webconfig.conf | 1 + main.go | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 4b67a55..d460eef 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -33,6 +33,7 @@ webconfig { log { level = "info" file = "/tmp/webconfig.log" + sarama_logger_enabled = false } metrics { diff --git a/main.go b/main.go index d5684ef..fe0cf0a 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package main import ( @@ -26,10 +26,10 @@ import ( "syscall" "github.com/IBM/sarama" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rdkcentral/webconfig/common" wchttp "github.com/rdkcentral/webconfig/http" "github.com/rdkcentral/webconfig/kafka" - "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" _ "go.uber.org/automaxprocs" "golang.org/x/sync/errgroup" @@ -93,7 +93,9 @@ func main() { log.SetLevel(logLevel) // setup sarama logger - sarama.Logger = log.StandardLogger() + if server.GetBoolean("webconfig.log.sarama_logger_enabled") { + sarama.Logger = log.StandardLogger() + } // setup router router := server.GetRouter(false) From 0c4814c089c3c605578051a1a908c9b22c87f7a1 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 20 Sep 2024 11:06:53 -0700 Subject: [PATCH 088/155] change frequently used headers from literals to consts --- common/const_var.go | 7 +- common/document.go | 4 +- common/writer.go | 8 +- http/document_handler.go | 6 +- http/document_handler_test.go | 56 ++++++------- http/factory_reset_upstream_test.go | 18 ++--- http/mqtt_connector_test.go | 4 +- http/multipart.go | 10 +-- http/multipart_test.go | 82 +++++++++---------- http/poke_handler_test.go | 20 ++--- http/refsubdocument_handler.go | 4 +- http/refsubdocument_handler_test.go | 13 +-- http/response.go | 14 ++-- http/rootdocument_handler_test.go | 14 ++-- http/supplementary_handler.go | 6 +- http/supported_groups_handler_test.go | 6 +- http/upstream_test.go | 110 +++++++++++++------------- http/validator.go | 12 +-- http/validator_test.go | 33 ++++---- util/multipart.go | 4 +- util/parser_test.go | 7 +- 21 files changed, 222 insertions(+), 216 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 513807e..8e56f84 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common const ( @@ -64,6 +64,9 @@ var ( ) const ( + HeaderContentType = "Content-Type" + HeaderApplicationJson = "application/json" + HeaderApplicationMsgpack = "application/msgpack" HeaderEtag = "Etag" HeaderIfNoneMatch = "If-None-Match" HeaderFirmwareVersion = "X-System-Firmware-Version" @@ -73,6 +76,7 @@ const ( HeaderProfileVersion = "X-System-Telemetry-Profile-Version" HeaderPartnerID = "X-System-PartnerID" HeaderAccountID = "X-System-AccountID" + HeaderProductClass = "X-System-Product-Class" HeaderUserAgent = "User-Agent" HeaderSchemaVersion = "X-System-Schema-Version" HeaderMetricsAgent = "X-Metrics-Agent" @@ -114,7 +118,6 @@ const ( SkipDbUpdate = "skip-db-update" ) - var ( SupportedPokeDocs = []string{"primary", "telemetry"} SupportedPokeRoutes = []string{"mqtt"} diff --git a/common/document.go b/common/document.go index d99f245..a3122c4 100644 --- a/common/document.go +++ b/common/document.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -194,7 +194,7 @@ func (d *Document) HttpBytes(fields log.Fields) ([]byte, error) { } header := make(http.Header) - header.Set("Content-type", MultipartContentType) + header.Set(HeaderContentType, MultipartContentType) header.Set("Etag", rootVersion) var traceId string diff --git a/common/writer.go b/common/writer.go index e8a16ee..8ea8504 100644 --- a/common/writer.go +++ b/common/writer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -38,9 +38,9 @@ func WriteMultipartBytes(mparts []Multipart) ([]byte, error) { writer.SetBoundary(Boundary) for _, m := range mparts { header := textproto.MIMEHeader{ - "Content-type": {"application/msgpack"}, - "Namespace": {m.Name}, - "Etag": {m.Version}, + HeaderContentType: {HeaderApplicationMsgpack}, + "Namespace": {m.Name}, + "Etag": {m.Version}, } p, err := writer.CreatePart(header) if err != nil { diff --git a/http/document_handler.go b/http/document_handler.go index 3f3e834..a662f7b 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,10 +24,10 @@ import ( "strings" "time" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) // TODO @@ -84,7 +84,7 @@ func (s *WebconfigServer) GetSubDocumentHandler(w http.ResponseWriter, r *http.R return } - w.Header().Set("Content-Type", "application/msgpack") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationMsgpack) writeStateHeaders(w, subdoc) w.WriteHeader(http.StatusOK) w.Write(subdoc.Payload()) diff --git a/http/document_handler_test.go b/http/document_handler_test.go index 7d11287..5d274c1 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -48,7 +48,7 @@ func TestSubDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -58,7 +58,7 @@ func TestSubDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -73,7 +73,7 @@ func TestSubDocumentHandler(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -82,7 +82,7 @@ func TestSubDocumentHandler(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -107,7 +107,7 @@ func TestDeleteDocumentHandler(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -117,7 +117,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -137,7 +137,7 @@ func TestDeleteDocumentHandler(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -146,7 +146,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -173,7 +173,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -182,7 +182,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get to verify req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -214,7 +214,7 @@ func TestPostWithDeviceId(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -226,7 +226,7 @@ func TestPostWithDeviceId(t *testing.T) { for _, mac := range allMacs { url := fmt.Sprintf("/api/v1/device/%v/document/%v", mac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -246,7 +246,7 @@ func TestPostWithDeviceId(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -257,7 +257,7 @@ func TestPostWithDeviceId(t *testing.T) { for _, mac := range allMacs { url := fmt.Sprintf("/api/v1/device/%v/document/%v", mac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -283,7 +283,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) gwrestoreBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header now := time.Now() @@ -298,7 +298,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", gwrestoreUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) @@ -320,7 +320,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header reqHeaderVersion = strconv.Itoa(int(now.Unix())) @@ -339,7 +339,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", remotedebuggerUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -364,7 +364,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -373,7 +373,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -443,7 +443,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) gwrestoreBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare a version header now := time.Now() @@ -458,7 +458,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", gwrestoreUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) @@ -480,7 +480,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header reqHeaderVersion = strconv.Itoa(int(now.Unix())) @@ -499,7 +499,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", remotedebuggerUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -524,7 +524,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -533,7 +533,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -597,7 +597,7 @@ func TestBadHeaderExpiryHandler(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // manage version and expiry headers now := time.Now() diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 3240ea6..5fc3c3a 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -100,7 +100,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -110,7 +110,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -126,7 +126,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -136,7 +136,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -196,7 +196,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -206,7 +206,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -222,7 +222,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -232,7 +232,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/mqtt_connector_test.go b/http/mqtt_connector_test.go index 48a2066..b6cdaae 100644 --- a/http/mqtt_connector_test.go +++ b/http/mqtt_connector_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -30,7 +30,7 @@ import ( func TestPayloadBuilder(t *testing.T) { srcHeader := make(http.Header) srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcData := util.Dict{ "device_id": "mac:044e5a22c9bf", diff --git a/http/multipart.go b/http/multipart.go index d46f665..fb4e8d0 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -166,7 +166,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } } - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return http.StatusOK, respHeader, respBytes, nil } @@ -210,7 +210,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // mget ==> no upstream if userAgent == "mget" { - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return http.StatusOK, respHeader, respBytes, nil } @@ -221,7 +221,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return respStatus, respHeader, respBytes, nil } @@ -230,7 +230,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // upstream handling // ============================= upstreamHeader := rHeader.Clone() - upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader.Set(common.HeaderContentType, common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) @@ -370,7 +370,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l // upstream handling // ============================= upstreamHeader := rHeader.Clone() - upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader.Set(common.HeaderContentType, common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) diff --git a/http/multipart_test.go b/http/multipart_test.go index 2aed59a..4111af3 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -50,7 +50,7 @@ func TestMultipartConfigHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -60,7 +60,7 @@ func TestMultipartConfigHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -79,7 +79,7 @@ func TestMultipartConfigHandler(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -89,7 +89,7 @@ func TestMultipartConfigHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -194,7 +194,7 @@ func TestCpeMiddleware(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -204,7 +204,7 @@ func TestCpeMiddleware(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -251,7 +251,7 @@ func TestVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -261,7 +261,7 @@ func TestVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -277,7 +277,7 @@ func TestVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -287,7 +287,7 @@ func TestVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -361,7 +361,7 @@ func TestVersionFiltering(t *testing.T) { status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType := respHeader.Get("Content-Type") + contentType := respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -395,7 +395,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -405,7 +405,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -421,7 +421,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -431,7 +431,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -570,7 +570,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -580,7 +580,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -596,7 +596,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -606,7 +606,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -623,7 +623,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType := respHeader.Get("Content-Type") + contentType := respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err := util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -687,7 +687,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType = respHeader.Get("Content-Type") + contentType = respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -722,7 +722,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType = respHeader.Get("Content-Type") + contentType = respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -747,7 +747,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -757,7 +757,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -774,7 +774,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -784,7 +784,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -868,7 +868,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -878,7 +878,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -895,7 +895,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -905,7 +905,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -922,7 +922,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(meshBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -932,7 +932,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -949,7 +949,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(mocaBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -959,7 +959,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1127,7 +1127,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -1137,7 +1137,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1151,7 +1151,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1160,7 +1160,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1174,7 +1174,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) privatessidBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1183,7 +1183,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", privatessidUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index ba63157..2e2a862 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -119,7 +119,7 @@ func TestBuildMqttSendDocument(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -128,7 +128,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -151,7 +151,7 @@ func TestBuildMqttSendDocument(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -160,7 +160,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -196,7 +196,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -208,7 +208,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -227,7 +227,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 7 change the subdoc again ==== lanBytes2 := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes2)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -236,7 +236,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -262,7 +262,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/refsubdocument_handler.go b/http/refsubdocument_handler.go index 6413ee4..cd8743d 100644 --- a/http/refsubdocument_handler.go +++ b/http/refsubdocument_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -53,7 +53,7 @@ func (s *WebconfigServer) GetRefSubDocumentHandler(w http.ResponseWriter, r *htt return } - w.Header().Set("Content-Type", "application/msgpack") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationMsgpack) if refsubdoc.Version() != nil { w.Header().Set(common.HeaderRefSubdocumentVersion, *refsubdoc.Version()) } diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go index 49171b0..c0bd996 100644 --- a/http/refsubdocument_handler_test.go +++ b/http/refsubdocument_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,8 +24,9 @@ import ( "net/http" "testing" - "github.com/rdkcentral/webconfig/util" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -39,7 +40,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/reference/%v/document", refId) req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -48,7 +49,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -58,7 +59,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -67,7 +68,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/response.go b/http/response.go index 560a467..7cfeacb 100644 --- a/http/response.go +++ b/http/response.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -53,7 +53,7 @@ func WriteByMarshal(w http.ResponseWriter, status int, o interface{}) { LogError(w, common.NewError(err)) return } - w.Header().Set("Content-type", "application/json") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) w.WriteHeader(status) w.Write(rbytes) } @@ -106,14 +106,14 @@ func WriteOkResponseByTemplate(w http.ResponseWriter, dataStr string, state int, SetAuditValue(w, "response", resp) rbytes = []byte(fmt.Sprintf(OkResponseTemplate, s, stateText, updatedTime)) } - w.Header().Set("Content-type", "application/json") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) w.WriteHeader(http.StatusOK) w.Write(rbytes) } // this is used to return default tr-181 payload while the cpe is not in the db func WriteContentTypeAndResponse(w http.ResponseWriter, rbytes []byte, version string, contentType string) { - w.Header().Set("Content-type", contentType) + w.Header().Set(common.HeaderContentType, contentType) w.Header().Set(common.HeaderEtag, version) w.WriteHeader(http.StatusOK) w.Write(rbytes) @@ -135,7 +135,7 @@ func WriteErrorResponse(w http.ResponseWriter, status int, err error) { } func Error(w http.ResponseWriter, status int, err error) { - // calling WriteHeader() multiple times will cause errors in "content-type" + // calling WriteHeader() multiple times will cause errors in common.HeaderContentType // ==> errors like 'superfluous response.WriteHeader call' in stderr switch status { case http.StatusNoContent, http.StatusNotModified, http.StatusForbidden: @@ -147,14 +147,14 @@ func Error(w http.ResponseWriter, status int, err error) { func WriteResponseBytes(w http.ResponseWriter, rbytes []byte, statusCode int, vargs ...string) { if len(vargs) > 0 { - w.Header().Set("Content-type", vargs[0]) + w.Header().Set(common.HeaderContentType, vargs[0]) } w.WriteHeader(statusCode) w.Write(rbytes) } func WriteFactoryResetResponse(w http.ResponseWriter) { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.WriteHeader(http.StatusOK) w.Write([]byte{}) } diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index d329258..8c4e0f6 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -48,7 +48,7 @@ func TestRootDocumentHandler(t *testing.T) { // ==== step 0 GET /rootdocument and expect 404 ==== rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) req, err := http.NewRequest("GET", rootdocUrl, nil) - req.Header.Set("Content-Type", "application/json") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationJson) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -94,7 +94,7 @@ func TestRootDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -103,7 +103,7 @@ func TestRootDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -121,7 +121,7 @@ func TestRootDocumentHandler(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -130,7 +130,7 @@ func TestRootDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -242,7 +242,7 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { // ==== step 1 GET /rootdocument and expect 404 ==== rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) req, err := http.NewRequest("GET", rootdocUrl, nil) - req.Header.Set("Content-Type", "application/json") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationJson) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 52e5660..82f0743 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -23,9 +23,9 @@ import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -101,7 +101,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r } rootVersion := util.GetRandomRootVersion() - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, rootVersion) // help with unit tests diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 833d656..b0c7f52 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -106,7 +106,7 @@ func TestSupportedGroupsHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -116,7 +116,7 @@ func TestSupportedGroupsHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/upstream_test.go b/http/upstream_test.go index 127dcc1..ff1025e 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -64,7 +64,7 @@ func TestUpstream(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -103,7 +103,7 @@ func TestUpstream(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -142,7 +142,7 @@ func TestUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -151,7 +151,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -165,7 +165,7 @@ func TestUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -174,7 +174,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -188,7 +188,7 @@ func TestUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -197,7 +197,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -244,7 +244,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -272,7 +272,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -318,7 +318,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -360,7 +360,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -371,7 +371,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { respBytes := reqBytes // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, etag) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -411,7 +411,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -420,7 +420,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -434,7 +434,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -443,7 +443,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -457,7 +457,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -466,7 +466,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -507,7 +507,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -535,7 +535,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -578,7 +578,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -617,7 +617,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -656,7 +656,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -665,7 +665,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -679,7 +679,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -688,7 +688,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -702,7 +702,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -711,7 +711,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -758,7 +758,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -787,7 +787,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -840,7 +840,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -889,7 +889,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -916,7 +916,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) @@ -956,7 +956,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -965,7 +965,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -979,7 +979,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -988,7 +988,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1002,7 +1002,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1011,7 +1011,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1058,7 +1058,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1087,7 +1087,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1139,7 +1139,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1182,7 +1182,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -1209,7 +1209,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) @@ -1249,7 +1249,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1258,7 +1258,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -1272,7 +1272,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1281,7 +1281,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1295,7 +1295,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1304,7 +1304,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1347,7 +1347,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1376,7 +1376,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1427,7 +1427,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/validator.go b/http/validator.go index d85921e..bde64c6 100644 --- a/http/validator.go +++ b/http/validator.go @@ -14,16 +14,16 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -56,8 +56,8 @@ func (s *WebconfigServer) Validate(w http.ResponseWriter, r *http.Request, valid // ==== validate content ==== // check content-type - contentType := r.Header.Get("Content-type") - if contentType != "application/msgpack" { + contentType := r.Header.Get(common.HeaderContentType) + if contentType != common.HeaderApplicationMsgpack { // TODO (1) if we should validate this header // (2) if unexpected, return 400 or 415 err := *common.NewHttp400Error("content-type not msgpack") @@ -93,8 +93,8 @@ func (s *WebconfigServer) ValidateRefData(w http.ResponseWriter, r *http.Request // ==== validate content ==== // check content-type - contentType := r.Header.Get("Content-type") - if contentType != "application/msgpack" { + contentType := r.Header.Get(common.HeaderContentType) + if contentType != common.HeaderApplicationMsgpack { // TODO (1) if we should validate this header // (2) if unexpected, return 400 or 415 err := *common.NewHttp400Error("content-type not msgpack") diff --git a/http/validator_test.go b/http/validator_test.go index 8d36dcf..aafc71c 100644 --- a/http/validator_test.go +++ b/http/validator_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -27,6 +27,7 @@ import ( "strings" "testing" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -46,7 +47,7 @@ func TestValidatorDisabled(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -55,7 +56,7 @@ func TestValidatorDisabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -86,7 +87,7 @@ func TestValidatorDisabled(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -95,7 +96,7 @@ func TestValidatorDisabled(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -139,7 +140,7 @@ func TestValidatorEnabled(t *testing.T) { server.SetValidateMacEnabled(true) url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -149,7 +150,7 @@ func TestValidatorEnabled(t *testing.T) { // post without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -159,7 +160,7 @@ func TestValidatorEnabled(t *testing.T) { // get with mac check server.SetValidateMacEnabled(true) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -169,7 +170,7 @@ func TestValidatorEnabled(t *testing.T) { // get without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -211,7 +212,7 @@ func TestValidatorEnabled(t *testing.T) { // delete with mac check server.SetValidateMacEnabled(true) req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -221,7 +222,7 @@ func TestValidatorEnabled(t *testing.T) { // delete without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -230,7 +231,7 @@ func TestValidatorEnabled(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -272,7 +273,7 @@ func TestValidatorWithLowerCase(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -281,7 +282,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -301,7 +302,7 @@ func TestValidatorWithLowerCase(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -310,7 +311,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/util/multipart.go b/util/multipart.go index 70c4683..ee91abf 100644 --- a/util/multipart.go +++ b/util/multipart.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -49,7 +49,7 @@ func ParseMultipartAsList(header http.Header, bbytes []byte) ([]common.Multipart return mparts, nil } - mediaType, params, err := mime.ParseMediaType(header.Get("Content-Type")) + mediaType, params, err := mime.ParseMediaType(header.Get(common.HeaderContentType)) if err != nil { return nil, common.NewError(err) } diff --git a/util/parser_test.go b/util/parser_test.go index 1835dae..23fbc65 100644 --- a/util/parser_test.go +++ b/util/parser_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -22,13 +22,14 @@ import ( "net/http" "testing" + "github.com/rdkcentral/webconfig/common" "gotest.tools/assert" ) func TestParseWebconfigResponseMessage(t *testing.T) { srcHeader := make(http.Header) srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcHeader.Add("Content-length", "120") srcData := Dict{ @@ -55,7 +56,7 @@ func TestParseWebconfigResponseMessageMultipleHeaders(t *testing.T) { srcHeader.Add("X-Color", "color:red") srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") srcHeader.Add("X-Color", "color:orange") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcHeader.Add("X-Color", "color:yellow:green") srcHeader.Add("Content-length", "120") srcHeader.Add("X-Color", "color:blue:indigo:violet") From 42329905ceca761d829e5d6e2b3992505e762991 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 30 Sep 2024 22:53:48 -0700 Subject: [PATCH 089/155] ix a bug that status=0 was logged when the last subdoc was deleted --- http/document_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/http/document_handler.go b/http/document_handler.go index a662f7b..e07890f 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -245,6 +245,7 @@ func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *htt if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) } + WriteOkResponse(w, nil) } else { Error(w, http.StatusInternalServerError, common.NewError(err)) } From b766d8cfa9411ea3a01bbb3e6f633f741b022a64 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 3 Oct 2024 21:43:34 -0700 Subject: [PATCH 090/155] log kafka async producer errors --- common/metrics.go | 19 ++++++++++++- http/webconfig_server.go | 60 ++++++++++++++++++++++++++++++++++++---- main.go | 9 ++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/common/metrics.go b/common/metrics.go index ad34337..18b11f2 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -74,6 +74,7 @@ type AppMetrics struct { indeploymentDecCount *prometheus.CounterVec failureIncCount *prometheus.CounterVec failureDecCount *prometheus.CounterVec + kafkaProducerErrCount *prometheus.CounterVec watchedCpes []string logrusLevel log.Level } @@ -298,6 +299,13 @@ func NewMetrics(conf *configuration.Config, args ...func(string) string) *AppMet }, stateMetricsLabels, ), + kafkaProducerErrCount: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: appName + "_kafka_producer_err_count", + Help: "A counter for the number of errors by kafka producer.", + }, + []string{"topic", "partition"}, + ), watchedCpes: watchedCpes, logrusLevel: logrusLevel, } @@ -326,6 +334,7 @@ func NewMetrics(conf *configuration.Config, args ...func(string) string) *AppMet appMetrics.indeploymentDecCount, appMetrics.failureIncCount, appMetrics.failureDecCount, + appMetrics.kafkaProducerErrCount, ) return appMetrics } @@ -534,6 +543,14 @@ func (m *AppMetrics) CountKafkaEvents(eventName string, status string, partition m.eventCounter.With(labels).Inc() } +func (m *AppMetrics) ObserveKafkaProducerErr(topic string, partition int32) { + labels := prometheus.Labels{ + "topic": topic, + "partition": strconv.Itoa(int(partition)), + } + m.kafkaProducerErrCount.With(labels).Inc() +} + func (m *AppMetrics) GetStateCounter(labels prometheus.Labels) (*StateCounter, error) { // REMINDER if a label is defined with 2 dimensions, then it must be referred // with 2 dimensions. Aggregation happens at prometheus level diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 05d4ea2..c51b811 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "context" "crypto/tls" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -36,15 +37,15 @@ import ( "go.opentelemetry.io/otel/trace" "github.com/IBM/sarama" + "github.com/go-akka/configuration" + "github.com/google/uuid" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/google/uuid" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -276,6 +277,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerTopic = conf.GetString("webconfig.kafka_producer.topic") saramaConfig := sarama.NewConfig() + saramaConfig.Producer.Return.Errors = true kafkaProducer, err = sarama.NewAsyncProducer(brokers, saramaConfig) if err != nil { panic(err) @@ -1070,7 +1072,7 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes } outMessage := &sarama.ProducerMessage{ Topic: s.KafkaProducerTopic(), - Key: sarama.ByteEncoder(mac), + Key: sarama.ByteEncoder(strings.ToLower(mac)), Value: sarama.ByteEncoder(bbytes), } s.Input() <- outMessage @@ -1117,3 +1119,51 @@ func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token str log.WithFields(tfields).Debug(tokenErr) } + +func (s *WebconfigServer) HandleKafkaProducerResults() { + if s.AsyncProducer == nil { + return + } + + for { + select { + case success := <-s.Successes(): + fields := make(log.Fields) + fields["logger"] = "kafkaproducer" + fields["output_topic"] = success.Topic + fields["output_partition"] = success.Partition + fields["output_offset"] = success.Offset + log.WithFields(fields).Debug("sent") + case pErr := <-s.Errors(): + if m := s.Metrics(); m != nil { + m.ObserveKafkaProducerErr(pErr.Msg.Topic, pErr.Msg.Partition) + } + fields := make(log.Fields) + fields["logger"] = "kafkaproducer" + fields["output_topic"] = pErr.Msg.Topic + fields["output_partition"] = pErr.Msg.Partition + kbytes, err := pErr.Msg.Key.Encode() + if err != nil { + log.WithFields(fields).Error(common.NewError(err)) + } else { + fields["output_key"] = string(kbytes) + } + + vbytes, err := pErr.Msg.Value.Encode() + if err != nil { + log.WithFields(fields).Error(common.NewError(err)) + } else { + var itf interface{} + err1 := json.Unmarshal(vbytes, &itf) + if err1 != nil { + log.WithFields(fields).Error(common.NewError(err1)) + fields["output_body_text"] = base64.StdEncoding.EncodeToString(vbytes) + } else { + fields["output_body"] = itf + } + } + + log.WithFields(fields).Error(pErr.Err) + } + } +} diff --git a/main.go b/main.go index fe0cf0a..8a47cb9 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/IBM/sarama" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -105,6 +106,9 @@ func main() { router.Handle("/metrics", promhttp.Handler()) metrics = common.NewMetrics(sc.Config) server.SetMetrics(metrics) + if server.KafkaProducerEnabled() { + go server.HandleKafkaProducerResults() + } handler := metrics.WebMetrics(router) server.Handler = handler } else { @@ -125,6 +129,11 @@ func main() { func() error { <-gCtx.Done() fmt.Printf("HTTP server shutdown NOW !!\n") + if server.KafkaProducerEnabled() { + if err := server.AsyncProducer.Close(); err != nil { + fmt.Fprintf(os.Stderr, "%v AsyncProducer.Close() err=%v\n", time.Now().Format(common.LoggingTimeFormat), err) + } + } return server.Shutdown(context.Background()) }, ) From da5879cb2cdb2cef46c898d0726c08168ce2e784 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 30 Sep 2024 22:53:48 -0700 Subject: [PATCH 091/155] ix a bug that status=0 was logged when the last subdoc was deleted --- http/document_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/http/document_handler.go b/http/document_handler.go index 3f3e834..957bbd0 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -245,6 +245,7 @@ func (s *WebconfigServer) DeleteSubDocumentHandler(w http.ResponseWriter, r *htt if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) } + WriteOkResponse(w, nil) } else { Error(w, http.StatusInternalServerError, common.NewError(err)) } From 217d7cfaaa1893845fc739f9e5df351a271acd76 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 3 Oct 2024 21:43:34 -0700 Subject: [PATCH 092/155] log kafka async producer errors --- common/metrics.go | 19 ++++++++++++- http/webconfig_server.go | 60 ++++++++++++++++++++++++++++++++++++---- main.go | 9 ++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/common/metrics.go b/common/metrics.go index ad34337..18b11f2 100644 --- a/common/metrics.go +++ b/common/metrics.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -74,6 +74,7 @@ type AppMetrics struct { indeploymentDecCount *prometheus.CounterVec failureIncCount *prometheus.CounterVec failureDecCount *prometheus.CounterVec + kafkaProducerErrCount *prometheus.CounterVec watchedCpes []string logrusLevel log.Level } @@ -298,6 +299,13 @@ func NewMetrics(conf *configuration.Config, args ...func(string) string) *AppMet }, stateMetricsLabels, ), + kafkaProducerErrCount: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: appName + "_kafka_producer_err_count", + Help: "A counter for the number of errors by kafka producer.", + }, + []string{"topic", "partition"}, + ), watchedCpes: watchedCpes, logrusLevel: logrusLevel, } @@ -326,6 +334,7 @@ func NewMetrics(conf *configuration.Config, args ...func(string) string) *AppMet appMetrics.indeploymentDecCount, appMetrics.failureIncCount, appMetrics.failureDecCount, + appMetrics.kafkaProducerErrCount, ) return appMetrics } @@ -534,6 +543,14 @@ func (m *AppMetrics) CountKafkaEvents(eventName string, status string, partition m.eventCounter.With(labels).Inc() } +func (m *AppMetrics) ObserveKafkaProducerErr(topic string, partition int32) { + labels := prometheus.Labels{ + "topic": topic, + "partition": strconv.Itoa(int(partition)), + } + m.kafkaProducerErrCount.With(labels).Inc() +} + func (m *AppMetrics) GetStateCounter(labels prometheus.Labels) (*StateCounter, error) { // REMINDER if a label is defined with 2 dimensions, then it must be referred // with 2 dimensions. Aggregation happens at prometheus level diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 05d4ea2..c51b811 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "context" "crypto/tls" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -36,15 +37,15 @@ import ( "go.opentelemetry.io/otel/trace" "github.com/IBM/sarama" + "github.com/go-akka/configuration" + "github.com/google/uuid" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/google/uuid" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -276,6 +277,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerTopic = conf.GetString("webconfig.kafka_producer.topic") saramaConfig := sarama.NewConfig() + saramaConfig.Producer.Return.Errors = true kafkaProducer, err = sarama.NewAsyncProducer(brokers, saramaConfig) if err != nil { panic(err) @@ -1070,7 +1072,7 @@ func (s *WebconfigServer) ForwardSuccessKafkaMessages(messages []common.EventMes } outMessage := &sarama.ProducerMessage{ Topic: s.KafkaProducerTopic(), - Key: sarama.ByteEncoder(mac), + Key: sarama.ByteEncoder(strings.ToLower(mac)), Value: sarama.ByteEncoder(bbytes), } s.Input() <- outMessage @@ -1117,3 +1119,51 @@ func (s *WebconfigServer) LogToken(xw *XResponseWriter, authorization, token str log.WithFields(tfields).Debug(tokenErr) } + +func (s *WebconfigServer) HandleKafkaProducerResults() { + if s.AsyncProducer == nil { + return + } + + for { + select { + case success := <-s.Successes(): + fields := make(log.Fields) + fields["logger"] = "kafkaproducer" + fields["output_topic"] = success.Topic + fields["output_partition"] = success.Partition + fields["output_offset"] = success.Offset + log.WithFields(fields).Debug("sent") + case pErr := <-s.Errors(): + if m := s.Metrics(); m != nil { + m.ObserveKafkaProducerErr(pErr.Msg.Topic, pErr.Msg.Partition) + } + fields := make(log.Fields) + fields["logger"] = "kafkaproducer" + fields["output_topic"] = pErr.Msg.Topic + fields["output_partition"] = pErr.Msg.Partition + kbytes, err := pErr.Msg.Key.Encode() + if err != nil { + log.WithFields(fields).Error(common.NewError(err)) + } else { + fields["output_key"] = string(kbytes) + } + + vbytes, err := pErr.Msg.Value.Encode() + if err != nil { + log.WithFields(fields).Error(common.NewError(err)) + } else { + var itf interface{} + err1 := json.Unmarshal(vbytes, &itf) + if err1 != nil { + log.WithFields(fields).Error(common.NewError(err1)) + fields["output_body_text"] = base64.StdEncoding.EncodeToString(vbytes) + } else { + fields["output_body"] = itf + } + } + + log.WithFields(fields).Error(pErr.Err) + } + } +} diff --git a/main.go b/main.go index fe0cf0a..8a47cb9 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ import ( "os" "os/signal" "syscall" + "time" "github.com/IBM/sarama" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -105,6 +106,9 @@ func main() { router.Handle("/metrics", promhttp.Handler()) metrics = common.NewMetrics(sc.Config) server.SetMetrics(metrics) + if server.KafkaProducerEnabled() { + go server.HandleKafkaProducerResults() + } handler := metrics.WebMetrics(router) server.Handler = handler } else { @@ -125,6 +129,11 @@ func main() { func() error { <-gCtx.Done() fmt.Printf("HTTP server shutdown NOW !!\n") + if server.KafkaProducerEnabled() { + if err := server.AsyncProducer.Close(); err != nil { + fmt.Fprintf(os.Stderr, "%v AsyncProducer.Close() err=%v\n", time.Now().Format(common.LoggingTimeFormat), err) + } + } return server.Shutdown(context.Background()) }, ) From 7ff2a1f26b7a7e8f4ae13443871bae461a2e3b37 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 8 Oct 2024 23:12:08 -0700 Subject: [PATCH 093/155] handle 202 returned by webpa poke --- http/http_client.go | 52 +++++++++++++++++++++++++-------------- http/poke_handler_test.go | 30 ++++++++++++++++++++++ http/response.go | 3 +++ http/webpa_connector.go | 22 ++++++++--------- 4 files changed, 78 insertions(+), 29 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index aad3085..f67598c 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -32,10 +32,10 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) @@ -52,7 +52,7 @@ type ErrorResponse struct { Message string `json:"message"` } -type StatusHandlerFunc func([]byte) ([]byte, http.Header, error, bool) +type StatusHandlerFunc func([]byte) ([]byte, http.Header, bool, error) type HttpClient struct { *http.Client @@ -107,7 +107,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl } } -func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, error, bool) { +func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { fields := common.FilterLogFields(auditFields) // verify a response is received @@ -121,11 +121,11 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] case "DELETE": req, err = http.NewRequest(method, url, nil) default: - return nil, nil, common.NewError(fmt.Errorf("method=%v", method)), false + return nil, nil, false, common.NewError(fmt.Errorf("method=%v", method)) } if err != nil { - return nil, nil, common.NewError(err), true + return nil, nil, true, common.NewError(err) } if header == nil { @@ -209,8 +209,8 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] res, err := c.Client.Do(req) // err should be *url.Error - tdiff := time.Now().Sub(startTime) - duration := tdiff.Nanoseconds() / 1000000 + tdiff := time.Since(startTime) + duration := tdiff.Milliseconds() fields[fmt.Sprintf("%v_duration", loggerName)] = duration delete(fields, bodyKey) @@ -236,25 +236,25 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] Message: ue.Error(), StatusCode: http.StatusGatewayTimeout, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } if errors.Is(innerErr, io.EOF) { rherr := common.RemoteHttpError{ Message: ue.Error(), StatusCode: http.StatusBadGateway, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } if _, ok := innerErr.(*net.OpError); ok { rherr := common.RemoteHttpError{ Message: ue.Error(), StatusCode: http.StatusServiceUnavailable, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } // Unknown err still appear as 500 } - return nil, nil, common.NewError(err), true + return nil, nil, true, common.NewError(err) } if res.Body != nil { defer res.Body.Close() @@ -276,7 +276,7 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] if userAgent != "mget" { log.WithFields(fields).Info(endMessage) } - return nil, nil, common.NewError(err), false + return nil, nil, true, common.NewError(err) } rbody := string(rbytes) @@ -331,11 +331,27 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] switch res.StatusCode { case http.StatusForbidden, http.StatusBadRequest, http.StatusNotFound: - return rbytes, nil, common.NewError(err), false + return rbytes, nil, false, common.NewError(err) + } + return rbytes, nil, true, common.NewError(err) + } else if res.StatusCode > 200 { + var pokeResponse PokeResponse + var message string + if err := json.Unmarshal(rbytes, &pokeResponse); err == nil { + if len(pokeResponse.Parameters) > 0 { + message = pokeResponse.Parameters[0].Message + } + } + if len(message) == 0 { + message = http.StatusText(res.StatusCode) + } + rherr := common.RemoteHttpError{ + Message: message, + StatusCode: res.StatusCode, } - return rbytes, nil, common.NewError(err), true + return rbytes, nil, false, common.NewError(rherr) } - return rbytes, res.Header, nil, false + return rbytes, res.Header, false, nil } func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { @@ -352,7 +368,7 @@ func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Heade if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - respBytes, respHeader, err, cont = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) + respBytes, respHeader, cont, err = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) if !cont { break } diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 2e2a862..5a599ea 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -37,6 +37,7 @@ import ( var ( mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) + mockWebpaPoke202Response = []byte(`{"parameters":[{"message":"Previous request is in progress","name":"Device.X_RDK_WebConfig.ForceSync"}],"statusCode":202}`) ) func TestPokeHandler(t *testing.T) { @@ -335,3 +336,32 @@ func TestPokeHandlerWebpa403(t *testing.T) { assert.NilError(t, err) res.Body.Close() } + +func TestPokeHandlerWebpa202(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write(mockWebpaPoke202Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusAccepted) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} diff --git a/http/response.go b/http/response.go index 7cfeacb..7107bfa 100644 --- a/http/response.go +++ b/http/response.go @@ -140,6 +140,9 @@ func Error(w http.ResponseWriter, status int, err error) { switch status { case http.StatusNoContent, http.StatusNotModified, http.StatusForbidden: w.WriteHeader(status) + case http.StatusAccepted: + SetAuditValue(w, "response", err) + WriteByMarshal(w, status, err) default: WriteErrorResponse(w, status, err) } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index f6aadbe..52378d9 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -25,9 +25,9 @@ import ( "net/http" "time" - "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) @@ -72,7 +72,7 @@ type WebpaConnector struct { apiVersion string } -func syncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { +func syncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { rerr := common.RemoteHttpError{ Message: string(rbytes), StatusCode: 520, @@ -82,16 +82,16 @@ func syncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { if err := json.Unmarshal(rbytes, &pres); err == nil { if len(pres.Parameters) > 0 { if pres.Parameters[0].Message == "Error unsupported namespace" || pres.Parameters[0].Message == "Request rejected" { - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } } } rerr.StatusCode = webpa520NewStatusCode - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } -func asyncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { +func asyncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { rerr := common.RemoteHttpError{ Message: string(rbytes), StatusCode: 520, @@ -101,13 +101,13 @@ func asyncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { if err := json.Unmarshal(rbytes, &pres); err == nil { if len(pres.Parameters) > 0 { if pres.Parameters[0].Message == "Error unsupported namespace" || pres.Parameters[0].Message == "Request rejected" { - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } } } rerr.StatusCode = webpa520NewStatusCode - return rbytes, nil, common.NewError(rerr), true + return rbytes, nil, true, common.NewError(rerr) } func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *WebpaConnector { @@ -231,7 +231,7 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field header.Set(common.HeaderTracestate, outTracestate) method := "PATCH" - _, _, err, cont := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) + _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -269,7 +269,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header ht if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, cont, _ := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { msg := fmt.Sprintf("finished success after 1 retry") if i > 1 { @@ -297,7 +297,7 @@ func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header htt if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - rbytes, _, err, cont = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) + rbytes, _, cont, err = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { // in the case of 524/in-progress, we continue var rherr common.RemoteHttpError From 261b2e5135c2dafb1ba7bf2183578f37a174eb3b Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 10 Oct 2024 00:15:59 -0700 Subject: [PATCH 094/155] update states only when the version in the notification matches the version in db --- db/cassandra/state_update_test.go | 34 +++++++++++------------ db/service.go | 45 +++++++++++++++++++------------ http/upstream_test.go | 20 +++++++------- kafka/consumer.go | 41 ++++++++++++++-------------- 4 files changed, 76 insertions(+), 64 deletions(-) diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 2d9bbb0..e9bdae6 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -53,14 +53,14 @@ func TestStateUpdate1(t *testing.T) { assert.Assert(t, ok) // update to state failure - template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` - bbytes := []byte(fmt.Sprintf(template1, cpeMac)) + template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template1, cpeMac, srcVersion)) var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -68,14 +68,14 @@ func TestStateUpdate1(t *testing.T) { assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") // update to state success - template2 := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:11.959437", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` - bbytes = []byte(fmt.Sprintf(template2, cpeMac)) + template2 := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes = []byte(fmt.Sprintf(template2, cpeMac, srcVersion)) m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) @@ -106,14 +106,14 @@ func TestStateUpdate2(t *testing.T) { assert.Assert(t, ok) // update to state failure - template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` - bbytes := []byte(fmt.Sprintf(template1, cpeMac)) + template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template1, cpeMac, srcVersion)) var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -121,14 +121,14 @@ func TestStateUpdate2(t *testing.T) { assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") // update to state success by http 304 - template2 := `{"device_id": "mac:%v", "http_status_code": 304, "transaction_uuid": "352b85d0-d479-4704-8f9a-bef78b1e7fbf", "version": "2023-05-05 07:42:50.395876"}` - bbytes = []byte(fmt.Sprintf(template2, cpeMac)) + template2 := `{"device_id": "mac:%v", "http_status_code": 304, "transaction_uuid": "352b85d0-d479-4704-8f9a-bef78b1e7fbf", "version": "%v"}` + bbytes = []byte(fmt.Sprintf(template2, cpeMac, srcVersion)) m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, ok) + assert.Assert(t, len(updatedSubdocIds) > 0) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) diff --git a/db/service.go b/db/service.go index 617ffc8..2f4cee7 100644 --- a/db/service.go +++ b/db/service.go @@ -347,14 +347,14 @@ func HashRootVersion(itf interface{}) string { return util.GetMurmur3Hash(buffer.Bytes()) } -func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) (bool, error) { - var updatedBy304 bool +func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) ([]string, error) { + updatedSubdocIds := []string{} // TODO: original config-version-report for ble, NO-OP for now if len(m.Reports) > 0 { - return updatedBy304, nil + return updatedSubdocIds, nil } - updatedTime := int(time.Now().UnixNano() / 1000000) + updatedTime := int(time.Now().UnixMilli()) // set metrics labels metricsAgent := "default" @@ -363,7 +363,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage } labels, err := c.GetRootDocumentLabels(cpeMac) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } labels["client"] = metricsAgent @@ -372,14 +372,14 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage if m.HttpStatusCode != nil { // all non-304 got discarded if *m.HttpStatusCode != http.StatusNotModified { - return updatedBy304, nil + return updatedSubdocIds, nil } // process 304 fields["src_caller"] = common.GetCaller() doc, err := c.GetDocument(cpeMac, fields) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } newState := common.Deployed @@ -388,21 +388,21 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage for groupId, oldSubdoc := range doc.Items() { // fix the bad condition when updated_time is negative if oldSubdoc.NeedsUpdateForHttp304() { - updatedBy304 = true + updatedSubdocIds = append(updatedSubdocIds, groupId) newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) oldState := *oldSubdoc.State() if err := c.SetSubDocument(cpeMac, groupId, newSubdoc, oldState, labels, fields); err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } } } - return updatedBy304, nil + return updatedSubdocIds, nil } // subdoc-report, should have some validation already if m.ApplicationStatus == nil || m.Namespace == nil { - return updatedBy304, common.NewError(fmt.Errorf("ill-formatted event")) + return updatedSubdocIds, common.NewError(fmt.Errorf("ill-formatted event")) } state := common.Failure @@ -415,23 +415,34 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return updatedBy304, nil + return updatedSubdocIds, nil } targetGroupId := *m.Namespace subdoc, err := c.GetSubDocument(cpeMac, *m.Namespace) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } var oldState int if subdoc.State() != nil { oldState = *subdoc.State() + if oldState == common.Deployed { + log.WithFields(fields).Warn("state=1 already") + return updatedSubdocIds, nil + } if oldState < common.Deployed || oldState > common.Failure { err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), } - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) + } + } + + if subdoc.Version() != nil && m.Version != nil { + if *subdoc.Version() != *m.Version { + log.WithFields(fields).Warnf("skip update dbversion=%v, m.version=%v", *subdoc.Version(), *m.Version) + return updatedSubdocIds, nil } } @@ -441,7 +452,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid updated_time(%v) in db", docUpdatedTime), } - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } } @@ -454,9 +465,9 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err = c.SetSubDocument(cpeMac, targetGroupId, newSubdoc, oldState, labels, fields) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } - return updatedBy304, nil + return updatedSubdocIds, nil } func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { diff --git a/http/upstream_test.go b/http/upstream_test.go index ff1025e..7dfb386 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -263,9 +263,9 @@ func TestUpstream(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -526,9 +526,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -777,9 +777,9 @@ func TestUpstreamUpdatedTime(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -1077,9 +1077,9 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -1366,9 +1366,9 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== diff --git a/kafka/consumer.go b/kafka/consumer.go index 85b6fd9..40e8ec1 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package kafka import ( @@ -76,31 +76,31 @@ func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { return nil } -func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, bool, error) { +func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, []string, error) { var m common.EventMessage err := json.Unmarshal(bbytes, &m) if err != nil { - return nil, false, common.NewError(err) + return nil, nil, common.NewError(err) } fields["body"] = m cpeMac, err := m.Validate(true) if err != nil { - return nil, false, common.NewError(err) + return nil, nil, common.NewError(err) } if m.ErrorDetails != nil && *m.ErrorDetails == "max_retry_reached" { - return &m, false, nil + return &m, nil, nil } fields["cpemac"] = cpeMac fields["cpe_mac"] = cpeMac - updatedBy304, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) if err != nil { // NOTE return the *eventMessage - return &m, updatedBy304, common.NewError(err) + return &m, updatedSubdocIds, common.NewError(err) } - return &m, updatedBy304, nil + return &m, updatedSubdocIds, nil } // NOTE we choose to return an EventMessage object just to pass along the metricsAgent @@ -204,7 +204,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram var err error logMessage := "discarded" var m *common.EventMessage - var updatedBy304 bool + var updatedSubdocIds []string eventName, rptHeaderValue := getEventName(message) switch eventName { @@ -214,10 +214,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") - m, updatedBy304, err = c.handleNotification(bbytes, fields) + m, updatedSubdocIds, err = c.handleNotification(bbytes, fields) logMessage = "ok" case "webpa-state": - m, updatedBy304, err = c.handleNotification(message.Value, fields) + m, updatedSubdocIds, err = c.handleNotification(message.Value, fields) logMessage = "ok" } @@ -259,18 +259,19 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram if c.KafkaProducerEnabled() && m != nil { c.ForwardKafkaMessage(message.Key, m, fields) if len(m.Reports) == 0 { - if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { + if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && len(updatedSubdocIds) > 0 { // build a root/success message - namespace := "root" applicationStatus := "success" - em := &common.EventMessage{ - Namespace: &namespace, - ApplicationStatus: &applicationStatus, - DeviceId: m.DeviceId, - TransactionUuid: m.TransactionUuid, - Version: m.Version, + for _, subdocId := range updatedSubdocIds { + em := &common.EventMessage{ + Namespace: &subdocId, + ApplicationStatus: &applicationStatus, + DeviceId: m.DeviceId, + TransactionUuid: m.TransactionUuid, + Version: m.Version, + } + c.ForwardKafkaMessage(message.Key, em, fields) } - c.ForwardKafkaMessage(message.Key, em, fields) } } } From 9108c39455f7867c13709d772a418a8f49defc58 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 11 Oct 2024 21:50:17 -0700 Subject: [PATCH 095/155] change the poke 204 condition to all deployed --- common/document.go | 2 +- http/poke_handler_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/document.go b/common/document.go index a3122c4..2fdb88d 100644 --- a/common/document.go +++ b/common/document.go @@ -120,7 +120,7 @@ func (d *Document) FilterForMqttSend() *Document { for subdocId, subDocument := range d.docmap { if subDocument.State() != nil { state := *subDocument.State() - if state == PendingDownload || state == Failure { + if state > Deployed { newdoc.SetSubDocument(subdocId, &subDocument) } } diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index 5a599ea..9a9d3a2 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -223,7 +223,7 @@ func TestBuildMqttSendDocument(t *testing.T) { fields = make(log.Fields) document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 0) + assert.Equal(t, document.Length(), 2) // ==== step 7 change the subdoc again ==== lanBytes2 := util.RandomBytes(100, 150) @@ -251,7 +251,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 8 check the document length === document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 1) + assert.Equal(t, document.Length(), 2) // ==== step 9 send a document through mqtt ==== req, err = http.NewRequest("POST", mqttUrl, nil) @@ -276,7 +276,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 10 check the length again === document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 0) + assert.Equal(t, document.Length(), 2) } func TestPokeHandlerInvalidMac(t *testing.T) { From a0a7c0106a97a892a446f2b2428583beec3b722d Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 15 Oct 2024 23:31:00 -0700 Subject: [PATCH 096/155] fix a bug that version matched subdocs are not backfilled from upstream --- db/service.go | 4 +- http/upstream_test.go | 221 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 2 deletions(-) diff --git a/db/service.go b/db/service.go index 2f4cee7..20dfbad 100644 --- a/db/service.go +++ b/db/service.go @@ -484,13 +484,13 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old if oldVersion, ok := versionMap[subdocId]; ok { if newSubdoc.Version() != nil { - if oldVersion == *newSubdoc.Version() { + if oldVersion == *newSubdoc.Version() && oldSubdoc != nil { return nil } } } - updatedTime := int(time.Now().UnixNano() / 1000000) + updatedTime := int(time.Now().UnixMilli()) newSubdoc.SetUpdatedTime(&updatedTime) newState := common.InDeployment diff --git a/http/upstream_test.go b/http/upstream_test.go index 7dfb386..f8c1bf3 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -24,6 +24,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strconv" "testing" @@ -1440,3 +1441,223 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) } } + +func TestUpstreamBackfill(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidBytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidBytes, + "lan": lanBytes, + "wan": wanBytes, + } + + var mockedRespBytes []byte + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mockedRespBytes == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + for k := range r.Header { + if k == "Content-Length" { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusOK) + w.Write(mockedRespBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderIfNoneMatch, "NONE-POST") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidBytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidBytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + mockedRespBytes = slices.Clone(rbytes) + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 handle a condition when no subdocs in db ==== + err = server.DeleteDocument(cpeMac) + assert.NilError(t, err) + err = server.DeleteRootDocument(cpeMac) + assert.NilError(t, err) + + // ==== step 9 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotModified) + res.Body.Close() + + // ==== step 10 verify all states, versions and payloads of all subdocs are backfilled ==== + document, err := server.GetDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, document.Length(), 3) + for _, sd := range document.Items() { + assert.Assert(t, sd.GetState() > 0) + assert.Assert(t, len(sd.GetVersion()) > 0) + assert.Assert(t, len(sd.Payload()) > 0) + } +} From 5f819c06bdc48eeb2953186b9a3cc9947e37afe7 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 10 Oct 2024 00:15:59 -0700 Subject: [PATCH 097/155] update states only when the version in the notification matches the version in db --- db/cassandra/state_update_test.go | 34 +++++++++++------------ db/service.go | 45 +++++++++++++++++++------------ http/upstream_test.go | 20 +++++++------- kafka/consumer.go | 41 ++++++++++++++-------------- 4 files changed, 76 insertions(+), 64 deletions(-) diff --git a/db/cassandra/state_update_test.go b/db/cassandra/state_update_test.go index 2d9bbb0..e9bdae6 100644 --- a/db/cassandra/state_update_test.go +++ b/db/cassandra/state_update_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -53,14 +53,14 @@ func TestStateUpdate1(t *testing.T) { assert.Assert(t, ok) // update to state failure - template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` - bbytes := []byte(fmt.Sprintf(template1, cpeMac)) + template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template1, cpeMac, srcVersion)) var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -68,14 +68,14 @@ func TestStateUpdate1(t *testing.T) { assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") // update to state success - template2 := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:11.959437", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` - bbytes = []byte(fmt.Sprintf(template2, cpeMac)) + template2 := `{"application_status": "success", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "0dd08490-7ab6-4080-b153-78ecef4412f6"}` + bbytes = []byte(fmt.Sprintf(template2, cpeMac, srcVersion)) m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) @@ -106,14 +106,14 @@ func TestStateUpdate2(t *testing.T) { assert.Assert(t, ok) // update to state failure - template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "2023-05-05 07:42:22.515324", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` - bbytes := []byte(fmt.Sprintf(template1, cpeMac)) + template1 := `{"application_status": "failure", "error_code": 204, "error_details": "failed_retrying:Error unsupported namespace", "device_id": "mac:%v", "namespace": "privatessid", "version": "%v", "transaction_uuid": "becd74ee-2c17-4abe-aa60-332a218c91aa"}` + bbytes := []byte(fmt.Sprintf(template1, cpeMac, srcVersion)) var m common.EventMessage err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) subdoc, err := tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Failure) @@ -121,14 +121,14 @@ func TestStateUpdate2(t *testing.T) { assert.Equal(t, *subdoc.ErrorDetails(), "failed_retrying:Error unsupported namespace") // update to state success by http 304 - template2 := `{"device_id": "mac:%v", "http_status_code": 304, "transaction_uuid": "352b85d0-d479-4704-8f9a-bef78b1e7fbf", "version": "2023-05-05 07:42:50.395876"}` - bbytes = []byte(fmt.Sprintf(template2, cpeMac)) + template2 := `{"device_id": "mac:%v", "http_status_code": 304, "transaction_uuid": "352b85d0-d479-4704-8f9a-bef78b1e7fbf", "version": "%v"}` + bbytes = []byte(fmt.Sprintf(template2, cpeMac, srcVersion)) m = common.EventMessage{} err = json.Unmarshal(bbytes, &m) assert.NilError(t, err) - ok, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) + updatedSubdocIds, err = db.UpdateDocumentState(tdbclient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, ok) + assert.Assert(t, len(updatedSubdocIds) > 0) subdoc, err = tdbclient.GetSubDocument(cpeMac, groupId) assert.NilError(t, err) assert.Equal(t, *subdoc.State(), common.Deployed) diff --git a/db/service.go b/db/service.go index 617ffc8..2f4cee7 100644 --- a/db/service.go +++ b/db/service.go @@ -347,14 +347,14 @@ func HashRootVersion(itf interface{}) string { return util.GetMurmur3Hash(buffer.Bytes()) } -func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) (bool, error) { - var updatedBy304 bool +func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage, fields log.Fields) ([]string, error) { + updatedSubdocIds := []string{} // TODO: original config-version-report for ble, NO-OP for now if len(m.Reports) > 0 { - return updatedBy304, nil + return updatedSubdocIds, nil } - updatedTime := int(time.Now().UnixNano() / 1000000) + updatedTime := int(time.Now().UnixMilli()) // set metrics labels metricsAgent := "default" @@ -363,7 +363,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage } labels, err := c.GetRootDocumentLabels(cpeMac) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } labels["client"] = metricsAgent @@ -372,14 +372,14 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage if m.HttpStatusCode != nil { // all non-304 got discarded if *m.HttpStatusCode != http.StatusNotModified { - return updatedBy304, nil + return updatedSubdocIds, nil } // process 304 fields["src_caller"] = common.GetCaller() doc, err := c.GetDocument(cpeMac, fields) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } newState := common.Deployed @@ -388,21 +388,21 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage for groupId, oldSubdoc := range doc.Items() { // fix the bad condition when updated_time is negative if oldSubdoc.NeedsUpdateForHttp304() { - updatedBy304 = true + updatedSubdocIds = append(updatedSubdocIds, groupId) newSubdoc := common.NewSubDocument(nil, nil, &newState, &updatedTime, &errorCode, &errorDetails) oldState := *oldSubdoc.State() if err := c.SetSubDocument(cpeMac, groupId, newSubdoc, oldState, labels, fields); err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } } } - return updatedBy304, nil + return updatedSubdocIds, nil } // subdoc-report, should have some validation already if m.ApplicationStatus == nil || m.Namespace == nil { - return updatedBy304, common.NewError(fmt.Errorf("ill-formatted event")) + return updatedSubdocIds, common.NewError(fmt.Errorf("ill-formatted event")) } state := common.Failure @@ -415,23 +415,34 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage errorDetails := "" errorDetailsPtr = &errorDetails } else if *m.ApplicationStatus == "pending" { - return updatedBy304, nil + return updatedSubdocIds, nil } targetGroupId := *m.Namespace subdoc, err := c.GetSubDocument(cpeMac, *m.Namespace) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } var oldState int if subdoc.State() != nil { oldState = *subdoc.State() + if oldState == common.Deployed { + log.WithFields(fields).Warn("state=1 already") + return updatedSubdocIds, nil + } if oldState < common.Deployed || oldState > common.Failure { err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), } - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) + } + } + + if subdoc.Version() != nil && m.Version != nil { + if *subdoc.Version() != *m.Version { + log.WithFields(fields).Warnf("skip update dbversion=%v, m.version=%v", *subdoc.Version(), *m.Version) + return updatedSubdocIds, nil } } @@ -441,7 +452,7 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err := common.Http404Error{ Message: fmt.Sprintf("invalid updated_time(%v) in db", docUpdatedTime), } - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } } @@ -454,9 +465,9 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage err = c.SetSubDocument(cpeMac, targetGroupId, newSubdoc, oldState, labels, fields) if err != nil { - return updatedBy304, common.NewError(err) + return updatedSubdocIds, common.NewError(err) } - return updatedBy304, nil + return updatedSubdocIds, nil } func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, oldSubdoc *common.SubDocument, versionMap map[string]string, fields log.Fields) error { diff --git a/http/upstream_test.go b/http/upstream_test.go index 127dcc1..4041dfe 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -263,9 +263,9 @@ func TestUpstream(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -526,9 +526,9 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -777,9 +777,9 @@ func TestUpstreamUpdatedTime(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -1077,9 +1077,9 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== @@ -1366,9 +1366,9 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { err := json.Unmarshal([]byte(notifBody), &m) assert.NilError(t, err) fields := make(log.Fields) - ok, err = db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(server.DatabaseClient, cpeMac, &m, fields) assert.NilError(t, err) - assert.Assert(t, !ok) + assert.Assert(t, len(updatedSubdocIds) == 0) } // ==== step 9 verify all states deployed ==== diff --git a/kafka/consumer.go b/kafka/consumer.go index 85b6fd9..40e8ec1 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package kafka import ( @@ -76,31 +76,31 @@ func (c *Consumer) Cleanup(sarama.ConsumerGroupSession) error { return nil } -func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, bool, error) { +func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common.EventMessage, []string, error) { var m common.EventMessage err := json.Unmarshal(bbytes, &m) if err != nil { - return nil, false, common.NewError(err) + return nil, nil, common.NewError(err) } fields["body"] = m cpeMac, err := m.Validate(true) if err != nil { - return nil, false, common.NewError(err) + return nil, nil, common.NewError(err) } if m.ErrorDetails != nil && *m.ErrorDetails == "max_retry_reached" { - return &m, false, nil + return &m, nil, nil } fields["cpemac"] = cpeMac fields["cpe_mac"] = cpeMac - updatedBy304, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) + updatedSubdocIds, err := db.UpdateDocumentState(c.DatabaseClient, cpeMac, &m, fields) if err != nil { // NOTE return the *eventMessage - return &m, updatedBy304, common.NewError(err) + return &m, updatedSubdocIds, common.NewError(err) } - return &m, updatedBy304, nil + return &m, updatedSubdocIds, nil } // NOTE we choose to return an EventMessage object just to pass along the metricsAgent @@ -204,7 +204,7 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram var err error logMessage := "discarded" var m *common.EventMessage - var updatedBy304 bool + var updatedSubdocIds []string eventName, rptHeaderValue := getEventName(message) switch eventName { @@ -214,10 +214,10 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram case "mqtt-state": header, bbytes := util.ParseHttp(message.Value) fields["destination"] = header.Get("Destination") - m, updatedBy304, err = c.handleNotification(bbytes, fields) + m, updatedSubdocIds, err = c.handleNotification(bbytes, fields) logMessage = "ok" case "webpa-state": - m, updatedBy304, err = c.handleNotification(message.Value, fields) + m, updatedSubdocIds, err = c.handleNotification(message.Value, fields) logMessage = "ok" } @@ -259,18 +259,19 @@ func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim saram if c.KafkaProducerEnabled() && m != nil { c.ForwardKafkaMessage(message.Key, m, fields) if len(m.Reports) == 0 { - if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && updatedBy304 { + if m.HttpStatusCode != nil && *m.HttpStatusCode == http.StatusNotModified && len(updatedSubdocIds) > 0 { // build a root/success message - namespace := "root" applicationStatus := "success" - em := &common.EventMessage{ - Namespace: &namespace, - ApplicationStatus: &applicationStatus, - DeviceId: m.DeviceId, - TransactionUuid: m.TransactionUuid, - Version: m.Version, + for _, subdocId := range updatedSubdocIds { + em := &common.EventMessage{ + Namespace: &subdocId, + ApplicationStatus: &applicationStatus, + DeviceId: m.DeviceId, + TransactionUuid: m.TransactionUuid, + Version: m.Version, + } + c.ForwardKafkaMessage(message.Key, em, fields) } - c.ForwardKafkaMessage(message.Key, em, fields) } } } From b83d57addf32b958c0d0c4cf6c70ef67e2566fa0 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 11 Oct 2024 21:50:17 -0700 Subject: [PATCH 098/155] change the poke 204 condition to all deployed --- common/document.go | 2 +- http/poke_handler_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/document.go b/common/document.go index d99f245..edd9d57 100644 --- a/common/document.go +++ b/common/document.go @@ -120,7 +120,7 @@ func (d *Document) FilterForMqttSend() *Document { for subdocId, subDocument := range d.docmap { if subDocument.State() != nil { state := *subDocument.State() - if state == PendingDownload || state == Failure { + if state > Deployed { newdoc.SetSubDocument(subdocId, &subDocument) } } diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index ba63157..f354dcd 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -222,7 +222,7 @@ func TestBuildMqttSendDocument(t *testing.T) { fields = make(log.Fields) document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 0) + assert.Equal(t, document.Length(), 2) // ==== step 7 change the subdoc again ==== lanBytes2 := util.RandomBytes(100, 150) @@ -250,7 +250,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 8 check the document length === document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 1) + assert.Equal(t, document.Length(), 2) // ==== step 9 send a document through mqtt ==== req, err = http.NewRequest("POST", mqttUrl, nil) @@ -275,7 +275,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 10 check the length again === document, err = db.BuildMqttSendDocument(server.DatabaseClient, cpeMac, fields) assert.NilError(t, err) - assert.Equal(t, document.Length(), 0) + assert.Equal(t, document.Length(), 2) } func TestPokeHandlerInvalidMac(t *testing.T) { From 872d747b3836f7614fe0d10efab85eaf506c522b Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 15 Oct 2024 23:31:00 -0700 Subject: [PATCH 099/155] fix a bug that version matched subdocs are not backfilled from upstream --- db/service.go | 4 +- http/upstream_test.go | 221 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 2 deletions(-) diff --git a/db/service.go b/db/service.go index 2f4cee7..20dfbad 100644 --- a/db/service.go +++ b/db/service.go @@ -484,13 +484,13 @@ func UpdateSubDocument(c DatabaseClient, cpeMac, subdocId string, newSubdoc, old if oldVersion, ok := versionMap[subdocId]; ok { if newSubdoc.Version() != nil { - if oldVersion == *newSubdoc.Version() { + if oldVersion == *newSubdoc.Version() && oldSubdoc != nil { return nil } } } - updatedTime := int(time.Now().UnixNano() / 1000000) + updatedTime := int(time.Now().UnixMilli()) newSubdoc.SetUpdatedTime(&updatedTime) newState := common.InDeployment diff --git a/http/upstream_test.go b/http/upstream_test.go index 4041dfe..90fcc94 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -24,6 +24,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strconv" "testing" @@ -1440,3 +1441,223 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { assert.Assert(t, res.Header.Get(common.HeaderSubdocumentUpdatedTime) == subdocUpdatedTimeMap[subdocId]) } } + +func TestUpstreamBackfill(t *testing.T) { + server := NewWebconfigServer(sc, true) + server.SetUpstreamEnabled(true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + wanBytes := util.RandomBytes(m, n) + privatessidBytes := util.RandomBytes(m, n) + srcbytesMap := map[string][]byte{ + "privatessid": privatessidBytes, + "lan": lanBytes, + "wan": wanBytes, + } + + var mockedRespBytes []byte + // ==== step 1 set up upstream mock server ==== + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if mockedRespBytes == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + for k := range r.Header { + if k == "Content-Length" { + continue + } + w.Header().Set(k, r.Header.Get(k)) + } + + w.Header().Set(common.HeaderStoreUpstreamResponse, "true") + w.WriteHeader(http.StatusOK) + w.Write(mockedRespBytes) + })) + server.SetUpstreamHost(mockServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockServer.URL, targetUpstreamHost) + + // ==== step 2 GET /config to create root document meta ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + supportedDocs1 := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + firmwareVersion1 := "CGM4331COM_4.11p7s1_PROD_sey" + modelName1 := "CGM4331COM" + partner1 := "comcast" + schemaVersion1 := "33554433-1.3,33554434-1.3" + req.Header.Set(common.HeaderIfNoneMatch, "NONE-POST") + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res := ExecuteRequest(req, router).Result() + assert.NilError(t, err) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotFound) + + // ==== step 3 add group privatessid ==== + subdocId := "privatessid" + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, privatessidBytes) + + // ==== step 4 add group lan ==== + subdocId = "lan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== step 5 add group wan ==== + subdocId = "wan" + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // ==== step 6 GET /config ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.NilError(t, err) + res.Body.Close() + etag := res.Header.Get(common.HeaderEtag) + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 3) + mpart, ok := mparts["privatessid"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, privatessidBytes) + privatessidVersion := mpart.Version + + mpart, ok = mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanVersion := mpart.Version + + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanVersion := mpart.Version + mockedRespBytes = slices.Clone(rbytes) + + // ==== step 7 verify the states are in_deployment ==== + subdocIds := []string{"privatessid", "lan", "wan"} + for _, subdocId := range subdocIds { + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, srcbytesMap[subdocId]) + state, err := strconv.Atoi(res.Header.Get(common.HeaderSubdocumentState)) + assert.NilError(t, err) + assert.Equal(t, state, common.InDeployment) + } + + // ==== step 8 handle a condition when no subdocs in db ==== + err = server.DeleteDocument(cpeMac) + assert.NilError(t, err) + err = server.DeleteRootDocument(cpeMac) + assert.NilError(t, err) + + // ==== step 9 GET /config with schemaVersion change to trigger upstream ==== + configUrl = configUrl + "?group_id=root,privatessid,lan,wan" + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + ifNoneMatch := fmt.Sprintf("%v,%v,%v,%v", etag, privatessidVersion, lanVersion, wanVersion) + req.Header.Set(common.HeaderIfNoneMatch, ifNoneMatch) + req.Header.Set(common.HeaderSupportedDocs, supportedDocs1) + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion1) + req.Header.Set(common.HeaderModelName, modelName1) + req.Header.Set(common.HeaderPartnerID, partner1) + req.Header.Set(common.HeaderSchemaVersion, schemaVersion1) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusNotModified) + res.Body.Close() + + // ==== step 10 verify all states, versions and payloads of all subdocs are backfilled ==== + document, err := server.GetDocument(cpeMac) + assert.NilError(t, err) + assert.Equal(t, document.Length(), 3) + for _, sd := range document.Items() { + assert.Assert(t, sd.GetState() > 0) + assert.Assert(t, len(sd.GetVersion()) > 0) + assert.Assert(t, len(sd.Payload()) > 0) + } +} From 132e15a2c63fb120c7ee4f3643acb603c012c38f Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 20 Sep 2024 11:06:53 -0700 Subject: [PATCH 100/155] change frequently used headers from literals to consts --- common/const_var.go | 7 +- common/document.go | 4 +- common/writer.go | 8 +- http/document_handler.go | 6 +- http/document_handler_test.go | 56 ++++++------- http/factory_reset_upstream_test.go | 18 ++--- http/mqtt_connector_test.go | 4 +- http/multipart.go | 10 +-- http/multipart_test.go | 82 +++++++++---------- http/poke_handler_test.go | 20 ++--- http/refsubdocument_handler.go | 4 +- http/refsubdocument_handler_test.go | 13 +-- http/response.go | 14 ++-- http/rootdocument_handler_test.go | 14 ++-- http/supplementary_handler.go | 6 +- http/supported_groups_handler_test.go | 6 +- http/upstream_test.go | 110 +++++++++++++------------- http/validator.go | 12 +-- http/validator_test.go | 33 ++++---- util/multipart.go | 4 +- util/parser_test.go | 7 +- 21 files changed, 222 insertions(+), 216 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 513807e..8e56f84 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common const ( @@ -64,6 +64,9 @@ var ( ) const ( + HeaderContentType = "Content-Type" + HeaderApplicationJson = "application/json" + HeaderApplicationMsgpack = "application/msgpack" HeaderEtag = "Etag" HeaderIfNoneMatch = "If-None-Match" HeaderFirmwareVersion = "X-System-Firmware-Version" @@ -73,6 +76,7 @@ const ( HeaderProfileVersion = "X-System-Telemetry-Profile-Version" HeaderPartnerID = "X-System-PartnerID" HeaderAccountID = "X-System-AccountID" + HeaderProductClass = "X-System-Product-Class" HeaderUserAgent = "User-Agent" HeaderSchemaVersion = "X-System-Schema-Version" HeaderMetricsAgent = "X-Metrics-Agent" @@ -114,7 +118,6 @@ const ( SkipDbUpdate = "skip-db-update" ) - var ( SupportedPokeDocs = []string{"primary", "telemetry"} SupportedPokeRoutes = []string{"mqtt"} diff --git a/common/document.go b/common/document.go index edd9d57..2fdb88d 100644 --- a/common/document.go +++ b/common/document.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -194,7 +194,7 @@ func (d *Document) HttpBytes(fields log.Fields) ([]byte, error) { } header := make(http.Header) - header.Set("Content-type", MultipartContentType) + header.Set(HeaderContentType, MultipartContentType) header.Set("Etag", rootVersion) var traceId string diff --git a/common/writer.go b/common/writer.go index e8a16ee..8ea8504 100644 --- a/common/writer.go +++ b/common/writer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -38,9 +38,9 @@ func WriteMultipartBytes(mparts []Multipart) ([]byte, error) { writer.SetBoundary(Boundary) for _, m := range mparts { header := textproto.MIMEHeader{ - "Content-type": {"application/msgpack"}, - "Namespace": {m.Name}, - "Etag": {m.Version}, + HeaderContentType: {HeaderApplicationMsgpack}, + "Namespace": {m.Name}, + "Etag": {m.Version}, } p, err := writer.CreatePart(header) if err != nil { diff --git a/http/document_handler.go b/http/document_handler.go index 957bbd0..e07890f 100644 --- a/http/document_handler.go +++ b/http/document_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,10 +24,10 @@ import ( "strings" "time" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) // TODO @@ -84,7 +84,7 @@ func (s *WebconfigServer) GetSubDocumentHandler(w http.ResponseWriter, r *http.R return } - w.Header().Set("Content-Type", "application/msgpack") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationMsgpack) writeStateHeaders(w, subdoc) w.WriteHeader(http.StatusOK) w.Write(subdoc.Payload()) diff --git a/http/document_handler_test.go b/http/document_handler_test.go index 7d11287..5d274c1 100644 --- a/http/document_handler_test.go +++ b/http/document_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -48,7 +48,7 @@ func TestSubDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -58,7 +58,7 @@ func TestSubDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -73,7 +73,7 @@ func TestSubDocumentHandler(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -82,7 +82,7 @@ func TestSubDocumentHandler(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -107,7 +107,7 @@ func TestDeleteDocumentHandler(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -117,7 +117,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -137,7 +137,7 @@ func TestDeleteDocumentHandler(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -146,7 +146,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -173,7 +173,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -182,7 +182,7 @@ func TestDeleteDocumentHandler(t *testing.T) { // get to verify req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -214,7 +214,7 @@ func TestPostWithDeviceId(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -226,7 +226,7 @@ func TestPostWithDeviceId(t *testing.T) { for _, mac := range allMacs { url := fmt.Sprintf("/api/v1/device/%v/document/%v", mac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -246,7 +246,7 @@ func TestPostWithDeviceId(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v?device_id=%v", cpeMac, subdocId, queryParams) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -257,7 +257,7 @@ func TestPostWithDeviceId(t *testing.T) { for _, mac := range allMacs { url := fmt.Sprintf("/api/v1/device/%v/document/%v", mac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -283,7 +283,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) gwrestoreBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header now := time.Now() @@ -298,7 +298,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", gwrestoreUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) @@ -320,7 +320,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header reqHeaderVersion = strconv.Itoa(int(now.Unix())) @@ -339,7 +339,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", remotedebuggerUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -364,7 +364,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -373,7 +373,7 @@ func TestSubDocumentHandlerWithVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -443,7 +443,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { gwrestoreUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) gwrestoreBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", gwrestoreUrl, bytes.NewReader(gwrestoreBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare a version header now := time.Now() @@ -458,7 +458,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", gwrestoreUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() resHeaderVersion := res.Header.Get(common.HeaderSubdocumentVersion) @@ -480,7 +480,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // prepare the version header reqHeaderVersion = strconv.Itoa(int(now.Unix())) @@ -499,7 +499,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", remotedebuggerUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -524,7 +524,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -533,7 +533,7 @@ func TestSubDocumentHandlerWithExpiredVersionHeader(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() @@ -597,7 +597,7 @@ func TestBadHeaderExpiryHandler(t *testing.T) { remotedebuggerUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) remotedebuggerBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", remotedebuggerUrl, bytes.NewReader(remotedebuggerBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) // manage version and expiry headers now := time.Now() diff --git a/http/factory_reset_upstream_test.go b/http/factory_reset_upstream_test.go index 3240ea6..5fc3c3a 100644 --- a/http/factory_reset_upstream_test.go +++ b/http/factory_reset_upstream_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -100,7 +100,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -110,7 +110,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -126,7 +126,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -136,7 +136,7 @@ func TestFactoryResetWithoutUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -196,7 +196,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -206,7 +206,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -222,7 +222,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -232,7 +232,7 @@ func TestFactoryResetWithUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/mqtt_connector_test.go b/http/mqtt_connector_test.go index 48a2066..b6cdaae 100644 --- a/http/mqtt_connector_test.go +++ b/http/mqtt_connector_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -30,7 +30,7 @@ import ( func TestPayloadBuilder(t *testing.T) { srcHeader := make(http.Header) srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcData := util.Dict{ "device_id": "mac:044e5a22c9bf", diff --git a/http/multipart.go b/http/multipart.go index d46f665..fb4e8d0 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -166,7 +166,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin } } - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return http.StatusOK, respHeader, respBytes, nil } @@ -210,7 +210,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // mget ==> no upstream if userAgent == "mget" { - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return http.StatusOK, respHeader, respBytes, nil } @@ -221,7 +221,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - respHeader.Set("Content-type", common.MultipartContentType) + respHeader.Set(common.HeaderContentType, common.MultipartContentType) respHeader.Set(common.HeaderEtag, document.RootVersion()) return respStatus, respHeader, respBytes, nil } @@ -230,7 +230,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // upstream handling // ============================= upstreamHeader := rHeader.Clone() - upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader.Set(common.HeaderContentType, common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) @@ -370,7 +370,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l // upstream handling // ============================= upstreamHeader := rHeader.Clone() - upstreamHeader.Set("Content-type", common.MultipartContentType) + upstreamHeader.Set(common.HeaderContentType, common.MultipartContentType) upstreamHeader.Set(common.HeaderEtag, document.RootVersion()) upstreamHeader.Set(common.HeaderUpstreamNewPartnerId, partnerId) diff --git a/http/multipart_test.go b/http/multipart_test.go index 2aed59a..4111af3 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -50,7 +50,7 @@ func TestMultipartConfigHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -60,7 +60,7 @@ func TestMultipartConfigHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -79,7 +79,7 @@ func TestMultipartConfigHandler(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -89,7 +89,7 @@ func TestMultipartConfigHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -194,7 +194,7 @@ func TestCpeMiddleware(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -204,7 +204,7 @@ func TestCpeMiddleware(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -251,7 +251,7 @@ func TestVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -261,7 +261,7 @@ func TestVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -277,7 +277,7 @@ func TestVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -287,7 +287,7 @@ func TestVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -361,7 +361,7 @@ func TestVersionFiltering(t *testing.T) { status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType := respHeader.Get("Content-Type") + contentType := respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -395,7 +395,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -405,7 +405,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -421,7 +421,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -431,7 +431,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -570,7 +570,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -580,7 +580,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -596,7 +596,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -606,7 +606,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -623,7 +623,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType := respHeader.Get("Content-Type") + contentType := respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err := util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -687,7 +687,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType = respHeader.Get("Content-Type") + contentType = respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -722,7 +722,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) - contentType = respHeader.Get("Content-Type") + contentType = respHeader.Get(common.HeaderContentType) assert.Assert(t, strings.Contains(contentType, "multipart/mixed")) mparts, err = util.ParseMultipart(respHeader, respBytes) assert.NilError(t, err) @@ -747,7 +747,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -757,7 +757,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -774,7 +774,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -784,7 +784,7 @@ func TestMultipartConfigMismatch(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -868,7 +868,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -878,7 +878,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -895,7 +895,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -905,7 +905,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -922,7 +922,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(meshBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -932,7 +932,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -949,7 +949,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(mocaBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -959,7 +959,7 @@ func TestStateCorrectionEnabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1127,7 +1127,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -1137,7 +1137,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1151,7 +1151,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1160,7 +1160,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1174,7 +1174,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { privatessidUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) privatessidBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", privatessidUrl, bytes.NewReader(privatessidBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1183,7 +1183,7 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", privatessidUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index f354dcd..b2e0f91 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -119,7 +119,7 @@ func TestBuildMqttSendDocument(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -128,7 +128,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -151,7 +151,7 @@ func TestBuildMqttSendDocument(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -160,7 +160,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -196,7 +196,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -208,7 +208,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -227,7 +227,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // ==== step 7 change the subdoc again ==== lanBytes2 := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes2)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -236,7 +236,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -262,7 +262,7 @@ func TestBuildMqttSendDocument(t *testing.T) { // get to verify req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/refsubdocument_handler.go b/http/refsubdocument_handler.go index 6413ee4..cd8743d 100644 --- a/http/refsubdocument_handler.go +++ b/http/refsubdocument_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -53,7 +53,7 @@ func (s *WebconfigServer) GetRefSubDocumentHandler(w http.ResponseWriter, r *htt return } - w.Header().Set("Content-Type", "application/msgpack") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationMsgpack) if refsubdoc.Version() != nil { w.Header().Set(common.HeaderRefSubdocumentVersion, *refsubdoc.Version()) } diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go index 49171b0..c0bd996 100644 --- a/http/refsubdocument_handler_test.go +++ b/http/refsubdocument_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -24,8 +24,9 @@ import ( "net/http" "testing" - "github.com/rdkcentral/webconfig/util" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -39,7 +40,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/reference/%v/document", refId) req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -48,7 +49,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -58,7 +59,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -67,7 +68,7 @@ func TestRefSubDocumentHandler(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/response.go b/http/response.go index 560a467..7cfeacb 100644 --- a/http/response.go +++ b/http/response.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -53,7 +53,7 @@ func WriteByMarshal(w http.ResponseWriter, status int, o interface{}) { LogError(w, common.NewError(err)) return } - w.Header().Set("Content-type", "application/json") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) w.WriteHeader(status) w.Write(rbytes) } @@ -106,14 +106,14 @@ func WriteOkResponseByTemplate(w http.ResponseWriter, dataStr string, state int, SetAuditValue(w, "response", resp) rbytes = []byte(fmt.Sprintf(OkResponseTemplate, s, stateText, updatedTime)) } - w.Header().Set("Content-type", "application/json") + w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) w.WriteHeader(http.StatusOK) w.Write(rbytes) } // this is used to return default tr-181 payload while the cpe is not in the db func WriteContentTypeAndResponse(w http.ResponseWriter, rbytes []byte, version string, contentType string) { - w.Header().Set("Content-type", contentType) + w.Header().Set(common.HeaderContentType, contentType) w.Header().Set(common.HeaderEtag, version) w.WriteHeader(http.StatusOK) w.Write(rbytes) @@ -135,7 +135,7 @@ func WriteErrorResponse(w http.ResponseWriter, status int, err error) { } func Error(w http.ResponseWriter, status int, err error) { - // calling WriteHeader() multiple times will cause errors in "content-type" + // calling WriteHeader() multiple times will cause errors in common.HeaderContentType // ==> errors like 'superfluous response.WriteHeader call' in stderr switch status { case http.StatusNoContent, http.StatusNotModified, http.StatusForbidden: @@ -147,14 +147,14 @@ func Error(w http.ResponseWriter, status int, err error) { func WriteResponseBytes(w http.ResponseWriter, rbytes []byte, statusCode int, vargs ...string) { if len(vargs) > 0 { - w.Header().Set("Content-type", vargs[0]) + w.Header().Set(common.HeaderContentType, vargs[0]) } w.WriteHeader(statusCode) w.Write(rbytes) } func WriteFactoryResetResponse(w http.ResponseWriter) { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.WriteHeader(http.StatusOK) w.Write([]byte{}) } diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index d329258..8c4e0f6 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -48,7 +48,7 @@ func TestRootDocumentHandler(t *testing.T) { // ==== step 0 GET /rootdocument and expect 404 ==== rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) req, err := http.NewRequest("GET", rootdocUrl, nil) - req.Header.Set("Content-Type", "application/json") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationJson) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -94,7 +94,7 @@ func TestRootDocumentHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -103,7 +103,7 @@ func TestRootDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -121,7 +121,7 @@ func TestRootDocumentHandler(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -130,7 +130,7 @@ func TestRootDocumentHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -242,7 +242,7 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { // ==== step 1 GET /rootdocument and expect 404 ==== rootdocUrl := fmt.Sprintf("/api/v1/device/%v/rootdocument", cpeMac) req, err := http.NewRequest("GET", rootdocUrl, nil) - req.Header.Set("Content-Type", "application/json") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationJson) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 52e5660..82f0743 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -23,9 +23,9 @@ import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -101,7 +101,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r } rootVersion := util.GetRandomRootVersion() - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, rootVersion) // help with unit tests diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index 833d656..b0c7f52 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -106,7 +106,7 @@ func TestSupportedGroupsHandler(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -116,7 +116,7 @@ func TestSupportedGroupsHandler(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/upstream_test.go b/http/upstream_test.go index 90fcc94..f8c1bf3 100644 --- a/http/upstream_test.go +++ b/http/upstream_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -65,7 +65,7 @@ func TestUpstream(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -104,7 +104,7 @@ func TestUpstream(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -143,7 +143,7 @@ func TestUpstream(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -152,7 +152,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -166,7 +166,7 @@ func TestUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -175,7 +175,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -189,7 +189,7 @@ func TestUpstream(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -198,7 +198,7 @@ func TestUpstream(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -245,7 +245,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -273,7 +273,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -319,7 +319,7 @@ func TestUpstream(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -361,7 +361,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -372,7 +372,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { respBytes := reqBytes // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, etag) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -412,7 +412,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -421,7 +421,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -435,7 +435,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -444,7 +444,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -458,7 +458,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -467,7 +467,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -508,7 +508,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -536,7 +536,7 @@ func TestUpstreamStateChangeFirmwareChange(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -579,7 +579,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -618,7 +618,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusOK) @@ -657,7 +657,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -666,7 +666,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -680,7 +680,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -689,7 +689,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -703,7 +703,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -712,7 +712,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -759,7 +759,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -788,7 +788,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -841,7 +841,7 @@ func TestUpstreamUpdatedTime(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -890,7 +890,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -917,7 +917,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) @@ -957,7 +957,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -966,7 +966,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -980,7 +980,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -989,7 +989,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1003,7 +1003,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1012,7 +1012,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1059,7 +1059,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1088,7 +1088,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1140,7 +1140,7 @@ func TestUpstreamResponseSkipDbUpdate(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1183,7 +1183,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // not necessarily always the case but we return 404 if the input is empty if len(reqBytes) == 0 { - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, "") w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.WriteHeader(http.StatusNotFound) @@ -1210,7 +1210,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { } // generate response - w.Header().Set("Content-type", common.MultipartContentType) + w.Header().Set(common.HeaderContentType, common.MultipartContentType) w.Header().Set(common.HeaderEtag, newRootVersion) w.Header().Set(common.HeaderStoreUpstreamResponse, "true") w.Header().Set(common.HeaderUpstreamResponse, common.SkipDbUpdate) @@ -1250,7 +1250,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(privatessidV13Bytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) @@ -1259,7 +1259,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -1273,7 +1273,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1282,7 +1282,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1296,7 +1296,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // post url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1305,7 +1305,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1348,7 +1348,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1377,7 +1377,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -1428,7 +1428,7 @@ func TestUpstreamResponseSkipDbUpdateNone(t *testing.T) { for _, subdocId := range subdocIds { url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/http/validator.go b/http/validator.go index d85921e..bde64c6 100644 --- a/http/validator.go +++ b/http/validator.go @@ -14,16 +14,16 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" log "github.com/sirupsen/logrus" ) @@ -56,8 +56,8 @@ func (s *WebconfigServer) Validate(w http.ResponseWriter, r *http.Request, valid // ==== validate content ==== // check content-type - contentType := r.Header.Get("Content-type") - if contentType != "application/msgpack" { + contentType := r.Header.Get(common.HeaderContentType) + if contentType != common.HeaderApplicationMsgpack { // TODO (1) if we should validate this header // (2) if unexpected, return 400 or 415 err := *common.NewHttp400Error("content-type not msgpack") @@ -93,8 +93,8 @@ func (s *WebconfigServer) ValidateRefData(w http.ResponseWriter, r *http.Request // ==== validate content ==== // check content-type - contentType := r.Header.Get("Content-type") - if contentType != "application/msgpack" { + contentType := r.Header.Get(common.HeaderContentType) + if contentType != common.HeaderApplicationMsgpack { // TODO (1) if we should validate this header // (2) if unexpected, return 400 or 415 err := *common.NewHttp400Error("content-type not msgpack") diff --git a/http/validator_test.go b/http/validator_test.go index 8d36dcf..aafc71c 100644 --- a/http/validator_test.go +++ b/http/validator_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -27,6 +27,7 @@ import ( "strings" "testing" + "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" "gotest.tools/assert" ) @@ -46,7 +47,7 @@ func TestValidatorDisabled(t *testing.T) { // post url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -55,7 +56,7 @@ func TestValidatorDisabled(t *testing.T) { // get req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -86,7 +87,7 @@ func TestValidatorDisabled(t *testing.T) { // delete req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -95,7 +96,7 @@ func TestValidatorDisabled(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -139,7 +140,7 @@ func TestValidatorEnabled(t *testing.T) { server.SetValidateMacEnabled(true) url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -149,7 +150,7 @@ func TestValidatorEnabled(t *testing.T) { // post without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("POST", url, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -159,7 +160,7 @@ func TestValidatorEnabled(t *testing.T) { // get with mac check server.SetValidateMacEnabled(true) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -169,7 +170,7 @@ func TestValidatorEnabled(t *testing.T) { // get without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -211,7 +212,7 @@ func TestValidatorEnabled(t *testing.T) { // delete with mac check server.SetValidateMacEnabled(true) req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -221,7 +222,7 @@ func TestValidatorEnabled(t *testing.T) { // delete without mac check server.SetValidateMacEnabled(false) req, err = http.NewRequest("DELETE", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -230,7 +231,7 @@ func TestValidatorEnabled(t *testing.T) { // get but expect 404 req, err = http.NewRequest("GET", url, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -272,7 +273,7 @@ func TestValidatorWithLowerCase(t *testing.T) { lanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) lanBytes := util.RandomBytes(100, 150) req, err := http.NewRequest("POST", lanUrl, bytes.NewReader(lanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res := ExecuteRequest(req, router).Result() rbytes, err := io.ReadAll(res.Body) @@ -281,7 +282,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // get req, err = http.NewRequest("GET", lanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -301,7 +302,7 @@ func TestValidatorWithLowerCase(t *testing.T) { wanUrl := fmt.Sprintf("/api/v1/device/%v/document/%v", lowerCpeMac, subdocId) wanBytes := util.RandomBytes(100, 150) req, err = http.NewRequest("POST", wanUrl, bytes.NewReader(wanBytes)) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) @@ -310,7 +311,7 @@ func TestValidatorWithLowerCase(t *testing.T) { // get req, err = http.NewRequest("GET", wanUrl, nil) - req.Header.Set("Content-Type", "application/msgpack") + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) assert.NilError(t, err) res = ExecuteRequest(req, router).Result() rbytes, err = io.ReadAll(res.Body) diff --git a/util/multipart.go b/util/multipart.go index 70c4683..ee91abf 100644 --- a/util/multipart.go +++ b/util/multipart.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -49,7 +49,7 @@ func ParseMultipartAsList(header http.Header, bbytes []byte) ([]common.Multipart return mparts, nil } - mediaType, params, err := mime.ParseMediaType(header.Get("Content-Type")) + mediaType, params, err := mime.ParseMediaType(header.Get(common.HeaderContentType)) if err != nil { return nil, common.NewError(err) } diff --git a/util/parser_test.go b/util/parser_test.go index 1835dae..23fbc65 100644 --- a/util/parser_test.go +++ b/util/parser_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -22,13 +22,14 @@ import ( "net/http" "testing" + "github.com/rdkcentral/webconfig/common" "gotest.tools/assert" ) func TestParseWebconfigResponseMessage(t *testing.T) { srcHeader := make(http.Header) srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcHeader.Add("Content-length", "120") srcData := Dict{ @@ -55,7 +56,7 @@ func TestParseWebconfigResponseMessageMultipleHeaders(t *testing.T) { srcHeader.Add("X-Color", "color:red") srcHeader.Add("Destination", "event:subdoc-report/portmapping/mac:044e5a22c9bf/status") srcHeader.Add("X-Color", "color:orange") - srcHeader.Add("Content-type", "application/json") + srcHeader.Add(common.HeaderContentType, common.HeaderApplicationJson) srcHeader.Add("X-Color", "color:yellow:green") srcHeader.Add("Content-length", "120") srcHeader.Add("X-Color", "color:blue:indigo:violet") From 04a2dc42384a3a78eba25bbf7917119878eba871 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 18 Oct 2024 23:03:22 -0700 Subject: [PATCH 101/155] check nil conditions for kafka async producer errors --- http/webconfig_server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index c51b811..1842ceb 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1135,6 +1135,9 @@ func (s *WebconfigServer) HandleKafkaProducerResults() { fields["output_offset"] = success.Offset log.WithFields(fields).Debug("sent") case pErr := <-s.Errors(): + if pErr == nil || pErr.Msg == nil { + continue + } if m := s.Metrics(); m != nil { m.ObserveKafkaProducerErr(pErr.Msg.Topic, pErr.Msg.Partition) } From 63e902ba926f65baa0e7bcf4a4d70e52a91b6a8a Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 21 Oct 2024 16:22:38 -0700 Subject: [PATCH 102/155] check the nil condition for kafka async producer successes --- http/webconfig_server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 1842ceb..a7e32c8 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1128,6 +1128,9 @@ func (s *WebconfigServer) HandleKafkaProducerResults() { for { select { case success := <-s.Successes(): + if success == nil { + continue + } fields := make(log.Fields) fields["logger"] = "kafkaproducer" fields["output_topic"] = success.Topic From bef8fc72d3cdc102bd70c59f981edd90d4b1afe4 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 18 Oct 2024 23:03:22 -0700 Subject: [PATCH 103/155] check nil conditions for kafka async producer errors --- http/webconfig_server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index c51b811..1842ceb 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1135,6 +1135,9 @@ func (s *WebconfigServer) HandleKafkaProducerResults() { fields["output_offset"] = success.Offset log.WithFields(fields).Debug("sent") case pErr := <-s.Errors(): + if pErr == nil || pErr.Msg == nil { + continue + } if m := s.Metrics(); m != nil { m.ObserveKafkaProducerErr(pErr.Msg.Topic, pErr.Msg.Partition) } From 9bd2c8fa49541e3e3777ecb4d215ca8ae3ce5e17 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 21 Oct 2024 16:22:38 -0700 Subject: [PATCH 104/155] check the nil condition for kafka async producer successes --- http/webconfig_server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 1842ceb..a7e32c8 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -1128,6 +1128,9 @@ func (s *WebconfigServer) HandleKafkaProducerResults() { for { select { case success := <-s.Successes(): + if success == nil { + continue + } fields := make(log.Fields) fields["logger"] = "kafkaproducer" fields["output_topic"] = success.Topic From b2548730585d7a64909f9110692080e062d488d2 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 22 Oct 2024 15:37:40 -0700 Subject: [PATCH 105/155] add bitmap supprot for subdoc webui --- common/bitmap.go | 4 +++- util/firmware_bitmap_test.go | 42 +++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/common/bitmap.go b/common/bitmap.go index dd9c777..89492cc 100644 --- a/common/bitmap.go +++ b/common/bitmap.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common // header X-System-Supported-Docs @@ -38,6 +38,7 @@ var ( {6, 6}, {7, 29}, // connectedbuilding {8, 35}, // xmspeedboost + {9, 40}, // webui }, 2: { {1, 7}, @@ -146,6 +147,7 @@ var ( "wifistatsconfig": 37, "mwoconfigs": 38, "interference": 39, + "webui": 40, } ) diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 8572f68..cf3d5a5 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -547,3 +547,43 @@ func TestParseSupportedDocsHeaderHcm(t *testing.T) { supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } + +func TestParseSupportedDocsHeaderWebui(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + "xmspeedboost", + "webui", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} From 1ae6e582212a594513a1219e1ee0573c84e7089d Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 25 Oct 2024 21:42:48 -0700 Subject: [PATCH 106/155] fix a bug that 500 was returned when the reference doc was not found --- db/service.go | 3 + http/refsubdocument_handler_test.go | 144 ++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/db/service.go b/db/service.go index 20dfbad..7bf6edb 100644 --- a/db/service.go +++ b/db/service.go @@ -649,6 +649,9 @@ func LoadRefSubDocuments(c DatabaseClient, document *common.Document, fields log if refId, ok := GetRefId(payload); ok { refsubdocument, err := c.GetRefSubDocument(refId) if err != nil { + if c.IsDbNotFound(err) { + continue + } return nil, common.NewError(err) } subDocument.SetPayload(refsubdocument.Payload()) diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go index c0bd996..c9a1e2f 100644 --- a/http/refsubdocument_handler_test.go +++ b/http/refsubdocument_handler_test.go @@ -75,3 +75,147 @@ func TestRefSubDocumentHandler(t *testing.T) { assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) } + +func TestSubDocumentWithInvalidRefDoc(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + referenceIndicatorBytes := make([]byte, 4) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 setup refdoc1 and subdoc1 ==== + refId1 := uuid.New().String() + bbytes1 := util.RandomBytes(100, 150) + subdocId1 := "defaultrfc" + + // post + url := fmt.Sprintf("/api/v1/reference/%v/document", refId1) + req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes1)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes1) + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId1) + xbytes := []byte(refId1) + tmpbytes := append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 2 setup refdoc2 and subdoc2 ==== + refId2 := uuid.New().String() + bbytes2 := util.RandomBytes(100, 150) + subdocId2 := "defaulttelemetry" + + // post + url = fmt.Sprintf("/api/v1/reference/%v/document", refId2) + req, err = http.NewRequest("POST", url, bytes.NewReader(bbytes2)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes2) + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId2) + xbytes = []byte(refId2) + tmpbytes = append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 3 GET /config ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok := mpartMap[subdocId1] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes1) + + mpart, ok = mpartMap[subdocId2] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes2) + + // ==== step 4 setup 3rd subdoc but link it to an non-existent refdoc3 ==== + refId3 := uuid.New().String() + subdocId3 := "defaultdcm" + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId3) + xbytes = []byte(refId3) + tmpbytes = append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 5 GET /config returns 2 subdocs and no errors ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok = mpartMap[subdocId1] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes1) + + mpart, ok = mpartMap[subdocId2] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes2) +} From cbca2d60aef9dacbd9a3a3420ca281b00155bee4 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 8 Oct 2024 23:12:08 -0700 Subject: [PATCH 107/155] handle 202 returned by webpa poke --- http/http_client.go | 52 +++++++++++++++++++++++++-------------- http/poke_handler_test.go | 30 ++++++++++++++++++++++ http/response.go | 3 +++ http/webpa_connector.go | 22 ++++++++--------- 4 files changed, 78 insertions(+), 29 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index aad3085..f67598c 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -32,10 +32,10 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) @@ -52,7 +52,7 @@ type ErrorResponse struct { Message string `json:"message"` } -type StatusHandlerFunc func([]byte) ([]byte, http.Header, error, bool) +type StatusHandlerFunc func([]byte) ([]byte, http.Header, bool, error) type HttpClient struct { *http.Client @@ -107,7 +107,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl } } -func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, error, bool) { +func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { fields := common.FilterLogFields(auditFields) // verify a response is received @@ -121,11 +121,11 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] case "DELETE": req, err = http.NewRequest(method, url, nil) default: - return nil, nil, common.NewError(fmt.Errorf("method=%v", method)), false + return nil, nil, false, common.NewError(fmt.Errorf("method=%v", method)) } if err != nil { - return nil, nil, common.NewError(err), true + return nil, nil, true, common.NewError(err) } if header == nil { @@ -209,8 +209,8 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] res, err := c.Client.Do(req) // err should be *url.Error - tdiff := time.Now().Sub(startTime) - duration := tdiff.Nanoseconds() / 1000000 + tdiff := time.Since(startTime) + duration := tdiff.Milliseconds() fields[fmt.Sprintf("%v_duration", loggerName)] = duration delete(fields, bodyKey) @@ -236,25 +236,25 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] Message: ue.Error(), StatusCode: http.StatusGatewayTimeout, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } if errors.Is(innerErr, io.EOF) { rherr := common.RemoteHttpError{ Message: ue.Error(), StatusCode: http.StatusBadGateway, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } if _, ok := innerErr.(*net.OpError); ok { rherr := common.RemoteHttpError{ Message: ue.Error(), StatusCode: http.StatusServiceUnavailable, } - return nil, nil, common.NewError(rherr), true + return nil, nil, true, common.NewError(rherr) } // Unknown err still appear as 500 } - return nil, nil, common.NewError(err), true + return nil, nil, true, common.NewError(err) } if res.Body != nil { defer res.Body.Close() @@ -276,7 +276,7 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] if userAgent != "mget" { log.WithFields(fields).Info(endMessage) } - return nil, nil, common.NewError(err), false + return nil, nil, true, common.NewError(err) } rbody := string(rbytes) @@ -331,11 +331,27 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] switch res.StatusCode { case http.StatusForbidden, http.StatusBadRequest, http.StatusNotFound: - return rbytes, nil, common.NewError(err), false + return rbytes, nil, false, common.NewError(err) + } + return rbytes, nil, true, common.NewError(err) + } else if res.StatusCode > 200 { + var pokeResponse PokeResponse + var message string + if err := json.Unmarshal(rbytes, &pokeResponse); err == nil { + if len(pokeResponse.Parameters) > 0 { + message = pokeResponse.Parameters[0].Message + } + } + if len(message) == 0 { + message = http.StatusText(res.StatusCode) + } + rherr := common.RemoteHttpError{ + Message: message, + StatusCode: res.StatusCode, } - return rbytes, nil, common.NewError(err), true + return rbytes, nil, false, common.NewError(rherr) } - return rbytes, res.Header, nil, false + return rbytes, res.Header, false, nil } func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { @@ -352,7 +368,7 @@ func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Heade if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - respBytes, respHeader, err, cont = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) + respBytes, respHeader, cont, err = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) if !cont { break } diff --git a/http/poke_handler_test.go b/http/poke_handler_test.go index b2e0f91..9a9d3a2 100644 --- a/http/poke_handler_test.go +++ b/http/poke_handler_test.go @@ -37,6 +37,7 @@ import ( var ( mockWebpaPokeResponse = []byte(`{"parameters":[{"name":"Device.X_RDK_WebConfig.ForceSync","message":"Success"}],"statusCode":200}`) mockWebpaPoke403Response = []byte(`{"message": "Invalid partner_id", "statusCode": 403}`) + mockWebpaPoke202Response = []byte(`{"parameters":[{"message":"Previous request is in progress","name":"Device.X_RDK_WebConfig.ForceSync"}],"statusCode":202}`) ) func TestPokeHandler(t *testing.T) { @@ -335,3 +336,32 @@ func TestPokeHandlerWebpa403(t *testing.T) { assert.NilError(t, err) res.Body.Close() } + +func TestPokeHandlerWebpa202(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + cpeMac := util.GenerateRandomCpeMac() + + // webpa mock server + webpaMockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + _, _ = w.Write(mockWebpaPoke202Response) + })) + defer webpaMockServer.Close() + server.SetWebpaHost(webpaMockServer.URL) + targetWebpaHost := server.WebpaHost() + assert.Equal(t, webpaMockServer.URL, targetWebpaHost) + + // ==== post new data ==== + lowerCpeMac := strings.ToLower(cpeMac) + url := fmt.Sprintf("/api/v1/device/%v/poke?cpe_action=true", lowerCpeMac) + req, err := http.NewRequest("POST", url, nil) + req.Header.Set("Authorization", "Bearer foobar") + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + assert.Equal(t, res.StatusCode, http.StatusAccepted) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() +} diff --git a/http/response.go b/http/response.go index 7cfeacb..7107bfa 100644 --- a/http/response.go +++ b/http/response.go @@ -140,6 +140,9 @@ func Error(w http.ResponseWriter, status int, err error) { switch status { case http.StatusNoContent, http.StatusNotModified, http.StatusForbidden: w.WriteHeader(status) + case http.StatusAccepted: + SetAuditValue(w, "response", err) + WriteByMarshal(w, status, err) default: WriteErrorResponse(w, status, err) } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index f6aadbe..52378d9 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -25,9 +25,9 @@ import ( "net/http" "time" - "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) @@ -72,7 +72,7 @@ type WebpaConnector struct { apiVersion string } -func syncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { +func syncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { rerr := common.RemoteHttpError{ Message: string(rbytes), StatusCode: 520, @@ -82,16 +82,16 @@ func syncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { if err := json.Unmarshal(rbytes, &pres); err == nil { if len(pres.Parameters) > 0 { if pres.Parameters[0].Message == "Error unsupported namespace" || pres.Parameters[0].Message == "Request rejected" { - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } } } rerr.StatusCode = webpa520NewStatusCode - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } -func asyncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { +func asyncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { rerr := common.RemoteHttpError{ Message: string(rbytes), StatusCode: 520, @@ -101,13 +101,13 @@ func asyncHandle520(rbytes []byte) ([]byte, http.Header, error, bool) { if err := json.Unmarshal(rbytes, &pres); err == nil { if len(pres.Parameters) > 0 { if pres.Parameters[0].Message == "Error unsupported namespace" || pres.Parameters[0].Message == "Request rejected" { - return rbytes, nil, common.NewError(rerr), false + return rbytes, nil, false, common.NewError(rerr) } } } rerr.StatusCode = webpa520NewStatusCode - return rbytes, nil, common.NewError(rerr), true + return rbytes, nil, true, common.NewError(rerr) } func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *WebpaConnector { @@ -231,7 +231,7 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field header.Set(common.HeaderTracestate, outTracestate) method := "PATCH" - _, _, err, cont := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) + _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -269,7 +269,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header ht if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, _, cont := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, cont, _ := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { msg := fmt.Sprintf("finished success after 1 retry") if i > 1 { @@ -297,7 +297,7 @@ func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header htt if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - rbytes, _, err, cont = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) + rbytes, _, cont, err = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { // in the case of 524/in-progress, we continue var rherr common.RemoteHttpError From bf83753199d6de488e25fb3bf8d6d5136b5cdf2b Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 30 Oct 2024 16:39:25 -0700 Subject: [PATCH 108/155] change the updated_time as usual even if state=1 --- db/service.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/db/service.go b/db/service.go index 7bf6edb..00d4532 100644 --- a/db/service.go +++ b/db/service.go @@ -427,10 +427,6 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage var oldState int if subdoc.State() != nil { oldState = *subdoc.State() - if oldState == common.Deployed { - log.WithFields(fields).Warn("state=1 already") - return updatedSubdocIds, nil - } if oldState < common.Deployed || oldState > common.Failure { err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), From 3845b7d8c9567387fbc10bd20fbf398c870df351 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 31 Oct 2024 23:37:31 -0700 Subject: [PATCH 109/155] return 409 if the root_document is locked --- common/error.go | 5 +++-- common/root_document.go | 11 ++++++++++- common/root_document_test.go | 19 ++++++++++++++++++- db/cassandra/root_document.go | 15 +++++++++++---- db/cassandra/root_document_test.go | 30 +++++++++++++++++++++++++++++- db/cassandra/schema.go | 4 +++- db/service.go | 5 +++++ db/sqlite/schema.go | 4 +++- http/multipart.go | 15 ++++++++++++--- http/multipart_test.go | 23 +++++++++++++++++++++++ 10 files changed, 117 insertions(+), 14 deletions(-) diff --git a/common/error.go b/common/error.go index e733968..18b207f 100644 --- a/common/error.go +++ b/common/error.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -31,7 +31,8 @@ const ( ) var ( - ErrNotOK = fmt.Errorf("!ok") + ErrNotOK = fmt.Errorf("!ok") + ErrRootDocumentLocked = fmt.Errorf("root document is locked") ) type Http400Error struct { diff --git a/common/root_document.go b/common/root_document.go index a3542fd..c662f09 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "encoding/json" "fmt" + "time" ) const ( @@ -36,6 +37,7 @@ type RootDocument struct { SchemaVersion string `json:"schema_version"` Version string `json:"version"` QueryParams string `json:"query_params"` + LockedTill int `json:"locked_till"` } // (bitmap, firmware_version, model_name, partner_id, schema_version, version), nil @@ -69,6 +71,9 @@ func (d *RootDocument) NonEmptyColumnMap() map[string]interface{} { if d.Bitmap > 0 { dict["bitmap"] = d.Bitmap } + if d.LockedTill > 0 { + dict["locked_till"] = int64(d.LockedTill) + } tempDict := map[string]string{ "firmware_version": d.FirmwareVersion, @@ -170,3 +175,7 @@ func (d *RootDocument) Clone() *RootDocument { obj := *d return &obj } + +func (d *RootDocument) Locked() bool { + return d.LockedTill > 0 && int(time.Now().UnixMilli()) < d.LockedTill +} diff --git a/common/root_document_test.go b/common/root_document_test.go index c2ed5e2..50c149f 100644 --- a/common/root_document_test.go +++ b/common/root_document_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "strings" "testing" + "time" "gotest.tools/assert" ) @@ -97,3 +98,19 @@ func TestRootDocumentEquals(t *testing.T) { ok = rootdoc1.Equals(rootdoc3) assert.Assert(t, !ok) } + +func TestRootDocumentLocked(t *testing.T) { + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + assert.Assert(t, !rootdoc.Locked()) + epoch := int(time.Now().UnixMilli()) + rootdoc.LockedTill = epoch + 1000 + assert.Assert(t, rootdoc.Locked()) + time.Sleep(time.Duration(1) * time.Second) + assert.Assert(t, !rootdoc.Locked()) +} diff --git a/db/cassandra/root_document.go b/db/cassandra/root_document.go index 6d3d554..ff9321a 100644 --- a/db/cassandra/root_document.go +++ b/db/cassandra/root_document.go @@ -14,15 +14,16 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "fmt" + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" - "github.com/prometheus/client_golang/prometheus" ) // shared.go: err := c.Query(stmt, cpeMac).MapScan(dict) @@ -32,11 +33,17 @@ func (c *CassandraClient) GetRootDocument(cpeMac string) (*common.RootDocument, defer func() { <-c.concurrentQueries }() var rd common.RootDocument - stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params FROM root_document WHERE cpe_mac=?" - err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams) + var tobj time.Time + stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params,locked_till FROM root_document WHERE cpe_mac=?" + err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams, &tobj) if err != nil { return nil, common.NewError(err) } + if tobj.IsZero() { + rd.LockedTill = 0 + } else { + rd.LockedTill = int(tobj.UnixMilli()) + } return &rd, nil } diff --git a/db/cassandra/root_document_test.go b/db/cassandra/root_document_test.go index 04ceb1f..98ffe38 100644 --- a/db/cassandra/root_document_test.go +++ b/db/cassandra/root_document_test.go @@ -14,11 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" @@ -169,3 +170,30 @@ func TestRootDocumentUpdate(t *testing.T) { assert.NilError(t, err) assert.DeepEqual(t, tgtRootdoc3, rootdoc3) } + +func TestRootDocumentLocked(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + + rootdoc := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + epoch := int(time.Now().UnixMilli()) + rootdoc.LockedTill = epoch + 1000 + + err := tdbclient.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + fetched, err := tdbclient.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.DeepEqual(t, rootdoc, fetched) + assert.Assert(t, fetched.Locked()) + + time.Sleep(time.Duration(1) * time.Second) + + assert.Assert(t, !fetched.Locked()) +} diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index b013804..6953e84 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -39,8 +39,10 @@ var ( cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, + locked_till timestamp, model_name text, partner_id text, + query_params text, route text, schema_version text, version text diff --git a/db/service.go b/db/service.go index 00d4532..a307c42 100644 --- a/db/service.go +++ b/db/service.go @@ -220,6 +220,11 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } } + // eval if the root_document is locked + if cloudRootDocument.Locked() { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + } + switch rootCmpEnum { case common.RootDocumentEquals: // create an empty "document" diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 3e4c132..81c368a 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -45,8 +45,10 @@ var ( cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, + locked_till timestamp, model_name text, partner_id text, + query_params text, route text, schema_version, version text diff --git a/http/multipart.go b/http/multipart.go index fb4e8d0..d6c29b5 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -89,9 +89,12 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) - // REMINDER 404 use standard response - if status == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) + switch status { + case http.StatusNotFound: + Error(w, status, nil) + return + case http.StatusConflict: + w.WriteHeader(status) return } @@ -132,6 +135,12 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if s.KafkaProducerEnabled() && s.StateCorrectionEnabled() && len(messages) > 0 { s.ForwardSuccessKafkaMessages(messages, fields) } + + // root_document locked + if errors.Is(err, common.ErrRootDocumentLocked) { + return http.StatusConflict, respHeader, nil, common.NewError(err) + } + if uconn == nil { if err != nil { if !s.IsDbNotFound(err) { diff --git a/http/multipart_test.go b/http/multipart_test.go index 4111af3..ce73135 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -26,6 +26,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db/cassandra" @@ -168,6 +169,28 @@ func TestMultipartConfigHandler(t *testing.T) { assert.NilError(t, err) parameters = response.Parameters assert.Equal(t, len(parameters), 1) + + // test root_document lock + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + rootdoc.LockedTill = int(time.Now().UnixMilli()) + 1000 + err = server.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + // get document again + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusConflict) + + time.Sleep(time.Duration(1) * time.Second) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) } func TestCpeMiddleware(t *testing.T) { From a01d3bb4cae777a7a4c84e4b163409e9f20d702a Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 30 Oct 2024 16:39:25 -0700 Subject: [PATCH 110/155] change the updated_time as usual even if state=1 --- db/service.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/db/service.go b/db/service.go index 20dfbad..ea754e1 100644 --- a/db/service.go +++ b/db/service.go @@ -427,10 +427,6 @@ func UpdateDocumentState(c DatabaseClient, cpeMac string, m *common.EventMessage var oldState int if subdoc.State() != nil { oldState = *subdoc.State() - if oldState == common.Deployed { - log.WithFields(fields).Warn("state=1 already") - return updatedSubdocIds, nil - } if oldState < common.Deployed || oldState > common.Failure { err := common.Http404Error{ Message: fmt.Sprintf("invalid state(%v) in db", oldState), From c75288f940ae99d531d795dcda0654a6e492f7fa Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 4 Nov 2024 11:35:17 -0800 Subject: [PATCH 111/155] add a control flag for root_document locking --- config/sample_webconfig.conf | 3 ++ db/cassandra/cassandra_client.go | 43 +++++++++++++++++---------- db/cassandra/cassandra_client_test.go | 10 ++++++- db/database_client.go | 7 +++-- db/service.go | 2 +- db/sqlite/sqlite_client.go | 31 ++++++++++++------- db/sqlite/sqlite_client_test.go | 10 ++++++- http/multipart_test.go | 11 ++++++- 8 files changed, 85 insertions(+), 32 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index d460eef..22dacea 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -253,4 +253,7 @@ webconfig { brokers = "localhost:9092" topic = "webconfig_downstream" } + + // this allows the root document locked if needed + lock_root_document_enabled = false } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 7855b9f..3804f22 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -23,12 +23,12 @@ import ( "os" "time" + "github.com/go-akka/configuration" + "github.com/gocql/gocql" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gocql/gocql" ) const ( @@ -48,11 +48,12 @@ type CassandraClient struct { *gocql.ClusterConfig *security.AesCodec *common.AppMetrics - concurrentQueries chan bool - localDc string - blockedSubdocIds []string - encryptedSubdocIds []string - stateCorrectionEnabled bool + concurrentQueries chan bool + localDc string + blockedSubdocIds []string + encryptedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool } /* @@ -161,16 +162,18 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") encryptedSubdocIds := conf.GetStringList("webconfig.encrypted_subdoc_ids") stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") return &CassandraClient{ - Session: session, - ClusterConfig: cluster, - AesCodec: codec, - concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), - localDc: localDc, - blockedSubdocIds: blockedSubdocIds, - encryptedSubdocIds: encryptedSubdocIds, - stateCorrectionEnabled: stateCorrectionEnabled, + Session: session, + ClusterConfig: cluster, + AesCodec: codec, + concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), + localDc: localDc, + blockedSubdocIds: blockedSubdocIds, + encryptedSubdocIds: encryptedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, }, nil } @@ -227,6 +230,14 @@ func (c *CassandraClient) SetStateCorrectionEnabled(enabled bool) { c.stateCorrectionEnabled = enabled } +func (c *CassandraClient) LockRootDocumentEnabled() bool { + return c.lockRootDocumentEnabled +} + +func (c *CassandraClient) SetLockRootDocumentEnabled(enabled bool) { + c.lockRootDocumentEnabled = enabled +} + // TODO we hardcoded for now but it should be changed to be configurable func (c *CassandraClient) IsEncryptedGroup(subdocId string) bool { return util.Contains(c.EncryptedSubdocIds(), subdocId) diff --git a/db/cassandra/cassandra_client_test.go b/db/cassandra/cassandra_client_test.go index 355b701..77c882e 100644 --- a/db/cassandra/cassandra_client_test.go +++ b/db/cassandra/cassandra_client_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -46,6 +46,14 @@ func TestCassandraClient(t *testing.T) { enabled = false tdbclient.SetStateCorrectionEnabled(enabled) assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + + // lock root_document flag + enabled = true + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) + enabled = false + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) } func TestGetConfig(t *testing.T) { diff --git a/db/database_client.go b/db/database_client.go index fa1065f..cc325f1 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -14,12 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( - "github.com/rdkcentral/webconfig/common" "github.com/prometheus/client_golang/prometheus" + "github.com/rdkcentral/webconfig/common" ) type DatabaseClient interface { @@ -68,4 +68,7 @@ type DatabaseClient interface { // enable state correction StateCorrectionEnabled() bool SetStateCorrectionEnabled(bool) + + LockRootDocumentEnabled() bool + SetLockRootDocumentEnabled(bool) } diff --git a/db/service.go b/db/service.go index a307c42..d776cea 100644 --- a/db/service.go +++ b/db/service.go @@ -221,7 +221,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } // eval if the root_document is locked - if cloudRootDocument.Locked() { + if c.LockRootDocumentEnabled() && cloudRootDocument.Locked() { return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) } diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 9ab3b12..56ee4f5 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -22,10 +22,10 @@ import ( "errors" "fmt" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/db" "github.com/go-akka/configuration" _ "github.com/mattn/go-sqlite3" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" ) const ( @@ -43,9 +43,10 @@ type SqliteClient struct { db.BaseClient *sql.DB *common.AppMetrics - concurrentQueries chan bool - blockedSubdocIds []string - stateCorrectionEnabled bool + concurrentQueries chan bool + blockedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool } func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { @@ -60,6 +61,7 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") db, err := sql.Open("sqlite3", dbfile) if err != nil { @@ -67,10 +69,11 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, } return &SqliteClient{ - DB: db, - concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), - blockedSubdocIds: blockedSubdocIds, - stateCorrectionEnabled: stateCorrectionEnabled, + DB: db, + concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), + blockedSubdocIds: blockedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, }, nil } @@ -148,6 +151,14 @@ func (c *SqliteClient) SetStateCorrectionEnabled(enabled bool) { c.stateCorrectionEnabled = enabled } +func (c *SqliteClient) LockRootDocumentEnabled() bool { + return c.lockRootDocumentEnabled +} + +func (c *SqliteClient) SetLockRootDocumentEnabled(enabled bool) { + c.lockRootDocumentEnabled = enabled +} + func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { if tdbclient != nil { return tdbclient, nil diff --git a/db/sqlite/sqlite_client_test.go b/db/sqlite/sqlite_client_test.go index 3a3f74f..f915f8e 100644 --- a/db/sqlite/sqlite_client_test.go +++ b/db/sqlite/sqlite_client_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -40,4 +40,12 @@ func TestSqliteClient(t *testing.T) { enabled = false tdbclient.SetStateCorrectionEnabled(enabled) assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + + // lock root_document flag + enabled = true + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) + enabled = false + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) } diff --git a/http/multipart_test.go b/http/multipart_test.go index ce73135..cc92820 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -177,7 +177,16 @@ func TestMultipartConfigHandler(t *testing.T) { err = server.SetRootDocument(cpeMac, rootdoc) assert.NilError(t, err) - // get document again + // get document again without the feature flag enabled + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get document again with the feature flag enabled + server.SetLockRootDocumentEnabled(true) + res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) assert.NilError(t, err) From f8a86a2ac2de30d79a77ca86aef558b5d54df741 Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 25 Oct 2024 21:42:48 -0700 Subject: [PATCH 112/155] fix a bug that 500 was returned when the reference doc was not found --- db/service.go | 3 + http/refsubdocument_handler_test.go | 144 ++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/db/service.go b/db/service.go index ea754e1..00d4532 100644 --- a/db/service.go +++ b/db/service.go @@ -645,6 +645,9 @@ func LoadRefSubDocuments(c DatabaseClient, document *common.Document, fields log if refId, ok := GetRefId(payload); ok { refsubdocument, err := c.GetRefSubDocument(refId) if err != nil { + if c.IsDbNotFound(err) { + continue + } return nil, common.NewError(err) } subDocument.SetPayload(refsubdocument.Payload()) diff --git a/http/refsubdocument_handler_test.go b/http/refsubdocument_handler_test.go index c0bd996..c9a1e2f 100644 --- a/http/refsubdocument_handler_test.go +++ b/http/refsubdocument_handler_test.go @@ -75,3 +75,147 @@ func TestRefSubDocumentHandler(t *testing.T) { assert.NilError(t, err) assert.Equal(t, res.StatusCode, http.StatusNotFound) } + +func TestSubDocumentWithInvalidRefDoc(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + referenceIndicatorBytes := make([]byte, 4) + + cpeMac := util.GenerateRandomCpeMac() + + // ==== step 1 setup refdoc1 and subdoc1 ==== + refId1 := uuid.New().String() + bbytes1 := util.RandomBytes(100, 150) + subdocId1 := "defaultrfc" + + // post + url := fmt.Sprintf("/api/v1/reference/%v/document", refId1) + req, err := http.NewRequest("POST", url, bytes.NewReader(bbytes1)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes1) + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId1) + xbytes := []byte(refId1) + tmpbytes := append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 2 setup refdoc2 and subdoc2 ==== + refId2 := uuid.New().String() + bbytes2 := util.RandomBytes(100, 150) + subdocId2 := "defaulttelemetry" + + // post + url = fmt.Sprintf("/api/v1/reference/%v/document", refId2) + req, err = http.NewRequest("POST", url, bytes.NewReader(bbytes2)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, bbytes2) + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId2) + xbytes = []byte(refId2) + tmpbytes = append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 3 GET /config ==== + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok := mpartMap[subdocId1] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes1) + + mpart, ok = mpartMap[subdocId2] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes2) + + // ==== step 4 setup 3rd subdoc but link it to an non-existent refdoc3 ==== + refId3 := uuid.New().String() + subdocId3 := "defaultdcm" + + // link + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId3) + xbytes = []byte(refId3) + tmpbytes = append(referenceIndicatorBytes, xbytes...) + req, err = http.NewRequest("POST", url, bytes.NewReader(tmpbytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + assert.Equal(t, res.StatusCode, http.StatusOK) + + // ==== step 5 GET /config returns 2 subdocs and no errors ==== + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mpartMap, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mpartMap), 2) + + // parse the actual data + mpart, ok = mpartMap[subdocId1] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes1) + + mpart, ok = mpartMap[subdocId2] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, bbytes2) +} From 93a333f5745d1e48a51e4aa8a20e3690a27188d8 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 31 Oct 2024 23:37:31 -0700 Subject: [PATCH 113/155] return 409 if the root_document is locked --- common/error.go | 5 +++-- common/root_document.go | 11 ++++++++++- common/root_document_test.go | 19 ++++++++++++++++++- db/cassandra/root_document.go | 15 +++++++++++---- db/cassandra/root_document_test.go | 30 +++++++++++++++++++++++++++++- db/cassandra/schema.go | 4 +++- db/service.go | 5 +++++ db/sqlite/schema.go | 4 +++- http/multipart.go | 15 ++++++++++++--- http/multipart_test.go | 23 +++++++++++++++++++++++ 10 files changed, 117 insertions(+), 14 deletions(-) diff --git a/common/error.go b/common/error.go index e733968..18b207f 100644 --- a/common/error.go +++ b/common/error.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -31,7 +31,8 @@ const ( ) var ( - ErrNotOK = fmt.Errorf("!ok") + ErrNotOK = fmt.Errorf("!ok") + ErrRootDocumentLocked = fmt.Errorf("root document is locked") ) type Http400Error struct { diff --git a/common/root_document.go b/common/root_document.go index a3542fd..c662f09 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "encoding/json" "fmt" + "time" ) const ( @@ -36,6 +37,7 @@ type RootDocument struct { SchemaVersion string `json:"schema_version"` Version string `json:"version"` QueryParams string `json:"query_params"` + LockedTill int `json:"locked_till"` } // (bitmap, firmware_version, model_name, partner_id, schema_version, version), nil @@ -69,6 +71,9 @@ func (d *RootDocument) NonEmptyColumnMap() map[string]interface{} { if d.Bitmap > 0 { dict["bitmap"] = d.Bitmap } + if d.LockedTill > 0 { + dict["locked_till"] = int64(d.LockedTill) + } tempDict := map[string]string{ "firmware_version": d.FirmwareVersion, @@ -170,3 +175,7 @@ func (d *RootDocument) Clone() *RootDocument { obj := *d return &obj } + +func (d *RootDocument) Locked() bool { + return d.LockedTill > 0 && int(time.Now().UnixMilli()) < d.LockedTill +} diff --git a/common/root_document_test.go b/common/root_document_test.go index c2ed5e2..50c149f 100644 --- a/common/root_document_test.go +++ b/common/root_document_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( "strings" "testing" + "time" "gotest.tools/assert" ) @@ -97,3 +98,19 @@ func TestRootDocumentEquals(t *testing.T) { ok = rootdoc1.Equals(rootdoc3) assert.Assert(t, !ok) } + +func TestRootDocumentLocked(t *testing.T) { + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + rootdoc := NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + assert.Assert(t, !rootdoc.Locked()) + epoch := int(time.Now().UnixMilli()) + rootdoc.LockedTill = epoch + 1000 + assert.Assert(t, rootdoc.Locked()) + time.Sleep(time.Duration(1) * time.Second) + assert.Assert(t, !rootdoc.Locked()) +} diff --git a/db/cassandra/root_document.go b/db/cassandra/root_document.go index 6d3d554..ff9321a 100644 --- a/db/cassandra/root_document.go +++ b/db/cassandra/root_document.go @@ -14,15 +14,16 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "fmt" + "time" + "github.com/prometheus/client_golang/prometheus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" - "github.com/prometheus/client_golang/prometheus" ) // shared.go: err := c.Query(stmt, cpeMac).MapScan(dict) @@ -32,11 +33,17 @@ func (c *CassandraClient) GetRootDocument(cpeMac string) (*common.RootDocument, defer func() { <-c.concurrentQueries }() var rd common.RootDocument - stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params FROM root_document WHERE cpe_mac=?" - err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams) + var tobj time.Time + stmt := "SELECT bitmap,firmware_version,model_name,partner_id,schema_version,version,query_params,locked_till FROM root_document WHERE cpe_mac=?" + err := c.Query(stmt, cpeMac).Scan(&rd.Bitmap, &rd.FirmwareVersion, &rd.ModelName, &rd.PartnerId, &rd.SchemaVersion, &rd.Version, &rd.QueryParams, &tobj) if err != nil { return nil, common.NewError(err) } + if tobj.IsZero() { + rd.LockedTill = 0 + } else { + rd.LockedTill = int(tobj.UnixMilli()) + } return &rd, nil } diff --git a/db/cassandra/root_document_test.go b/db/cassandra/root_document_test.go index 04ceb1f..98ffe38 100644 --- a/db/cassandra/root_document_test.go +++ b/db/cassandra/root_document_test.go @@ -14,11 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" @@ -169,3 +170,30 @@ func TestRootDocumentUpdate(t *testing.T) { assert.NilError(t, err) assert.DeepEqual(t, tgtRootdoc3, rootdoc3) } + +func TestRootDocumentLocked(t *testing.T) { + cpeMac := util.GenerateRandomCpeMac() + + bitmap := 123 + version := "foo" + schemaVersion := "33554433-1.3,33554434-1.3" + modelName := "bar" + partnerId := "cox" + firmwareVersion := "TG4482PC2_4.12p7s3_PROD_sey" + + rootdoc := common.NewRootDocument(bitmap, firmwareVersion, modelName, partnerId, schemaVersion, version, "") + epoch := int(time.Now().UnixMilli()) + rootdoc.LockedTill = epoch + 1000 + + err := tdbclient.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + fetched, err := tdbclient.GetRootDocument(cpeMac) + assert.NilError(t, err) + assert.DeepEqual(t, rootdoc, fetched) + assert.Assert(t, fetched.Locked()) + + time.Sleep(time.Duration(1) * time.Second) + + assert.Assert(t, !fetched.Locked()) +} diff --git a/db/cassandra/schema.go b/db/cassandra/schema.go index b013804..6953e84 100644 --- a/db/cassandra/schema.go +++ b/db/cassandra/schema.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -39,8 +39,10 @@ var ( cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, + locked_till timestamp, model_name text, partner_id text, + query_params text, route text, schema_version text, version text diff --git a/db/service.go b/db/service.go index 00d4532..a307c42 100644 --- a/db/service.go +++ b/db/service.go @@ -220,6 +220,11 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } } + // eval if the root_document is locked + if cloudRootDocument.Locked() { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + } + switch rootCmpEnum { case common.RootDocumentEquals: // create an empty "document" diff --git a/db/sqlite/schema.go b/db/sqlite/schema.go index 3e4c132..81c368a 100644 --- a/db/sqlite/schema.go +++ b/db/sqlite/schema.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -45,8 +45,10 @@ var ( cpe_mac text PRIMARY KEY, bitmap bigint, firmware_version text, + locked_till timestamp, model_name text, partner_id text, + query_params text, route text, schema_version, version text diff --git a/http/multipart.go b/http/multipart.go index fb4e8d0..d6c29b5 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -89,9 +89,12 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) - // REMINDER 404 use standard response - if status == http.StatusNotFound { - Error(w, http.StatusNotFound, nil) + switch status { + case http.StatusNotFound: + Error(w, status, nil) + return + case http.StatusConflict: + w.WriteHeader(status) return } @@ -132,6 +135,12 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if s.KafkaProducerEnabled() && s.StateCorrectionEnabled() && len(messages) > 0 { s.ForwardSuccessKafkaMessages(messages, fields) } + + // root_document locked + if errors.Is(err, common.ErrRootDocumentLocked) { + return http.StatusConflict, respHeader, nil, common.NewError(err) + } + if uconn == nil { if err != nil { if !s.IsDbNotFound(err) { diff --git a/http/multipart_test.go b/http/multipart_test.go index 4111af3..ce73135 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -26,6 +26,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db/cassandra" @@ -168,6 +169,28 @@ func TestMultipartConfigHandler(t *testing.T) { assert.NilError(t, err) parameters = response.Parameters assert.Equal(t, len(parameters), 1) + + // test root_document lock + rootdoc, err := server.GetRootDocument(cpeMac) + assert.NilError(t, err) + rootdoc.LockedTill = int(time.Now().UnixMilli()) + 1000 + err = server.SetRootDocument(cpeMac, rootdoc) + assert.NilError(t, err) + + // get document again + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusConflict) + + time.Sleep(time.Duration(1) * time.Second) + + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) } func TestCpeMiddleware(t *testing.T) { From 6297aa056370fdca192459e2f21b6baa00569af8 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 4 Nov 2024 11:35:17 -0800 Subject: [PATCH 114/155] add a control flag for root_document locking --- config/sample_webconfig.conf | 3 ++ db/cassandra/cassandra_client.go | 43 +++++++++++++++++---------- db/cassandra/cassandra_client_test.go | 10 ++++++- db/database_client.go | 7 +++-- db/service.go | 2 +- db/sqlite/sqlite_client.go | 31 ++++++++++++------- db/sqlite/sqlite_client_test.go | 10 ++++++- http/multipart_test.go | 11 ++++++- 8 files changed, 85 insertions(+), 32 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index d460eef..22dacea 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -253,4 +253,7 @@ webconfig { brokers = "localhost:9092" topic = "webconfig_downstream" } + + // this allows the root document locked if needed + lock_root_document_enabled = false } diff --git a/db/cassandra/cassandra_client.go b/db/cassandra/cassandra_client.go index 7855b9f..3804f22 100644 --- a/db/cassandra/cassandra_client.go +++ b/db/cassandra/cassandra_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -23,12 +23,12 @@ import ( "os" "time" + "github.com/go-akka/configuration" + "github.com/gocql/gocql" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/security" "github.com/rdkcentral/webconfig/util" - "github.com/go-akka/configuration" - "github.com/gocql/gocql" ) const ( @@ -48,11 +48,12 @@ type CassandraClient struct { *gocql.ClusterConfig *security.AesCodec *common.AppMetrics - concurrentQueries chan bool - localDc string - blockedSubdocIds []string - encryptedSubdocIds []string - stateCorrectionEnabled bool + concurrentQueries chan bool + localDc string + blockedSubdocIds []string + encryptedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool } /* @@ -161,16 +162,18 @@ func NewCassandraClient(conf *configuration.Config, testOnly bool) (*CassandraCl blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") encryptedSubdocIds := conf.GetStringList("webconfig.encrypted_subdoc_ids") stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") return &CassandraClient{ - Session: session, - ClusterConfig: cluster, - AesCodec: codec, - concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), - localDc: localDc, - blockedSubdocIds: blockedSubdocIds, - encryptedSubdocIds: encryptedSubdocIds, - stateCorrectionEnabled: stateCorrectionEnabled, + Session: session, + ClusterConfig: cluster, + AesCodec: codec, + concurrentQueries: make(chan bool, dbconf.GetInt32("concurrent_queries", 500)), + localDc: localDc, + blockedSubdocIds: blockedSubdocIds, + encryptedSubdocIds: encryptedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, }, nil } @@ -227,6 +230,14 @@ func (c *CassandraClient) SetStateCorrectionEnabled(enabled bool) { c.stateCorrectionEnabled = enabled } +func (c *CassandraClient) LockRootDocumentEnabled() bool { + return c.lockRootDocumentEnabled +} + +func (c *CassandraClient) SetLockRootDocumentEnabled(enabled bool) { + c.lockRootDocumentEnabled = enabled +} + // TODO we hardcoded for now but it should be changed to be configurable func (c *CassandraClient) IsEncryptedGroup(subdocId string) bool { return util.Contains(c.EncryptedSubdocIds(), subdocId) diff --git a/db/cassandra/cassandra_client_test.go b/db/cassandra/cassandra_client_test.go index 355b701..77c882e 100644 --- a/db/cassandra/cassandra_client_test.go +++ b/db/cassandra/cassandra_client_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package cassandra import ( @@ -46,6 +46,14 @@ func TestCassandraClient(t *testing.T) { enabled = false tdbclient.SetStateCorrectionEnabled(enabled) assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + + // lock root_document flag + enabled = true + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) + enabled = false + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) } func TestGetConfig(t *testing.T) { diff --git a/db/database_client.go b/db/database_client.go index fa1065f..cc325f1 100644 --- a/db/database_client.go +++ b/db/database_client.go @@ -14,12 +14,12 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package db import ( - "github.com/rdkcentral/webconfig/common" "github.com/prometheus/client_golang/prometheus" + "github.com/rdkcentral/webconfig/common" ) type DatabaseClient interface { @@ -68,4 +68,7 @@ type DatabaseClient interface { // enable state correction StateCorrectionEnabled() bool SetStateCorrectionEnabled(bool) + + LockRootDocumentEnabled() bool + SetLockRootDocumentEnabled(bool) } diff --git a/db/service.go b/db/service.go index a307c42..d776cea 100644 --- a/db/service.go +++ b/db/service.go @@ -221,7 +221,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } // eval if the root_document is locked - if cloudRootDocument.Locked() { + if c.LockRootDocumentEnabled() && cloudRootDocument.Locked() { return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) } diff --git a/db/sqlite/sqlite_client.go b/db/sqlite/sqlite_client.go index 9ab3b12..56ee4f5 100644 --- a/db/sqlite/sqlite_client.go +++ b/db/sqlite/sqlite_client.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -22,10 +22,10 @@ import ( "errors" "fmt" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/db" "github.com/go-akka/configuration" _ "github.com/mattn/go-sqlite3" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/db" ) const ( @@ -43,9 +43,10 @@ type SqliteClient struct { db.BaseClient *sql.DB *common.AppMetrics - concurrentQueries chan bool - blockedSubdocIds []string - stateCorrectionEnabled bool + concurrentQueries chan bool + blockedSubdocIds []string + stateCorrectionEnabled bool + lockRootDocumentEnabled bool } func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { @@ -60,6 +61,7 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, blockedSubdocIds := conf.GetStringList("webconfig.blocked_subdoc_ids") stateCorrectionEnabled := conf.GetBoolean("webconfig.state_correction_enabled") + lockRootDocumentEnabled := conf.GetBoolean("webconfig.lock_root_document_enabled") db, err := sql.Open("sqlite3", dbfile) if err != nil { @@ -67,10 +69,11 @@ func NewSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, } return &SqliteClient{ - DB: db, - concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), - blockedSubdocIds: blockedSubdocIds, - stateCorrectionEnabled: stateCorrectionEnabled, + DB: db, + concurrentQueries: make(chan bool, conf.GetInt32("webconfig.database.sqlite.concurrent_queries", defaultDbConcurrentQueries)), + blockedSubdocIds: blockedSubdocIds, + stateCorrectionEnabled: stateCorrectionEnabled, + lockRootDocumentEnabled: lockRootDocumentEnabled, }, nil } @@ -148,6 +151,14 @@ func (c *SqliteClient) SetStateCorrectionEnabled(enabled bool) { c.stateCorrectionEnabled = enabled } +func (c *SqliteClient) LockRootDocumentEnabled() bool { + return c.lockRootDocumentEnabled +} + +func (c *SqliteClient) SetLockRootDocumentEnabled(enabled bool) { + c.lockRootDocumentEnabled = enabled +} + func GetTestSqliteClient(conf *configuration.Config, testOnly bool) (*SqliteClient, error) { if tdbclient != nil { return tdbclient, nil diff --git a/db/sqlite/sqlite_client_test.go b/db/sqlite/sqlite_client_test.go index 3a3f74f..f915f8e 100644 --- a/db/sqlite/sqlite_client_test.go +++ b/db/sqlite/sqlite_client_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package sqlite import ( @@ -40,4 +40,12 @@ func TestSqliteClient(t *testing.T) { enabled = false tdbclient.SetStateCorrectionEnabled(enabled) assert.Equal(t, tdbclient.StateCorrectionEnabled(), enabled) + + // lock root_document flag + enabled = true + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) + enabled = false + tdbclient.SetLockRootDocumentEnabled(enabled) + assert.Equal(t, tdbclient.LockRootDocumentEnabled(), enabled) } diff --git a/http/multipart_test.go b/http/multipart_test.go index ce73135..cc92820 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -177,7 +177,16 @@ func TestMultipartConfigHandler(t *testing.T) { err = server.SetRootDocument(cpeMac, rootdoc) assert.NilError(t, err) - // get document again + // get document again without the feature flag enabled + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get document again with the feature flag enabled + server.SetLockRootDocumentEnabled(true) + res = ExecuteRequest(req, router).Result() _, err = io.ReadAll(res.Body) assert.NilError(t, err) From b5b428cf5a2687e1ec6e93695d71872fe0588360 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 22 Oct 2024 15:37:40 -0700 Subject: [PATCH 115/155] add bitmap supprot for subdoc webui --- common/bitmap.go | 4 +++- util/firmware_bitmap_test.go | 42 +++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/common/bitmap.go b/common/bitmap.go index dd9c777..89492cc 100644 --- a/common/bitmap.go +++ b/common/bitmap.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common // header X-System-Supported-Docs @@ -38,6 +38,7 @@ var ( {6, 6}, {7, 29}, // connectedbuilding {8, 35}, // xmspeedboost + {9, 40}, // webui }, 2: { {1, 7}, @@ -146,6 +147,7 @@ var ( "wifistatsconfig": 37, "mwoconfigs": 38, "interference": 39, + "webui": 40, } ) diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index 8572f68..cf3d5a5 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -547,3 +547,43 @@ func TestParseSupportedDocsHeaderHcm(t *testing.T) { supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) } + +func TestParseSupportedDocsHeaderWebui(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + "xmspeedboost", + "webui", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} From 33f6466c89bf8d4fe2c49f05447255c642359821 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 Nov 2024 17:32:40 -0800 Subject: [PATCH 116/155] add an option to read profiles from upstream --- config/sample_webconfig.conf | 3 + http/supplementary_handler.go | 75 +++++++-- http/supplementary_handler_test.go | 241 ++++++++++++++++++++++++++++- http/upstream_connector.go | 37 ++++- http/webconfig_server.go | 12 ++ http/webconfig_server_test.go | 10 +- util/string.go | 43 ++++- util/string_test.go | 17 +- 8 files changed, 419 insertions(+), 19 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 22dacea..9ad5f2f 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -256,4 +256,7 @@ webconfig { // this allows the root document locked if needed lock_root_document_enabled = false + + // get extra profiles from upstream + upstream_profiles_enabled = false } diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 82f0743..20dc8fd 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -29,6 +29,10 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + notFoundProfileText = `{"profiles":[]}` +) + func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r *http.Request) { // ==== data integrity check ==== params := mux.Vars(r) @@ -50,18 +54,17 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r } // append the extra query_params if any + var rootdoc *common.RootDocument var queryParams string - if s.SupplementaryAppendingEnabled() { - rootdoc, err := s.GetRootDocument(mac) + var err error + if s.SupplementaryAppendingEnabled() || s.UpstreamProfilesEnabled() { + rootdoc, err = s.GetRootDocument(mac) if err != nil { if !s.IsDbNotFound(err) { Error(w, http.StatusInternalServerError, common.NewError(err)) return } } - if rootdoc != nil { - queryParams = rootdoc.QueryParams - } } // partner handling @@ -70,21 +73,73 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r partnerId = "" } + if s.SupplementaryAppendingEnabled() && rootdoc != nil { + queryParams = rootdoc.QueryParams + } + urlSuffix := util.GetTelemetryQueryString(r.Header, mac, queryParams, partnerId) fields["is_telemetry"] = true - rbytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + baseProfileBytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + xconfNotFound := false if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - Error(w, rherr.StatusCode, rherr) + if rherr.StatusCode == http.StatusNotFound { + if s.UpstreamProfilesEnabled() { + xconfNotFound = true + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } + if !xconfNotFound { + Error(w, http.StatusInternalServerError, common.NewError(err)) return } - Error(w, http.StatusInternalServerError, common.NewError(err)) - return } - mpart, err := util.TelemetryBytesToMultipart(rbytes) + var profileBytes []byte + if s.UpstreamProfilesEnabled() && rootdoc != nil && len(rootdoc.QueryParams) > 0 { + // Get profiles from the second source + extraProfileBytes, _, err := s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) + if err != nil { + exitNow := true + var rherr common.RemoteHttpError + if errors.As(err, &rherr) { + if rherr.StatusCode == http.StatusNotFound { + exitNow = false + extraProfileBytes = nil + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } + if exitNow { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + } + + if xconfNotFound { + baseProfileBytes = []byte(notFoundProfileText) + } + + // append profiles stored at webconfig + profileBytes, err = util.AppendProfiles(baseProfileBytes, extraProfileBytes) + if err != nil { + Error(w, http.StatusInternalServerError, err) + return + } + } else { + profileBytes = baseProfileBytes + } + + mpart, err := util.TelemetryBytesToMultipart(profileBytes) if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) return diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index 54e2642..1328b9c 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -890,3 +890,242 @@ func TestSupplementaryXconfReadAllErr(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) } + +var ( + mockUpstreamProfileResponse1 = []byte(`[ + { + "name": "subname1", + "value": { + "Parameter": [ + { + "reference": "Device.X_RDK_GatewayManagement.Gateway.1.MacAddress", + "reportEmpty": true, + "type": "dataModel" + } + ] + }, + "versionHash": "977e16c4" + }, + { + "name": "subname2", + "value": { + "Parameter": [ + { + "reference": "Device.X_RDK_GatewayManagement.Gateway.2.MacAddress", + "reportEmpty": true, + "type": "dataModel" + } + ] + }, + "versionHash": "4f207ebd" + } +]`) +) + +func TestSupplementaryUpstreamProfiles(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + cxbytes, err := util.CompactJson([]byte(mockProfileResponse)) + assert.NilError(t, err) + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(cxbytes) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + cubytes, err := util.CompactJson(mockUpstreamProfileResponse1) + assert.NilError(t, err) + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(cubytes) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok := mparts["telemetry"] + assert.Assert(t, ok) + + output := common.TR181Output{} + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes := []byte(output.Parameters[0].Value) + + var itf util.Dict + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok := itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok := profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 1) + profile1Itf := profilesJs[0] + + profile1, ok := profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok := profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok := coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + var srcItf map[string]interface{} + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 4 enable the feature flag but no query_param expect 200 with 1 mpart ==== + server.SetUpstreamProfilesEnabled(true) + defer server.SetUpstreamProfilesEnabled(false) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["telemetry"] + assert.Assert(t, ok) + + output = *new(common.TR181Output) + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes = []byte(output.Parameters[0].Value) + + itf = make(util.Dict) + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok = itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok = profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 1) + profile1Itf = profilesJs[0] + profile1, ok = profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok = profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok = coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + srcItf = make(map[string]interface{}) + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 5 set query_param in the root_document expect 200 with 1 mpart ==== + rdoc := new(common.RootDocument) + rdoc.QueryParams = "key1=val1" + rdoc.ModelName = modelName + rdoc.PartnerId = partnerID + rdoc.FirmwareVersion = firmwareVersion + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + t.Logf("%s\n", rbytes) + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["telemetry"] + assert.Assert(t, ok) + + output = *new(common.TR181Output) + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes = []byte(output.Parameters[0].Value) + + itf = make(util.Dict) + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok = itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok = profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 3) + profile1Itf = profilesJs[0] + profile1, ok = profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + profile2Itf := profilesJs[1] + profile2, ok := profile2Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile2["name"].(string), "subname1") + + profile3Itf := profilesJs[2] + profile3, ok := profile3Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile3["name"].(string), "subname2") +} diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 165aac6..d006bc5 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -22,15 +22,16 @@ import ( "fmt" "net/http" + "github.com/go-akka/configuration" "github.com/rdkcentral/webconfig/common" owcommon "github.com/rdkcentral/webconfig/common" - "github.com/go-akka/configuration" log "github.com/sirupsen/logrus" ) const ( upstreamHostDefault = "http://localhost:1234" - upstreamUrlTemplateDefault = "/api/v1/device/%v/upstream" + defaultUpstreamUrlTemplate = "/api/v1/device/%v/upstream" + defaultProfileUrlTemplate = "/api/v1/device/%v/profile?%v" ) type UpstreamConnector struct { @@ -38,6 +39,7 @@ type UpstreamConnector struct { host string serviceName string upstreamUrlTemplate string + profileUrlTemplate string } func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *UpstreamConnector { @@ -45,13 +47,16 @@ func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *Up confKey := fmt.Sprintf("webconfig.%v.host", serviceName) host := conf.GetString(confKey, upstreamHostDefault) confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) - upstreamUrlTemplate := conf.GetString(confKey, upstreamUrlTemplateDefault) + upstreamUrlTemplate := conf.GetString(confKey, defaultUpstreamUrlTemplate) + confKey = fmt.Sprintf("webconfig.%v.profile_url_template", serviceName) + profileUrlTemplate := conf.GetString(confKey, defaultProfileUrlTemplate) return &UpstreamConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, upstreamUrlTemplate: upstreamUrlTemplate, + profileUrlTemplate: profileUrlTemplate, } } @@ -90,3 +95,27 @@ func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes } return rbytes, header, nil } + +func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { + url := c.UpstreamHost() + fmt.Sprintf(c.profileUrlTemplate, mac, queryParams) + + if itf, ok := fields["audit_id"]; ok { + auditId := itf.(string) + if len(auditId) > 0 { + header.Set(common.HeaderAuditid, auditId) + } + } + + if itf, ok := fields["app_name"]; ok { + appName := itf.(string) + if len(appName) > 0 { + header.Set(common.HeaderSourceAppName, appName) + } + } + + rbytes, header, err := c.DoWithRetries("GET", url, header, nil, fields, c.ServiceName()) + if err != nil { + return rbytes, header, owcommon.NewError(err) + } + return rbytes, header, nil +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index a7e32c8..63d7d7c 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -118,6 +118,7 @@ type WebconfigServer struct { webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string + upstreamProfilesEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -284,6 +285,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } } + upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), @@ -318,6 +321,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, + upstreamProfilesEnabled: upstreamProfilesEnabled, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -634,6 +638,14 @@ func (s *WebconfigServer) SetKafkaProducerTopic(x string) { s.kafkaProducerTopic = x } +func (s *WebconfigServer) UpstreamProfilesEnabled() bool { + return s.upstreamProfilesEnabled +} + +func (s *WebconfigServer) SetUpstreamProfilesEnabled(enabled bool) { + s.upstreamProfilesEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 1699106..6f92f75 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -97,4 +97,12 @@ func TestWebconfigServerSetterGetter(t *testing.T) { validPartners = []string{"name3", "name4", "name5"} server.SetValidPartners(validPartners) assert.DeepEqual(t, server.ValidPartners(), validPartners) + + // get profiles from upstream + enabled = true + server.SetUpstreamProfilesEnabled(enabled) + assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) + enabled = false + server.SetUpstreamProfilesEnabled(enabled) + assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) } diff --git a/util/string.go b/util/string.go index 439b0bb..05eab77 100644 --- a/util/string.go +++ b/util/string.go @@ -14,18 +14,26 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( + "encoding/json" "fmt" "net/http" "net/url" "strconv" "strings" - "github.com/rdkcentral/webconfig/common" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" +) + +const ( + Comma = ',' + RightSquareBracket = ']' + RightCurlBracket = '}' + baseProfileStr = `{"profiles":[` ) var ( @@ -184,3 +192,34 @@ func IsValidUTF8(bbytes []byte) bool { str2 := strings.ToValidUTF8(str1, "#") return str1 == str2 } + +func AppendProfiles(pbytes, sbytes []byte) ([]byte, error) { + var builder strings.Builder + if len(pbytes) < 3 { + builder.WriteString(baseProfileStr) + } else { + s := strings.TrimSpace(string(pbytes)) + builder.WriteString(s[:len(s)-2]) + } + if len(sbytes) > 2 { + if len(pbytes) > 20 { + builder.WriteRune(',') + } + builder.Write(sbytes[1 : len(sbytes)-1]) + } + builder.WriteRune(RightSquareBracket) + builder.WriteRune(RightCurlBracket) + return []byte(builder.String()), nil +} + +func CompactJson(sbytes []byte) ([]byte, error) { + var itf interface{} + if err := json.Unmarshal(sbytes, &itf); err != nil { + return nil, common.NewError(err) + } + jsbytes, err := json.Marshal(itf) + if err != nil { + return nil, common.NewError(err) + } + return jsbytes, nil +} diff --git a/util/string_test.go b/util/string_test.go index cf6a894..73640bc 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -14,10 +14,11 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( + "encoding/json" "net/http" "net/url" "testing" @@ -159,3 +160,17 @@ func TestTelemetryQueryWithWanMac(t *testing.T) { expected := "env=PROD&partnerId=comcast&version=2.0&model=TG1682G&accountId=1234567890&firmwareVersion=TG1682_3.14p9s6_PROD_sey&estbMacAddress=567890ABCDEF" assert.Equal(t, qstr, expected) } + +func TestAppendProfiles(t *testing.T) { + mockedBaseProfilesResponse := `{"profiles":[{"name":"XfinityWIFI_SYNC","value":{"Description":"XfinityWIFI_SYNC to capture XWIFI info every 12 hours","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"},{"Name":"reportVersion","Reference":"Profile.Version"}],"URL":"https://stbrtl.stb.r53.xcal.tv"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"reference":"Profile.Name","type":"dataModel"},{"reference":"Profile.Version","type":"dataModel"},{"name":"Profile","reference":"Device.DeviceInfo.X_RDK_RDKProfileName","type":"dataModel"},{"name":"Time","reference":"Device.Time.X_RDK_CurrentUTCTime","type":"dataModel"},{"name":"mac","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_MAC","type":"dataModel"},{"name":"CMMAC_split","reference":"Device.DeviceInfo.X_COMCAST-COM_CM_MAC","type":"dataModel"},{"name":"erouterIpv4","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IP","type":"dataModel"},{"name":"erouterIpv6","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IPv6","type":"dataModel"},{"name":"PartnerId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_Syndication.PartnerId","type":"dataModel"},{"name":"Version","reference":"Device.DeviceInfo.SoftwareVersion","type":"dataModel"},{"name":"AccountId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.AccountInfo.AccountID","type":"dataModel"},{"name":"cpe_passpoint_enable","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_InterworkingServiceEnable","type":"dataModel"},{"name":"cpe_passpoint_inter_parameters","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_InterworkingService.Parameters","type":"dataModel"},{"name":"cpe_passpoint_parameters","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_Passpoint.Parameters","type":"dataModel"},{"name":"cpe_passpoint_rdk_enable","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_Passpoint.Enable","type":"dataModel"},{"name":"open5_bss_active","reference":"Device.WiFi.SSID.6.Enable","type":"dataModel"},{"name":"secure5_bss_active","reference":"Device.WiFi.SSID.10.Enable","type":"dataModel"},{"name":"secure5_radius_server_ip","reference":"Device.WiFi.AccessPoint.10.Security.RadiusServerIPAddr","type":"dataModel"},{"name":"wifi_enabled","reference":"Device.DeviceInfo.X_COMCAST_COM_xfinitywifiEnable","type":"dataModel"},{"name":"primary_tunnel","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.PrimaryRemoteEndpoint","type":"dataModel"},{"name":"secondary_tunnel","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.SecondaryRemoteEndpoint","type":"dataModel"},{"name":"open5_advertisement_str","reference":"Device.WiFi.AccessPoint.6.SSIDAdvertisementEnabled","type":"dataModel"},{"name":"secure5_advertisement_str","reference":"Device.WiFi.AccessPoint.10.SSIDAdvertisementEnabled","type":"dataModel"},{"name":"device_vlan_2","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.Interface.2.VLANID","type":"dataModel"},{"name":"open5_ssid_str","reference":"Device.WiFi.SSID.6.SSID","type":"dataModel"},{"name":"secure5_ssid_str","reference":"Device.WiFi.SSID.10.SSID","type":"dataModel"},{"name":"secure5_securitymode_str","reference":"Device.WiFi.AccessPoint.10.Security.ModeEnabled","type":"dataModel"},{"name":"secure5_pri_port","reference":"Device.WiFi.AccessPoint.10.Security.RadiusServerPort","type":"dataModel"},{"name":"dscp_marker","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.DSCPMarkPolicy","type":"dataModel"},{"name":"WIFI_CH_2_split","reference":"Device.WiFi.Radio.2.Channel","type":"dataModel"},{"name":"WIFI_CW_2_split","reference":"Device.WiFi.Radio.2.OperatingChannelBandwidth","type":"dataModel"},{"name":"open5_bssid_str","reference":"Device.WiFi.SSID.6.BSSID","type":"dataModel"},{"name":"secure5_bssid_str","reference":"Device.WiFi.SSID.10.BSSID","type":"dataModel"},{"name":"open5_status_str","reference":"Device.WiFi.SSID.6.Status","type":"dataModel"},{"name":"secure5_status_str","reference":"Device.WiFi.SSID.10.Status","type":"dataModel"},{"name":"open5_beaconpower_str","reference":"Device.WiFi.AccessPoint.6.X_RDKCENTRAL-COM_ManagementFramePowerControl","type":"dataModel"},{"name":"open5_beaconrate_str","reference":"Device.WiFi.AccessPoint.6.X_RDKCENTRAL-COM_BeaconRate","type":"dataModel"},{"name":"secure5_beaconpower_str","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_ManagementFramePowerControl","type":"dataModel"},{"name":"secure5_beaconrate_str","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_BeaconRate","type":"dataModel"},{"name":"radio5_beaconinterval","reference":"Device.WiFi.Radio.2.X_COMCAST-COM_BeaconInterval","type":"dataModel"},{"name":"secure5_encryption","reference":"Device.WiFi.AccessPoint.10.Security.X_CISCO_COM_EncryptionMethod","type":"dataModel"},{"name":"secure5_sec_radius_server_ip","reference":"Device.WiFi.AccessPoint.10.Security.SecondaryRadiusServerIPAddr","type":"dataModel"},{"name":"secure5_sec_port","reference":"Device.WiFi.AccessPoint.10.Security.SecondaryRadiusServerPort","type":"dataModel"},{"name":"UPTIME_split","reference":"Device.DeviceInfo.UpTime","type":"dataModel"},{"name":"open5_radius_server_ip","reference":"Device.WiFi.AccessPoint.6.Security.RadiusServerIPAddr","type":"dataModel"},{"name":"open5_pri_port","reference":"Device.WiFi.AccessPoint.6.Security.RadiusServerPort","type":"dataModel"},{"name":"open5_isolation_enable","reference":"Device.WiFi.AccessPoint.6.IsolationEnable","type":"dataModel"},{"name":"secure5_isolation_enable","reference":"Device.WiFi.AccessPoint.10.IsolationEnable","type":"dataModel"},{"name":"secure24_bss_active","reference":"Device.WiFi.SSID.9.Enable","type":"dataModel"},{"name":"open24_bss_active","reference":"Device.WiFi.SSID.5.Enable","type":"dataModel"}],"Protocol":"HTTP","ReportingAdjustments":{"FirstReportingInterval":300},"ReportingInterval":86400,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.6"},"versionHash":"d9c6f386"},{"name":"WIFI_MOTION_Telemetry","value":{"Description":"CSCWFM_Telemetry","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"}],"URL":"https://stbrtl-oi.stb.r53.xcal.tv"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"name":"Profile_Name","reference":"Profile.Name","type":"dataModel"},{"name":"Profile","reference":"Device.DeviceInfo.X_RDK_RDKProfileName","type":"dataModel"},{"name":"Time","reference":"Device.Time.X_RDK_CurrentUTCTime","type":"dataModel"},{"name":"mac","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_MAC","type":"dataModel"},{"name":"CMMAC_split","reference":"Device.DeviceInfo.X_COMCAST-COM_CM_MAC","type":"dataModel"},{"name":"erouterIpv4","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IP","type":"dataModel"},{"name":"erouterIpv6","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IPv6","type":"dataModel"},{"name":"PartnerId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_Syndication.PartnerId","type":"dataModel"},{"name":"Version","reference":"Device.DeviceInfo.SoftwareVersion","type":"dataModel"},{"name":"AccountId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.AccountInfo.AccountID","type":"dataModel"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXMrbussub_fail","name":"SYS_ERROR_WFMrbussub_fail","type":"event","use":"count"},{"component":"CSCWFMBRG","eventName":"CSCWFM_CSIpipe_restart","name":"SYS_SH_WFMpipe_restart","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_ctrlifcreate_fail","name":"SYS_ERROR_WFMifcreate_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSC_RXMrbusinit_fail","name":"SYS_ERROR_WFM_rbusinit_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_CSIsessionacquire_fail","name":"SYS_ERROR_WFMSessionAcq_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXMeventscreate_fail","name":"SYS_ERROR_WFMeventscreate_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_CSIsessionenable_fail","name":"SYS_ERROR_WFMsessionenable_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXM_restart","name":"SYS_SH_WFMRXM_restart","type":"event","use":"count"},{"component":"CSCWFM","eventName":"CSCWFM_borg_restart","name":"SYS_SH_WFMborg_restart","type":"event","use":"count"},{"component":"CSCWFM","eventName":"CSCWFM_mqtt_restart","name":"SYS_SH_WFMmqtt_restart","type":"event","use":"count"},{"component":"CSCWFMBRG","eventName":"CSCWFM_sounding_state","name":"WFMsoundingstate_split","type":"event","use":"absolute"},{"name":"WFMEnable_split","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.CognitiveMotionDetection.Enable","type":"dataModel"},{"name":"WFMStatus_split","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_XHFW.WiFiMotionStatus","type":"dataModel"},{"logFile":"ZilkerLog.txt","marker":"wfmCogAgentCommFailCnt_split","search":"wfmCogAgentCommFailCnt:","type":"grep","use":"count"},{"logFile":"ZilkerLog.txt","marker":"wfmCogAgentConnected_split","search":"wfmCogAgentConnected:","type":"grep","use":"absolute"}],"Protocol":"HTTP","ReportingInterval":900,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.1"},"versionHash":"bf86fd16"}]}` + mockedExtraProfilesResponse := `[{"name":"james_test_profile_001","value":{"ActivationTimeout":600,"Description":"Telemetry 2.0 HSD Gateway WiFi Radio","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"},{"Name":"reportVersion","Reference":"Profile.Version"}],"URL":"https://rdkrtldev.stb.r53.xcal.tv/"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"reference":"Profile.Name","type":"dataModel"},{"reference":"Profile.Description","type":"dataModel"},{"reference":"Profile.Version","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.MaxBitRate","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.OperatingFrequencyBand","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.ChannelsInUse","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Channel","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.AutoChannelEnable","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.OperatingChannelBandwidth","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.RadioResetCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.ErrorsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.ErrorsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.DiscardPacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.DiscardPacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PLCPErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.FCSErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_NoiseFloor","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.Noise","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_ChannelUtilization","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_ActivityFactor","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.MaxBitRate","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.OperatingFrequencyBand","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.ChannelsInUse","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Channel","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.AutoChannelEnable","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.OperatingChannelBandwidth","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.RadioResetCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.ErrorsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.ErrorsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.DiscardPacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.DiscardPacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PLCPErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.FCSErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_NoiseFloor","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.Noise","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_ChannelUtilization","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_ActivityFactor","type":"dataModel"}],"Protocol":"HTTP","ReportingInterval":60,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.1"},"versionHash":"ed0de6ef"}]` + + appendedBytes, err := AppendProfiles([]byte(mockedBaseProfilesResponse), []byte(mockedExtraProfilesResponse)) + assert.NilError(t, err) + + var itf interface{} + err = json.Unmarshal(appendedBytes, &itf) + assert.NilError(t, err) + _, err = json.MarshalIndent(itf, "", " ") + assert.NilError(t, err) +} From b289cbe1967f09bade86dc807e97f728eb3ea813 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 Nov 2024 22:20:23 -0800 Subject: [PATCH 117/155] fix a bug that error_code and error_details were not cleanup during state correction --- db/service.go | 10 ++++++++++ http/multipart_test.go | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/db/service.go b/db/service.go index d776cea..0db393f 100644 --- a/db/service.go +++ b/db/service.go @@ -194,6 +194,8 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if len(cloudVersion) == 0 { continue } + cloudErrorCode := *subdocument.ErrorCode() + cloudErrorDetails := *subdocument.ErrorDetails() deviceVersion := deviceVersionMap[subdocId] if cloudVersion == deviceVersion && cloudState >= common.PendingDownload && cloudState <= common.Failure { labels := prometheus.Labels{ @@ -203,6 +205,14 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // update state newState := common.Deployed subdocument.SetState(&newState) + if cloudErrorCode > 0 { + var newErrorCode int + subdocument.SetErrorCode(&newErrorCode) + } + if len(cloudErrorDetails) > 0 { + var newErrorDetails string + subdocument.SetErrorDetails(&newErrorDetails) + } if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } diff --git a/http/multipart_test.go b/http/multipart_test.go index cc92820..abfed54 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1065,7 +1065,11 @@ func TestStateCorrectionEnabled(t *testing.T) { assert.NilError(t, err) meshState := common.Failure + meshErrorCode := 307 + meshErrorDetails := "NACK:OneWifi," meshSubdocument.SetState(&meshState) + meshSubdocument.SetErrorCode(&meshErrorCode) + meshSubdocument.SetErrorDetails(&meshErrorDetails) err = server.SetSubDocument(cpeMac, "mesh", meshSubdocument) assert.NilError(t, err) @@ -1100,6 +1104,8 @@ func TestStateCorrectionEnabled(t *testing.T) { meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Failure) + assert.Equal(t, *meshSubdocument.ErrorCode(), meshErrorCode) + assert.Equal(t, *meshSubdocument.ErrorDetails(), meshErrorDetails) mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) @@ -1132,6 +1138,8 @@ func TestStateCorrectionEnabled(t *testing.T) { meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Deployed) + assert.Equal(t, *meshSubdocument.ErrorCode(), 0) + assert.Equal(t, *meshSubdocument.ErrorDetails(), "") mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) From e037db01b16de22b749c15418afa6d9434aa3e43 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 Nov 2024 22:20:23 -0800 Subject: [PATCH 118/155] fix a bug that error_code and error_details were not cleanup during state correction --- db/service.go | 10 ++++++++++ http/multipart_test.go | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/db/service.go b/db/service.go index d776cea..0db393f 100644 --- a/db/service.go +++ b/db/service.go @@ -194,6 +194,8 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if len(cloudVersion) == 0 { continue } + cloudErrorCode := *subdocument.ErrorCode() + cloudErrorDetails := *subdocument.ErrorDetails() deviceVersion := deviceVersionMap[subdocId] if cloudVersion == deviceVersion && cloudState >= common.PendingDownload && cloudState <= common.Failure { labels := prometheus.Labels{ @@ -203,6 +205,14 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // update state newState := common.Deployed subdocument.SetState(&newState) + if cloudErrorCode > 0 { + var newErrorCode int + subdocument.SetErrorCode(&newErrorCode) + } + if len(cloudErrorDetails) > 0 { + var newErrorDetails string + subdocument.SetErrorDetails(&newErrorDetails) + } if err := c.SetSubDocument(mac, subdocId, &subdocument, cloudState, labels, fields); err != nil { return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } diff --git a/http/multipart_test.go b/http/multipart_test.go index cc92820..abfed54 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1065,7 +1065,11 @@ func TestStateCorrectionEnabled(t *testing.T) { assert.NilError(t, err) meshState := common.Failure + meshErrorCode := 307 + meshErrorDetails := "NACK:OneWifi," meshSubdocument.SetState(&meshState) + meshSubdocument.SetErrorCode(&meshErrorCode) + meshSubdocument.SetErrorDetails(&meshErrorDetails) err = server.SetSubDocument(cpeMac, "mesh", meshSubdocument) assert.NilError(t, err) @@ -1100,6 +1104,8 @@ func TestStateCorrectionEnabled(t *testing.T) { meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Failure) + assert.Equal(t, *meshSubdocument.ErrorCode(), meshErrorCode) + assert.Equal(t, *meshSubdocument.ErrorDetails(), meshErrorDetails) mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) @@ -1132,6 +1138,8 @@ func TestStateCorrectionEnabled(t *testing.T) { meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Deployed) + assert.Equal(t, *meshSubdocument.ErrorCode(), 0) + assert.Equal(t, *meshSubdocument.ErrorDetails(), "") mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) From e00d70e744f5b1d6e27683003b91249491e5a2ec Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 16 Nov 2024 10:42:36 -0800 Subject: [PATCH 119/155] enforce strict format in device GET route --- common/error.go | 1 + config/sample_webconfig.conf | 3 + db/service.go | 10 +- http/multipart.go | 15 ++- http/multipart_test.go | 171 +++++++++++++++++++++++++++++++++ http/webconfig_server.go | 11 +++ http/webconfig_server_test.go | 8 ++ util/string.go | 61 ------------ util/string_test.go | 80 ---------------- util/validator.go | 124 ++++++++++++++++++++++++ util/validator_test.go | 174 ++++++++++++++++++++++++++++++++++ 11 files changed, 511 insertions(+), 147 deletions(-) create mode 100644 util/validator.go create mode 100644 util/validator_test.go diff --git a/common/error.go b/common/error.go index 18b207f..fa22ddf 100644 --- a/common/error.go +++ b/common/error.go @@ -33,6 +33,7 @@ const ( var ( ErrNotOK = fmt.Errorf("!ok") ErrRootDocumentLocked = fmt.Errorf("root document is locked") + ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") ) type Http400Error struct { diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 9ad5f2f..d32fcff 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -259,4 +259,7 @@ webconfig { // get extra profiles from upstream upstream_profiles_enabled = false + + // only valid query_params formats are accepted + query_params_validation_enabled = false } diff --git a/db/service.go b/db/service.go index 0db393f..22c9b9b 100644 --- a/db/service.go +++ b/db/service.go @@ -231,8 +231,14 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } // eval if the root_document is locked - if c.LockRootDocumentEnabled() && cloudRootDocument.Locked() { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + if cloudRootDocument.Locked() { + if c.LockRootDocumentEnabled() { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + } else { + tfields := common.FilterLogFields(fields) + tfields["logger"] = "rootdoc" + log.WithFields(tfields).Warn("dryrun409") + } } switch rootCmpEnum { diff --git a/http/multipart.go b/http/multipart.go index d6c29b5..09ed8c1 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -75,11 +75,18 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. return } fields := xw.Audit() - fields["cpe_mac"] = mac - if qGroupIds, ok := r.URL.Query()["group_id"]; ok { - fields["group_id"] = qGroupIds[0] - r.Header.Set(common.HeaderDocName, qGroupIds[0]) + + // enforce strict query parameters check + err := util.ValidateQueryParams(r, fields) + if err != nil && s.QueryParamsValidationEnabled() { + if errors.Is(err, common.ErrInvalidQueryParams) { + Error(w, http.StatusBadRequest, nil) + log.WithFields(fields).Error(err) + return + } + Error(w, http.StatusInternalServerError, err) + return } // handle empty schema version header diff --git a/http/multipart_test.go b/http/multipart_test.go index abfed54..ce3b84d 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1308,3 +1308,174 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { _, ok = mpartMap["privatessid"] assert.Assert(t, !ok) } + +func TestValidateQueryParams(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetQueryParamsValidationEnabled(true) + assert.Assert(t, server.QueryParamsValidationEnabled()) + defer server.SetQueryParamsValidationEnabled(false) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // case 1 + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 2 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 3 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 4 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 5 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 6 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 7 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 8 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123") + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + etag := res.Header.Get(common.HeaderEtag) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanMpartVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanMpartVersion := mpart.Version + matchedIfNoneMatch := fmt.Sprintf("%v,%v,%v", etag, lanMpartVersion, wanMpartVersion) + + // case 9 versions matched 304 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, matchedIfNoneMatch) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 63d7d7c..b1875b9 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -119,6 +119,7 @@ type WebconfigServer struct { kafkaProducerEnabled bool kafkaProducerTopic string upstreamProfilesEnabled bool + queryParamsValidationEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -286,6 +287,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") + queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") ws := &WebconfigServer{ Server: &http.Server{ @@ -322,6 +324,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, upstreamProfilesEnabled: upstreamProfilesEnabled, + queryParamsValidationEnabled: queryParamsValidationEnabled, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -646,6 +649,14 @@ func (s *WebconfigServer) SetUpstreamProfilesEnabled(enabled bool) { s.upstreamProfilesEnabled = enabled } +func (s *WebconfigServer) QueryParamsValidationEnabled() bool { + return s.queryParamsValidationEnabled +} + +func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { + s.queryParamsValidationEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 6f92f75..404c3eb 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -105,4 +105,12 @@ func TestWebconfigServerSetterGetter(t *testing.T) { enabled = false server.SetUpstreamProfilesEnabled(enabled) assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) + + // enforce strict query parameters validation + enabled = true + server.SetQueryParamsValidationEnabled(enabled) + assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + enabled = false + server.SetQueryParamsValidationEnabled(enabled) + assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) } diff --git a/util/string.go b/util/string.go index 05eab77..064cc8e 100644 --- a/util/string.go +++ b/util/string.go @@ -61,18 +61,6 @@ func GenerateRandomCpeMac() string { return strings.ToUpper(u[len(u)-12:]) } -func ValidateMac(mac string) bool { - if len(mac) != 12 { - return false - } - for _, r := range mac { - if r < 48 || r > 70 || (r > 57 && r < 65) { - return false - } - } - return true -} - func GetTelemetryQueryString(header http.Header, mac, queryParams, partnerId string) string { // build the query parameters in a fixed order params := []string{} @@ -129,55 +117,6 @@ func GetMacDiff(wanMac, mac string) int { return wanMacVal - macVal } -func ValidatePokeQuery(values url.Values) (string, error) { - // handle ?doc=xxx - if docQueryParamStrs, ok := values["doc"]; ok { - if len(docQueryParamStrs) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(docQueryParamStrs[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeDocs, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - - } - return queryStr, nil - } - - // handle ?route=xxx - if qparams, ok := values["route"]; ok { - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(qparams[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeRoutes, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - - } - return queryStr, nil - } - - // return default - return "primary", nil -} - func GetEstbMacAddress(mac string) string { // if the mac cannot be parsed, then return back the input i, err := strconv.ParseInt(mac, 16, 64) diff --git a/util/string_test.go b/util/string_test.go index 73640bc..3c6df1a 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -20,7 +20,6 @@ package util import ( "encoding/json" "net/http" - "net/url" "testing" "github.com/rdkcentral/webconfig/common" @@ -34,28 +33,6 @@ func TestString(t *testing.T) { assert.Equal(t, c, expected) } -func TestValidateMac(t *testing.T) { - mac := "001122334455" - assert.Assert(t, ValidateMac(mac)) - - mac = "4444ABCDEF01" - assert.Assert(t, ValidateMac(mac)) - - mac = "00112233445Z" - assert.Assert(t, !ValidateMac(mac)) - - mac = "001122334455Z" - assert.Assert(t, !ValidateMac(mac)) - - mac = "0H1122334455" - assert.Assert(t, !ValidateMac(mac)) - - for i := 0; i < 10; i++ { - mac := GenerateRandomCpeMac() - assert.Assert(t, ValidateMac(mac)) - } -} - func TestGetAuditId(t *testing.T) { auditId := GetAuditId() assert.Equal(t, len(auditId), 32) @@ -81,63 +58,6 @@ func TestTelemetryQuery(t *testing.T) { assert.Equal(t, qstr, expected) } -func TestValidatePokeQuery(t *testing.T) { - values := url.Values{} - - values["doc"] = []string{ - "primary,telemetry", - "hello,world", - } - _, err := ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary,hello,world", - } - _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary,telemetry", - } - _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary", - } - s, err := ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - values["doc"] = []string{ - "telemetry", - } - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "telemetry") - - delete(values, "doc") - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - values["doc"] = []string{ - "primary", - } - values["route"] = []string{ - "mqtt", - } - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - delete(values, "doc") - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "mqtt") -} - func TestIsValidUTF8(t *testing.T) { b1 := []byte(`{"foo":"bar","hello":123,"world":true}`) assert.Assert(t, IsValidUTF8(b1)) diff --git a/util/validator.go b/util/validator.go new file mode 100644 index 0000000..4ac602c --- /dev/null +++ b/util/validator.go @@ -0,0 +1,124 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package util + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/rdkcentral/webconfig/common" + log "github.com/sirupsen/logrus" +) + +func ValidateMac(mac string) bool { + if len(mac) != 12 { + return false + } + for _, r := range mac { + if r < 48 || r > 70 || (r > 57 && r < 65) { + return false + } + } + return true +} + +func ValidatePokeQuery(values url.Values) (string, error) { + // handle ?doc=xxx + if docQueryParamStrs, ok := values["doc"]; ok { + if len(docQueryParamStrs) > 1 { + err := fmt.Errorf("multiple doc parameter is not allowed") + return "", common.NewError(err) + } + + qparams := strings.Split(docQueryParamStrs[0], ",") + if len(qparams) > 1 { + err := fmt.Errorf("multiple doc parameter is not allowed") + return "", common.NewError(err) + } + + queryStr := qparams[0] + if !Contains(common.SupportedPokeDocs, queryStr) { + err := fmt.Errorf("invalid query parameter: %v", queryStr) + return "", common.NewError(err) + + } + return queryStr, nil + } + + // handle ?route=xxx + if qparams, ok := values["route"]; ok { + if len(qparams) > 1 { + err := fmt.Errorf("multiple route parameter is not allowed") + return "", common.NewError(err) + } + + qparams := strings.Split(qparams[0], ",") + if len(qparams) > 1 { + err := fmt.Errorf("multiple route parameter is not allowed") + return "", common.NewError(err) + } + + queryStr := qparams[0] + if !Contains(common.SupportedPokeRoutes, queryStr) { + err := fmt.Errorf("invalid query parameter: %v", queryStr) + return "", common.NewError(err) + + } + return queryStr, nil + } + + // return default + return "primary", nil +} + +func ValidateQueryParams(r *http.Request, fields log.Fields) error { + groupIdValues, ok := r.URL.Query()["group_id"] + if !ok || len(groupIdValues) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + fields["group_id"] = groupIdValues[0] + r.Header.Set(common.HeaderDocName, groupIdValues[0]) + + subdocIds := strings.Split(groupIdValues[0], ",") + if len(subdocIds) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + + if len(subdocIds) > 0 && subdocIds[0] != "root" { + return common.NewError(common.ErrInvalidQueryParams) + } + + for _, subdocId := range subdocIds[1:] { + if _, ok := common.SubdocBitIndexMap[subdocId]; !ok { + return common.NewError(common.ErrInvalidQueryParams) + } + } + + ifNoneMatch := r.Header.Get(common.HeaderIfNoneMatch) + if len(ifNoneMatch) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + + versions := strings.Split(ifNoneMatch, ",") + if len(versions) != len(subdocIds) { + return common.NewError(common.ErrInvalidQueryParams) + } + return nil +} diff --git a/util/validator_test.go b/util/validator_test.go new file mode 100644 index 0000000..9109b82 --- /dev/null +++ b/util/validator_test.go @@ -0,0 +1,174 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package util + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/rdkcentral/webconfig/common" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestValidateMac(t *testing.T) { + mac := "001122334455" + assert.Assert(t, ValidateMac(mac)) + + mac = "4444ABCDEF01" + assert.Assert(t, ValidateMac(mac)) + + mac = "00112233445Z" + assert.Assert(t, !ValidateMac(mac)) + + mac = "001122334455Z" + assert.Assert(t, !ValidateMac(mac)) + + mac = "0H1122334455" + assert.Assert(t, !ValidateMac(mac)) + + for i := 0; i < 10; i++ { + mac := GenerateRandomCpeMac() + assert.Assert(t, ValidateMac(mac)) + } +} + +func TestValidatePokeQuery(t *testing.T) { + values := url.Values{} + + values["doc"] = []string{ + "primary,telemetry", + "hello,world", + } + _, err := ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary,hello,world", + } + _, err = ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary,telemetry", + } + _, err = ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary", + } + s, err := ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + values["doc"] = []string{ + "telemetry", + } + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "telemetry") + + delete(values, "doc") + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + values["doc"] = []string{ + "primary", + } + values["route"] = []string{ + "mqtt", + } + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + delete(values, "doc") + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "mqtt") +} + +func TestValidateQueryParams(t *testing.T) { + cpeMac := GenerateRandomCpeMac() + + // case 1 + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + fields := make(log.Fields) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 2 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 3 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 4 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 5 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 6 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 7 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 8 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid,lan", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + err = ValidateQueryParams(req, fields) + assert.NilError(t, err) +} From f4b217e0a05153fdda395bf7d310f8e476311fff Mon Sep 17 00:00:00 2001 From: James Chao Date: Sun, 17 Nov 2024 11:28:50 -0800 Subject: [PATCH 120/155] add token trust validation --- common/error.go | 1 + config/sample_webconfig.conf | 3 ++ http/multipart_test.go | 29 +++++++++++++-- http/webconfig_server.go | 30 +++++++++++++--- http/webconfig_server_test.go | 8 +++++ security/main_test.go | 6 ++-- security/token.go | 68 +++++++++++++++++++---------------- security/token_test.go | 22 +++++++----- 8 files changed, 119 insertions(+), 48 deletions(-) diff --git a/common/error.go b/common/error.go index fa22ddf..9a1f4d1 100644 --- a/common/error.go +++ b/common/error.go @@ -34,6 +34,7 @@ var ( ErrNotOK = fmt.Errorf("!ok") ErrRootDocumentLocked = fmt.Errorf("root document is locked") ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") + ErrLowTrust = fmt.Errorf("token trust is lower than threshold") ) type Http400Error struct { diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index d32fcff..e7bcdea 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -262,4 +262,7 @@ webconfig { // only valid query_params formats are accepted query_params_validation_enabled = false + + // only devices using tokens with trust higher or equal can GET the full document + min_trust = 0 } diff --git a/http/multipart_test.go b/http/multipart_test.go index ce3b84d..3d26e81 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -253,7 +253,7 @@ func TestCpeMiddleware(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router1).Result() - rbytes, err = io.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusForbidden) @@ -264,7 +264,32 @@ func TestCpeMiddleware(t *testing.T) { assert.NilError(t, err) req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", token)) res = ExecuteRequest(req, router1).Result() - rbytes, err = io.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // change the min trust to 1000 + server1.SetMinTrust(1000) + assert.Equal(t, 1000, server1.MinTrust()) + zeroToken := server1.Generate(cpeMac, 86400, 0) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", zeroToken)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + + // change the min trust back to 0 + server1.SetMinTrust(0) + assert.Equal(t, 0, server1.MinTrust()) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", zeroToken)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index b1875b9..9cc3aca 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -120,6 +120,7 @@ type WebconfigServer struct { kafkaProducerTopic string upstreamProfilesEnabled bool queryParamsValidationEnabled bool + minTrust int } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -204,12 +205,14 @@ func GetDatabaseClient(sc *common.ServerConfig) db.DatabaseClient { func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer { conf := sc.Config var dbclient db.DatabaseClient + var tokenManager *security.TokenManager // setup up database client if testOnly { dbclient = GetTestDatabaseClient(sc) } else { dbclient = GetDatabaseClient(sc) + tokenManager = security.NewTokenManager(conf) } // setup jwks manager @@ -288,6 +291,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") + minTrust := int(conf.GetInt32("webconfig.min_trust")) ws := &WebconfigServer{ Server: &http.Server{ @@ -296,7 +300,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer WriteTimeout: time.Duration(conf.GetInt32("webconfig.server.write_timeout_in_secs", 3)) * time.Second, }, DatabaseClient: dbclient, - TokenManager: security.NewTokenManager(conf), + TokenManager: tokenManager, JwksManager: jwksManager, ServerConfig: sc, WebpaConnector: NewWebpaConnector(conf, tlsConfig), @@ -325,6 +329,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerTopic: kafkaProducerTopic, upstreamProfilesEnabled: upstreamProfilesEnabled, queryParamsValidationEnabled: queryParamsValidationEnabled, + minTrust: minTrust, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -390,14 +395,21 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { return } - if ok, partnerId, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true + fields := xw.Audit() + fields["src_partner"] = partnerId + fields["trust"] = trust + if err := s.ValidatePartner(partnerId); err != nil { - fields := xw.Audit() - fields["src_partner"] = partnerId + // isValid = false partnerId = "unknown" tokenErr = common.NewError(err) } + if trust < s.MinTrust() { + isValid = false + tokenErr = common.NewError(common.ErrLowTrust) + } xw.SetPartnerId(partnerId) } else { tokenErr = common.NewError(err) @@ -475,7 +487,7 @@ func (s *WebconfigServer) TestingCpeMiddleware(next http.Handler) http.Handler { return } - if ok, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + if ok, _, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true } } @@ -657,6 +669,14 @@ func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { s.queryParamsValidationEnabled = enabled } +func (s *WebconfigServer) MinTrust() int { + return s.minTrust +} + +func (s *WebconfigServer) SetMinTrust(trust int) { + s.minTrust = trust +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 404c3eb..fd4c138 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -113,4 +113,12 @@ func TestWebconfigServerSetterGetter(t *testing.T) { enabled = false server.SetQueryParamsValidationEnabled(enabled) assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + + //configure trust level + trust := 1000 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) + trust = 500 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) } diff --git a/security/main_test.go b/security/main_test.go index b1223c8..d822e54 100644 --- a/security/main_test.go +++ b/security/main_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -39,7 +39,9 @@ func TestMain(m *testing.M) { NewTestCodec(sc.Config) - tokenManager = NewTokenManager(sc.Config) + if sc.Config.GetBoolean("webconfig.jwt.enabled", false) || os.Getenv("TOKEN_TEST") == "1" { + tokenManager = NewTokenManager(sc.Config) + } log.SetOutput(io.Discard) returnCode := m.Run() diff --git a/security/token.go b/security/token.go index 169cd0f..47270a4 100644 --- a/security/token.go +++ b/security/token.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -26,11 +26,11 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -42,13 +42,13 @@ type ThemisClaims struct { Mac string `json:"mac"` PartnerId string `json:"partner-id"` Serial string `json:"serial"` - Trust string `json:"trust"` + Trust int `json:"trust"` Uuid string `json:"uuid"` Capabilities []string `json:"capabilities"` jwt.RegisteredClaims } -type VerifyFunc func(map[string]*rsa.PublicKey, []string, []string, ...string) (bool, string, error) +type VerifyFunc func(map[string]*rsa.PublicKey, []string, []string, ...string) (bool, string, int, error) type TokenManager struct { encodeKey *rsa.PrivateKey @@ -61,11 +61,6 @@ type TokenManager struct { } func NewTokenManager(conf *configuration.Config) *TokenManager { - jwtEnabled := conf.GetBoolean("webconfig.jwt.enabled", false) - if !jwtEnabled { - return nil - } - panicExitEnabled := conf.GetBoolean("webconfig.panic_exit_enabled", false) // prepare args for TokenManager @@ -134,19 +129,25 @@ func loadEncodeKey(keyfile string) (*rsa.PrivateKey, error) { } // TODO this is not an officially supported function. -func (m *TokenManager) Generate(mac string, ttl int64, vargs ...string) string { +func (m *TokenManager) Generate(mac string, ttl int64, itfs ...interface{}) string { // %% NOTE mac should be lowercase to be consistent with reference doc // static themis fields copied from examples in the webconfig confluence kid := "webconfig_key" serial := "ABCNDGE" - trust := "1000" + trust := 1000 capUuid := "1234567891234" capabilities := []string{"x1:issuer:test:.*:all"} - partner := "comcast" - if len(vargs) > 0 { - partner = vargs[0] + + for _, itf := range itfs { + switch ty := itf.(type) { + case string: + partner = ty + case int: + trust = ty + } } + utcnow := time.Now() claims := ThemisClaims{ @@ -224,19 +225,19 @@ func ParseKidFromTokenHeader(tokenString string) (string, error) { } func (m *TokenManager) VerifyApiToken(token string) (bool, error) { - ok, _, err := m.verifyFn(m.decodeKeys, m.apiKids, m.apiCapabilities, token) + ok, _, _, err := m.verifyFn(m.decodeKeys, m.apiKids, m.apiCapabilities, token) if err != nil { return ok, common.NewError(err) } return ok, err } -func (m *TokenManager) VerifyCpeToken(token string, mac string) (bool, string, error) { - ok, partner, err := m.verifyFn(m.decodeKeys, m.cpeKids, m.cpeCapabilities, token, mac) +func (m *TokenManager) VerifyCpeToken(token string, mac string) (bool, string, int, error) { + ok, partner, trust, err := m.verifyFn(m.decodeKeys, m.cpeKids, m.cpeCapabilities, token, mac) if err != nil { - return ok, "", common.NewError(err) + return ok, "", trust, common.NewError(err) } - return ok, partner, nil + return ok, partner, trust, nil } func (m *TokenManager) SetVerifyFunc(fn VerifyFunc) { @@ -293,9 +294,10 @@ func (m *TokenManager) ParseCpeToken(tokenStr string) (map[string]string, error) return data, nil } -func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, error) { +func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, int, error) { tokenString := vargs[0] var kid string + var trust int parser := &jwt.Parser{} @@ -305,11 +307,11 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi // check kid rawkid, ok := token.Header["kid"] if !ok { - return false, "", common.NewError(fmt.Errorf("missing kid in token")) + return false, "", trust, common.NewError(fmt.Errorf("missing kid in token")) } kid, ok = rawkid.(string) if !ok { - return false, "", common.NewError(fmt.Errorf("error in reading kid from header")) + return false, "", trust, common.NewError(fmt.Errorf("error in reading kid from header")) } ok = false @@ -320,7 +322,7 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } if !ok { - return false, "", common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) + return false, "", trust, common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) } // check capabilities, if requiredCapabilities is nonempty @@ -343,21 +345,21 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } if !isCapable { - return false, "", common.NewError(fmt.Errorf("token without proper capabilities")) + return false, "", trust, common.NewError(fmt.Errorf("token without proper capabilities")) } } } else { - return false, "", common.NewError(err) + return false, "", trust, common.NewError(err) } decodeKey, ok := decodeKeys[kid] if !ok { - return false, "", common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) + return false, "", trust, common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) } claims := jwt.MapClaims{} if _, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) { return decodeKey, nil }); err != nil { - return false, "", common.NewError(err) + return false, "", trust, common.NewError(err) } if len(vargs) > 1 { @@ -369,12 +371,12 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi if strings.ToLower(mac) == strings.ToLower(macstr) { isMatched = true } else { - return false, "", common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) } } } if !isMatched { - return false, "", common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) } } @@ -384,5 +386,9 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi partner = itf.(string) } - return true, partner, nil + if itf, ok := claims["trust"]; ok { + trust = util.ToInt(itf) + } + + return true, partner, trust, nil } diff --git a/security/token_test.go b/security/token_test.go index 5f6bab5..6f73aa6 100644 --- a/security/token_test.go +++ b/security/token_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -55,11 +55,7 @@ func TestLoadingKeyFiles(t *testing.T) { } func TestTokenValidation(t *testing.T) { - sc, err := common.GetTestServerConfig() - if err != nil { - panic(err) - } - if !sc.GetBoolean("webconfig.jwt.enabled") { + if tokenManager == nil { t.Skip("webconfig.jwt.enabled = false") } @@ -67,16 +63,26 @@ func TestTokenValidation(t *testing.T) { token := tokenManager.Generate(strings.ToLower(cpeMac), 86400) // default comcast - ok, parsedPartner, err := tokenManager.VerifyCpeToken(token, cpeMac) + ok, parsedPartner, trust, err := tokenManager.VerifyCpeToken(token, cpeMac) assert.NilError(t, err) assert.Assert(t, ok) assert.Equal(t, parsedPartner, "comcast") + assert.Equal(t, trust, 1000) // create a partner token partner1 := "cox" token1 := tokenManager.Generate(strings.ToLower(cpeMac), 86400, partner1) - ok, parsedPartner, err = tokenManager.VerifyCpeToken(token1, cpeMac) + ok, parsedPartner, trust, err = tokenManager.VerifyCpeToken(token1, cpeMac) + assert.NilError(t, err) + assert.Assert(t, ok) + assert.Equal(t, parsedPartner, partner1) + assert.Equal(t, trust, 1000) + + // create a partner token with non-default trust + token2 := tokenManager.Generate(strings.ToLower(cpeMac), 86400, partner1, 500) + ok, parsedPartner, trust, err = tokenManager.VerifyCpeToken(token2, cpeMac) assert.NilError(t, err) assert.Assert(t, ok) assert.Equal(t, parsedPartner, partner1) + assert.Equal(t, trust, 500) } From 07e4e16dcea444ad7cf0fd54700fa55f5ea8f7a6 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sat, 16 Nov 2024 10:42:36 -0800 Subject: [PATCH 121/155] cherrypick strict format check and resolve conflicts --- common/error.go | 1 + config/sample_webconfig.conf | 3 + db/service.go | 10 +- http/multipart.go | 15 ++- http/multipart_test.go | 171 +++++++++++++++++++++++++++++++++ http/webconfig_server.go | 12 +++ http/webconfig_server_test.go | 10 +- util/string.go | 61 ------------ util/string_test.go | 80 ---------------- util/validator.go | 124 ++++++++++++++++++++++++ util/validator_test.go | 174 ++++++++++++++++++++++++++++++++++ 11 files changed, 513 insertions(+), 148 deletions(-) create mode 100644 util/validator.go create mode 100644 util/validator_test.go diff --git a/common/error.go b/common/error.go index 18b207f..fa22ddf 100644 --- a/common/error.go +++ b/common/error.go @@ -33,6 +33,7 @@ const ( var ( ErrNotOK = fmt.Errorf("!ok") ErrRootDocumentLocked = fmt.Errorf("root document is locked") + ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") ) type Http400Error struct { diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 22dacea..00b2f03 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -256,4 +256,7 @@ webconfig { // this allows the root document locked if needed lock_root_document_enabled = false + + // only valid query_params formats are accepted + query_params_validation_enabled = false } diff --git a/db/service.go b/db/service.go index 0db393f..22c9b9b 100644 --- a/db/service.go +++ b/db/service.go @@ -231,8 +231,14 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } // eval if the root_document is locked - if c.LockRootDocumentEnabled() && cloudRootDocument.Locked() { - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + if cloudRootDocument.Locked() { + if c.LockRootDocumentEnabled() { + return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, messages, common.NewError(common.ErrRootDocumentLocked) + } else { + tfields := common.FilterLogFields(fields) + tfields["logger"] = "rootdoc" + log.WithFields(tfields).Warn("dryrun409") + } } switch rootCmpEnum { diff --git a/http/multipart.go b/http/multipart.go index d6c29b5..09ed8c1 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -75,11 +75,18 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. return } fields := xw.Audit() - fields["cpe_mac"] = mac - if qGroupIds, ok := r.URL.Query()["group_id"]; ok { - fields["group_id"] = qGroupIds[0] - r.Header.Set(common.HeaderDocName, qGroupIds[0]) + + // enforce strict query parameters check + err := util.ValidateQueryParams(r, fields) + if err != nil && s.QueryParamsValidationEnabled() { + if errors.Is(err, common.ErrInvalidQueryParams) { + Error(w, http.StatusBadRequest, nil) + log.WithFields(fields).Error(err) + return + } + Error(w, http.StatusInternalServerError, err) + return } // handle empty schema version header diff --git a/http/multipart_test.go b/http/multipart_test.go index abfed54..ce3b84d 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1308,3 +1308,174 @@ func TestCorruptedEncryptedDocumentHandler(t *testing.T) { _, ok = mpartMap["privatessid"] assert.Assert(t, !ok) } + +func TestValidateQueryParams(t *testing.T) { + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + server.SetQueryParamsValidationEnabled(true) + assert.Assert(t, server.QueryParamsValidationEnabled()) + defer server.SetQueryParamsValidationEnabled(false) + + cpeMac := util.GenerateRandomCpeMac() + // ==== group 1 lan ==== + subdocId := "lan" + m, n := 50, 100 + lanBytes := util.RandomBytes(m, n) + + // post + url := fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err := http.NewRequest("POST", url, bytes.NewReader(lanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, lanBytes) + + // ==== group 2 wan ==== + subdocId = "wan" + wanBytes := util.RandomBytes(m, n) + + // post + url = fmt.Sprintf("/api/v1/device/%v/document/%v", cpeMac, subdocId) + req, err = http.NewRequest("POST", url, bytes.NewReader(wanBytes)) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // get + req, err = http.NewRequest("GET", url, nil) + req.Header.Set(common.HeaderContentType, common.HeaderApplicationMsgpack) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + assert.DeepEqual(t, rbytes, wanBytes) + + // case 1 + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 2 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 3 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 4 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 5 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 6 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 7 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusBadRequest) + + // case 8 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123") + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 2) + etag := res.Header.Get(common.HeaderEtag) + mpart, ok := mparts["lan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, lanBytes) + lanMpartVersion := mpart.Version + mpart, ok = mparts["wan"] + assert.Assert(t, ok) + assert.DeepEqual(t, mpart.Bytes, wanBytes) + wanMpartVersion := mpart.Version + matchedIfNoneMatch := fmt.Sprintf("%v,%v,%v", etag, lanMpartVersion, wanMpartVersion) + + // case 9 versions matched 304 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,lan,wan", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, matchedIfNoneMatch) + res = ExecuteRequest(req, router).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusNotModified) +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index a7e32c8..06dfe11 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -118,6 +118,7 @@ type WebconfigServer struct { webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string + queryParamsValidationEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -284,6 +285,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } } + queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), @@ -318,6 +321,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, + queryParamsValidationEnabled: queryParamsValidationEnabled, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -634,6 +638,14 @@ func (s *WebconfigServer) SetKafkaProducerTopic(x string) { s.kafkaProducerTopic = x } +func (s *WebconfigServer) QueryParamsValidationEnabled() bool { + return s.queryParamsValidationEnabled +} + +func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { + s.queryParamsValidationEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 1699106..62224a6 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -97,4 +97,12 @@ func TestWebconfigServerSetterGetter(t *testing.T) { validPartners = []string{"name3", "name4", "name5"} server.SetValidPartners(validPartners) assert.DeepEqual(t, server.ValidPartners(), validPartners) + + // enforce strict query parameters validation + enabled = true + server.SetQueryParamsValidationEnabled(enabled) + assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + enabled = false + server.SetQueryParamsValidationEnabled(enabled) + assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) } diff --git a/util/string.go b/util/string.go index 439b0bb..bd1eac4 100644 --- a/util/string.go +++ b/util/string.go @@ -53,18 +53,6 @@ func GenerateRandomCpeMac() string { return strings.ToUpper(u[len(u)-12:]) } -func ValidateMac(mac string) bool { - if len(mac) != 12 { - return false - } - for _, r := range mac { - if r < 48 || r > 70 || (r > 57 && r < 65) { - return false - } - } - return true -} - func GetTelemetryQueryString(header http.Header, mac, queryParams, partnerId string) string { // build the query parameters in a fixed order params := []string{} @@ -121,55 +109,6 @@ func GetMacDiff(wanMac, mac string) int { return wanMacVal - macVal } -func ValidatePokeQuery(values url.Values) (string, error) { - // handle ?doc=xxx - if docQueryParamStrs, ok := values["doc"]; ok { - if len(docQueryParamStrs) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(docQueryParamStrs[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeDocs, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - - } - return queryStr, nil - } - - // handle ?route=xxx - if qparams, ok := values["route"]; ok { - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(qparams[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeRoutes, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - - } - return queryStr, nil - } - - // return default - return "primary", nil -} - func GetEstbMacAddress(mac string) string { // if the mac cannot be parsed, then return back the input i, err := strconv.ParseInt(mac, 16, 64) diff --git a/util/string_test.go b/util/string_test.go index cf6a894..8fb5288 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -19,7 +19,6 @@ package util import ( "net/http" - "net/url" "testing" "github.com/rdkcentral/webconfig/common" @@ -33,28 +32,6 @@ func TestString(t *testing.T) { assert.Equal(t, c, expected) } -func TestValidateMac(t *testing.T) { - mac := "001122334455" - assert.Assert(t, ValidateMac(mac)) - - mac = "4444ABCDEF01" - assert.Assert(t, ValidateMac(mac)) - - mac = "00112233445Z" - assert.Assert(t, !ValidateMac(mac)) - - mac = "001122334455Z" - assert.Assert(t, !ValidateMac(mac)) - - mac = "0H1122334455" - assert.Assert(t, !ValidateMac(mac)) - - for i := 0; i < 10; i++ { - mac := GenerateRandomCpeMac() - assert.Assert(t, ValidateMac(mac)) - } -} - func TestGetAuditId(t *testing.T) { auditId := GetAuditId() assert.Equal(t, len(auditId), 32) @@ -80,63 +57,6 @@ func TestTelemetryQuery(t *testing.T) { assert.Equal(t, qstr, expected) } -func TestValidatePokeQuery(t *testing.T) { - values := url.Values{} - - values["doc"] = []string{ - "primary,telemetry", - "hello,world", - } - _, err := ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary,hello,world", - } - _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary,telemetry", - } - _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) - - values["doc"] = []string{ - "primary", - } - s, err := ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - values["doc"] = []string{ - "telemetry", - } - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "telemetry") - - delete(values, "doc") - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - values["doc"] = []string{ - "primary", - } - values["route"] = []string{ - "mqtt", - } - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "primary") - - delete(values, "doc") - s, err = ValidatePokeQuery(values) - assert.NilError(t, err) - assert.Equal(t, s, "mqtt") -} - func TestIsValidUTF8(t *testing.T) { b1 := []byte(`{"foo":"bar","hello":123,"world":true}`) assert.Assert(t, IsValidUTF8(b1)) diff --git a/util/validator.go b/util/validator.go new file mode 100644 index 0000000..4ac602c --- /dev/null +++ b/util/validator.go @@ -0,0 +1,124 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package util + +import ( + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/rdkcentral/webconfig/common" + log "github.com/sirupsen/logrus" +) + +func ValidateMac(mac string) bool { + if len(mac) != 12 { + return false + } + for _, r := range mac { + if r < 48 || r > 70 || (r > 57 && r < 65) { + return false + } + } + return true +} + +func ValidatePokeQuery(values url.Values) (string, error) { + // handle ?doc=xxx + if docQueryParamStrs, ok := values["doc"]; ok { + if len(docQueryParamStrs) > 1 { + err := fmt.Errorf("multiple doc parameter is not allowed") + return "", common.NewError(err) + } + + qparams := strings.Split(docQueryParamStrs[0], ",") + if len(qparams) > 1 { + err := fmt.Errorf("multiple doc parameter is not allowed") + return "", common.NewError(err) + } + + queryStr := qparams[0] + if !Contains(common.SupportedPokeDocs, queryStr) { + err := fmt.Errorf("invalid query parameter: %v", queryStr) + return "", common.NewError(err) + + } + return queryStr, nil + } + + // handle ?route=xxx + if qparams, ok := values["route"]; ok { + if len(qparams) > 1 { + err := fmt.Errorf("multiple route parameter is not allowed") + return "", common.NewError(err) + } + + qparams := strings.Split(qparams[0], ",") + if len(qparams) > 1 { + err := fmt.Errorf("multiple route parameter is not allowed") + return "", common.NewError(err) + } + + queryStr := qparams[0] + if !Contains(common.SupportedPokeRoutes, queryStr) { + err := fmt.Errorf("invalid query parameter: %v", queryStr) + return "", common.NewError(err) + + } + return queryStr, nil + } + + // return default + return "primary", nil +} + +func ValidateQueryParams(r *http.Request, fields log.Fields) error { + groupIdValues, ok := r.URL.Query()["group_id"] + if !ok || len(groupIdValues) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + fields["group_id"] = groupIdValues[0] + r.Header.Set(common.HeaderDocName, groupIdValues[0]) + + subdocIds := strings.Split(groupIdValues[0], ",") + if len(subdocIds) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + + if len(subdocIds) > 0 && subdocIds[0] != "root" { + return common.NewError(common.ErrInvalidQueryParams) + } + + for _, subdocId := range subdocIds[1:] { + if _, ok := common.SubdocBitIndexMap[subdocId]; !ok { + return common.NewError(common.ErrInvalidQueryParams) + } + } + + ifNoneMatch := r.Header.Get(common.HeaderIfNoneMatch) + if len(ifNoneMatch) == 0 { + return common.NewError(common.ErrInvalidQueryParams) + } + + versions := strings.Split(ifNoneMatch, ",") + if len(versions) != len(subdocIds) { + return common.NewError(common.ErrInvalidQueryParams) + } + return nil +} diff --git a/util/validator_test.go b/util/validator_test.go new file mode 100644 index 0000000..9109b82 --- /dev/null +++ b/util/validator_test.go @@ -0,0 +1,174 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package util + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/rdkcentral/webconfig/common" + log "github.com/sirupsen/logrus" + "gotest.tools/assert" +) + +func TestValidateMac(t *testing.T) { + mac := "001122334455" + assert.Assert(t, ValidateMac(mac)) + + mac = "4444ABCDEF01" + assert.Assert(t, ValidateMac(mac)) + + mac = "00112233445Z" + assert.Assert(t, !ValidateMac(mac)) + + mac = "001122334455Z" + assert.Assert(t, !ValidateMac(mac)) + + mac = "0H1122334455" + assert.Assert(t, !ValidateMac(mac)) + + for i := 0; i < 10; i++ { + mac := GenerateRandomCpeMac() + assert.Assert(t, ValidateMac(mac)) + } +} + +func TestValidatePokeQuery(t *testing.T) { + values := url.Values{} + + values["doc"] = []string{ + "primary,telemetry", + "hello,world", + } + _, err := ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary,hello,world", + } + _, err = ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary,telemetry", + } + _, err = ValidatePokeQuery(values) + assert.Assert(t, err != nil) + + values["doc"] = []string{ + "primary", + } + s, err := ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + values["doc"] = []string{ + "telemetry", + } + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "telemetry") + + delete(values, "doc") + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + values["doc"] = []string{ + "primary", + } + values["route"] = []string{ + "mqtt", + } + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "primary") + + delete(values, "doc") + s, err = ValidatePokeQuery(values) + assert.NilError(t, err) + assert.Equal(t, s, "mqtt") +} + +func TestValidateQueryParams(t *testing.T) { + cpeMac := GenerateRandomCpeMac() + + // case 1 + deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + fields := make(log.Fields) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 2 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 3 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 4 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 5 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 6 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,foo", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 7 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + err = ValidateQueryParams(req, fields) + assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) + + // case 8 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid,lan", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") + err = ValidateQueryParams(req, fields) + assert.NilError(t, err) +} From ddbbb08dc410d9ded8daa4e7b70b85b61c35d020 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 19 Nov 2024 22:25:55 -0800 Subject: [PATCH 122/155] accept non bitmap subdocs when the validator is enabled during device get route --- config/sample_webconfig.conf | 3 +++ http/multipart.go | 2 +- http/webconfig_server.go | 16 ++++++++++++++++ http/webconfig_server_test.go | 10 ++++++++++ util/validator.go | 6 +++--- util/validator_test.go | 30 +++++++++++++++++++++--------- 6 files changed, 54 insertions(+), 13 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index e7bcdea..6d2b232 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -265,4 +265,7 @@ webconfig { // only devices using tokens with trust higher or equal can GET the full document min_trust = 0 + + // subdoc ids accepted by the validator even if they are without bitmap definition + valid_subdoc_ids = [] } diff --git a/http/multipart.go b/http/multipart.go index 09ed8c1..5e39531 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -78,7 +78,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. fields["cpe_mac"] = mac // enforce strict query parameters check - err := util.ValidateQueryParams(r, fields) + err := util.ValidateQueryParams(r, s.ValidSubdocIdMap(), fields) if err != nil && s.QueryParamsValidationEnabled() { if errors.Is(err, common.ErrInvalidQueryParams) { Error(w, http.StatusBadRequest, nil) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 9cc3aca..4cb161a 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -26,6 +26,7 @@ import ( "errors" "fmt" "io" + "maps" "net/http" "os" "strings" @@ -121,6 +122,7 @@ type WebconfigServer struct { upstreamProfilesEnabled bool queryParamsValidationEnabled bool minTrust int + validSubdocIdMap map[string]int } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -292,6 +294,11 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") minTrust := int(conf.GetInt32("webconfig.min_trust")) + validSubdocIds := conf.GetStringList("webconfig.valid_subdoc_ids") + validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) + for _, x := range validSubdocIds { + validSubdocIdMap[x] = 1 + } ws := &WebconfigServer{ Server: &http.Server{ @@ -330,6 +337,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer upstreamProfilesEnabled: upstreamProfilesEnabled, queryParamsValidationEnabled: queryParamsValidationEnabled, minTrust: minTrust, + validSubdocIdMap: validSubdocIdMap, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -677,6 +685,14 @@ func (s *WebconfigServer) SetMinTrust(trust int) { s.minTrust = trust } +func (s *WebconfigServer) ValidSubdocIdMap() map[string]int { + return s.validSubdocIdMap +} + +func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { + s.validSubdocIdMap = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index fd4c138..0c028aa 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -121,4 +121,14 @@ func TestWebconfigServerSetterGetter(t *testing.T) { trust = 500 server.SetMinTrust(trust) assert.Equal(t, server.MinTrust(), trust) + + validSubdocIdMap := map[string]int{ + "red": 1, + "orange": 2, + "yellow": 3, + "green": 4, + } + server.SetValidSubdocIdMap(validSubdocIdMap) + assert.DeepEqual(t, validSubdocIdMap, server.ValidSubdocIdMap()) + } diff --git a/util/validator.go b/util/validator.go index 4ac602c..754044f 100644 --- a/util/validator.go +++ b/util/validator.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -88,7 +88,7 @@ func ValidatePokeQuery(values url.Values) (string, error) { return "primary", nil } -func ValidateQueryParams(r *http.Request, fields log.Fields) error { +func ValidateQueryParams(r *http.Request, validSubdocIdMap map[string]int, fields log.Fields) error { groupIdValues, ok := r.URL.Query()["group_id"] if !ok || len(groupIdValues) == 0 { return common.NewError(common.ErrInvalidQueryParams) @@ -106,7 +106,7 @@ func ValidateQueryParams(r *http.Request, fields log.Fields) error { } for _, subdocId := range subdocIds[1:] { - if _, ok := common.SubdocBitIndexMap[subdocId]; !ok { + if _, ok := validSubdocIdMap[subdocId]; !ok { return common.NewError(common.ErrInvalidQueryParams) } } diff --git a/util/validator_test.go b/util/validator_test.go index 9109b82..4313f03 100644 --- a/util/validator_test.go +++ b/util/validator_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( "errors" "fmt" + "maps" "net/http" "net/url" "testing" @@ -110,34 +111,37 @@ func TestValidatePokeQuery(t *testing.T) { func TestValidateQueryParams(t *testing.T) { cpeMac := GenerateRandomCpeMac() + validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) + validSubdocIdMap["red"] = 1 + validSubdocIdMap["orange"] = 1 // case 1 deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) req, err := http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) fields := make(log.Fields) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 2 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 3 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 4 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 5 @@ -145,7 +149,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 6 @@ -153,7 +157,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 7 @@ -161,7 +165,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 8 @@ -169,6 +173,14 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.NilError(t, err) + + // case 9 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid,red,orange", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456,678") + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.NilError(t, err) } From 7ccaf8cdea3ec402b7a21100514154ee57458150 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 19 Nov 2024 22:25:55 -0800 Subject: [PATCH 123/155] cherrypick non bitmap validation and resolve conflicts --- config/sample_webconfig.conf | 3 +++ http/multipart.go | 2 +- http/webconfig_server.go | 16 ++++++++++++++++ http/webconfig_server_test.go | 9 +++++++++ util/validator.go | 6 +++--- util/validator_test.go | 30 +++++++++++++++++++++--------- 6 files changed, 53 insertions(+), 13 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 00b2f03..013f180 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -259,4 +259,7 @@ webconfig { // only valid query_params formats are accepted query_params_validation_enabled = false + + // subdoc ids accepted by the validator even if they are without bitmap definition + valid_subdoc_ids = [] } diff --git a/http/multipart.go b/http/multipart.go index 09ed8c1..5e39531 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -78,7 +78,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. fields["cpe_mac"] = mac // enforce strict query parameters check - err := util.ValidateQueryParams(r, fields) + err := util.ValidateQueryParams(r, s.ValidSubdocIdMap(), fields) if err != nil && s.QueryParamsValidationEnabled() { if errors.Is(err, common.ErrInvalidQueryParams) { Error(w, http.StatusBadRequest, nil) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 06dfe11..7397acd 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -26,6 +26,7 @@ import ( "errors" "fmt" "io" + "maps" "net/http" "os" "strings" @@ -119,6 +120,7 @@ type WebconfigServer struct { kafkaProducerEnabled bool kafkaProducerTopic string queryParamsValidationEnabled bool + validSubdocIdMap map[string]int } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -286,6 +288,11 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") + validSubdocIds := conf.GetStringList("webconfig.valid_subdoc_ids") + validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) + for _, x := range validSubdocIds { + validSubdocIdMap[x] = 1 + } ws := &WebconfigServer{ Server: &http.Server{ @@ -322,6 +329,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, queryParamsValidationEnabled: queryParamsValidationEnabled, + validSubdocIdMap: validSubdocIdMap, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -646,6 +654,14 @@ func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { s.queryParamsValidationEnabled = enabled } +func (s *WebconfigServer) ValidSubdocIdMap() map[string]int { + return s.validSubdocIdMap +} + +func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { + s.validSubdocIdMap = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 62224a6..74ee523 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -105,4 +105,13 @@ func TestWebconfigServerSetterGetter(t *testing.T) { enabled = false server.SetQueryParamsValidationEnabled(enabled) assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + + validSubdocIdMap := map[string]int{ + "red": 1, + "orange": 2, + "yellow": 3, + "green": 4, + } + server.SetValidSubdocIdMap(validSubdocIdMap) + assert.DeepEqual(t, validSubdocIdMap, server.ValidSubdocIdMap()) } diff --git a/util/validator.go b/util/validator.go index 4ac602c..754044f 100644 --- a/util/validator.go +++ b/util/validator.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -88,7 +88,7 @@ func ValidatePokeQuery(values url.Values) (string, error) { return "primary", nil } -func ValidateQueryParams(r *http.Request, fields log.Fields) error { +func ValidateQueryParams(r *http.Request, validSubdocIdMap map[string]int, fields log.Fields) error { groupIdValues, ok := r.URL.Query()["group_id"] if !ok || len(groupIdValues) == 0 { return common.NewError(common.ErrInvalidQueryParams) @@ -106,7 +106,7 @@ func ValidateQueryParams(r *http.Request, fields log.Fields) error { } for _, subdocId := range subdocIds[1:] { - if _, ok := common.SubdocBitIndexMap[subdocId]; !ok { + if _, ok := validSubdocIdMap[subdocId]; !ok { return common.NewError(common.ErrInvalidQueryParams) } } diff --git a/util/validator_test.go b/util/validator_test.go index 9109b82..4313f03 100644 --- a/util/validator_test.go +++ b/util/validator_test.go @@ -14,12 +14,13 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( "errors" "fmt" + "maps" "net/http" "net/url" "testing" @@ -110,34 +111,37 @@ func TestValidatePokeQuery(t *testing.T) { func TestValidateQueryParams(t *testing.T) { cpeMac := GenerateRandomCpeMac() + validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) + validSubdocIdMap["red"] = 1 + validSubdocIdMap["orange"] = 1 // case 1 deviceConfigUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) req, err := http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) fields := make(log.Fields) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 2 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?foo=bar", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 3 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 4 deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root", cpeMac) req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 5 @@ -145,7 +149,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 6 @@ -153,7 +157,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 7 @@ -161,7 +165,7 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.Assert(t, errors.Is(err, common.ErrInvalidQueryParams)) // case 8 @@ -169,6 +173,14 @@ func TestValidateQueryParams(t *testing.T) { req, err = http.NewRequest("GET", deviceConfigUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456") - err = ValidateQueryParams(req, fields) + err = ValidateQueryParams(req, validSubdocIdMap, fields) + assert.NilError(t, err) + + // case 9 + deviceConfigUrl = fmt.Sprintf("/api/v1/device/%v/config?group_id=root,privatessid,homessid,red,orange", cpeMac) + req, err = http.NewRequest("GET", deviceConfigUrl, nil) + assert.NilError(t, err) + req.Header.Set(common.HeaderIfNoneMatch, "123,234,345,456,678") + err = ValidateQueryParams(req, validSubdocIdMap, fields) assert.NilError(t, err) } From 9a1c893a53b9fc636af1c29a5c04ecd2f70d7da0 Mon Sep 17 00:00:00 2001 From: James Chao Date: Sun, 17 Nov 2024 11:28:50 -0800 Subject: [PATCH 124/155] cherrypick the feature to validate token trust value and resolve conflicts --- common/error.go | 1 + config/sample_webconfig.conf | 3 ++ http/multipart_test.go | 29 +++++++++++++-- http/webconfig_server.go | 30 +++++++++++++--- http/webconfig_server_test.go | 8 +++++ security/main_test.go | 6 ++-- security/token.go | 68 +++++++++++++++++++---------------- security/token_test.go | 22 +++++++----- 8 files changed, 119 insertions(+), 48 deletions(-) diff --git a/common/error.go b/common/error.go index fa22ddf..9a1f4d1 100644 --- a/common/error.go +++ b/common/error.go @@ -34,6 +34,7 @@ var ( ErrNotOK = fmt.Errorf("!ok") ErrRootDocumentLocked = fmt.Errorf("root document is locked") ErrInvalidQueryParams = fmt.Errorf("invalid query parameters") + ErrLowTrust = fmt.Errorf("token trust is lower than threshold") ) type Http400Error struct { diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 013f180..0d0b4ed 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -262,4 +262,7 @@ webconfig { // subdoc ids accepted by the validator even if they are without bitmap definition valid_subdoc_ids = [] + + // only devices using tokens with trust higher or equal can GET the full document + min_trust = 0 } diff --git a/http/multipart_test.go b/http/multipart_test.go index ce3b84d..3d26e81 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -253,7 +253,7 @@ func TestCpeMiddleware(t *testing.T) { req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) res = ExecuteRequest(req, router1).Result() - rbytes, err = io.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusForbidden) @@ -264,7 +264,32 @@ func TestCpeMiddleware(t *testing.T) { assert.NilError(t, err) req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", token)) res = ExecuteRequest(req, router1).Result() - rbytes, err = io.ReadAll(res.Body) + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + // change the min trust to 1000 + server1.SetMinTrust(1000) + assert.Equal(t, 1000, server1.MinTrust()) + zeroToken := server1.Generate(cpeMac, 86400, 0) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", zeroToken)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusForbidden) + + // change the min trust back to 0 + server1.SetMinTrust(0) + assert.Equal(t, 0, server1.MinTrust()) + req, err = http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %v", zeroToken)) + res = ExecuteRequest(req, router1).Result() + _, err = io.ReadAll(res.Body) assert.NilError(t, err) res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusOK) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 7397acd..01b4ed0 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -121,6 +121,7 @@ type WebconfigServer struct { kafkaProducerTopic string queryParamsValidationEnabled bool validSubdocIdMap map[string]int + minTrust int } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -205,12 +206,14 @@ func GetDatabaseClient(sc *common.ServerConfig) db.DatabaseClient { func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer { conf := sc.Config var dbclient db.DatabaseClient + var tokenManager *security.TokenManager // setup up database client if testOnly { dbclient = GetTestDatabaseClient(sc) } else { dbclient = GetDatabaseClient(sc) + tokenManager = security.NewTokenManager(conf) } // setup jwks manager @@ -293,6 +296,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer for _, x := range validSubdocIds { validSubdocIdMap[x] = 1 } + minTrust := int(conf.GetInt32("webconfig.min_trust")) ws := &WebconfigServer{ Server: &http.Server{ @@ -301,7 +305,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer WriteTimeout: time.Duration(conf.GetInt32("webconfig.server.write_timeout_in_secs", 3)) * time.Second, }, DatabaseClient: dbclient, - TokenManager: security.NewTokenManager(conf), + TokenManager: tokenManager, JwksManager: jwksManager, ServerConfig: sc, WebpaConnector: NewWebpaConnector(conf, tlsConfig), @@ -330,6 +334,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer kafkaProducerTopic: kafkaProducerTopic, queryParamsValidationEnabled: queryParamsValidationEnabled, validSubdocIdMap: validSubdocIdMap, + minTrust: minTrust, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -395,14 +400,21 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { return } - if ok, partnerId, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true + fields := xw.Audit() + fields["src_partner"] = partnerId + fields["trust"] = trust + if err := s.ValidatePartner(partnerId); err != nil { - fields := xw.Audit() - fields["src_partner"] = partnerId + // isValid = false partnerId = "unknown" tokenErr = common.NewError(err) } + if trust < s.MinTrust() { + isValid = false + tokenErr = common.NewError(common.ErrLowTrust) + } xw.SetPartnerId(partnerId) } else { tokenErr = common.NewError(err) @@ -480,7 +492,7 @@ func (s *WebconfigServer) TestingCpeMiddleware(next http.Handler) http.Handler { return } - if ok, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { + if ok, _, _, _ := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true } } @@ -662,6 +674,14 @@ func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { s.validSubdocIdMap = x } +func (s *WebconfigServer) MinTrust() int { + return s.minTrust +} + +func (s *WebconfigServer) SetMinTrust(trust int) { + s.minTrust = trust +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 74ee523..37a7a5f 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -114,4 +114,12 @@ func TestWebconfigServerSetterGetter(t *testing.T) { } server.SetValidSubdocIdMap(validSubdocIdMap) assert.DeepEqual(t, validSubdocIdMap, server.ValidSubdocIdMap()) + + //configure trust level + trust := 1000 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) + trust = 500 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) } diff --git a/security/main_test.go b/security/main_test.go index b1223c8..d822e54 100644 --- a/security/main_test.go +++ b/security/main_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -39,7 +39,9 @@ func TestMain(m *testing.M) { NewTestCodec(sc.Config) - tokenManager = NewTokenManager(sc.Config) + if sc.Config.GetBoolean("webconfig.jwt.enabled", false) || os.Getenv("TOKEN_TEST") == "1" { + tokenManager = NewTokenManager(sc.Config) + } log.SetOutput(io.Discard) returnCode := m.Run() diff --git a/security/token.go b/security/token.go index 169cd0f..47270a4 100644 --- a/security/token.go +++ b/security/token.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -26,11 +26,11 @@ import ( "strings" "time" - "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/util" "github.com/go-akka/configuration" "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/util" ) const ( @@ -42,13 +42,13 @@ type ThemisClaims struct { Mac string `json:"mac"` PartnerId string `json:"partner-id"` Serial string `json:"serial"` - Trust string `json:"trust"` + Trust int `json:"trust"` Uuid string `json:"uuid"` Capabilities []string `json:"capabilities"` jwt.RegisteredClaims } -type VerifyFunc func(map[string]*rsa.PublicKey, []string, []string, ...string) (bool, string, error) +type VerifyFunc func(map[string]*rsa.PublicKey, []string, []string, ...string) (bool, string, int, error) type TokenManager struct { encodeKey *rsa.PrivateKey @@ -61,11 +61,6 @@ type TokenManager struct { } func NewTokenManager(conf *configuration.Config) *TokenManager { - jwtEnabled := conf.GetBoolean("webconfig.jwt.enabled", false) - if !jwtEnabled { - return nil - } - panicExitEnabled := conf.GetBoolean("webconfig.panic_exit_enabled", false) // prepare args for TokenManager @@ -134,19 +129,25 @@ func loadEncodeKey(keyfile string) (*rsa.PrivateKey, error) { } // TODO this is not an officially supported function. -func (m *TokenManager) Generate(mac string, ttl int64, vargs ...string) string { +func (m *TokenManager) Generate(mac string, ttl int64, itfs ...interface{}) string { // %% NOTE mac should be lowercase to be consistent with reference doc // static themis fields copied from examples in the webconfig confluence kid := "webconfig_key" serial := "ABCNDGE" - trust := "1000" + trust := 1000 capUuid := "1234567891234" capabilities := []string{"x1:issuer:test:.*:all"} - partner := "comcast" - if len(vargs) > 0 { - partner = vargs[0] + + for _, itf := range itfs { + switch ty := itf.(type) { + case string: + partner = ty + case int: + trust = ty + } } + utcnow := time.Now() claims := ThemisClaims{ @@ -224,19 +225,19 @@ func ParseKidFromTokenHeader(tokenString string) (string, error) { } func (m *TokenManager) VerifyApiToken(token string) (bool, error) { - ok, _, err := m.verifyFn(m.decodeKeys, m.apiKids, m.apiCapabilities, token) + ok, _, _, err := m.verifyFn(m.decodeKeys, m.apiKids, m.apiCapabilities, token) if err != nil { return ok, common.NewError(err) } return ok, err } -func (m *TokenManager) VerifyCpeToken(token string, mac string) (bool, string, error) { - ok, partner, err := m.verifyFn(m.decodeKeys, m.cpeKids, m.cpeCapabilities, token, mac) +func (m *TokenManager) VerifyCpeToken(token string, mac string) (bool, string, int, error) { + ok, partner, trust, err := m.verifyFn(m.decodeKeys, m.cpeKids, m.cpeCapabilities, token, mac) if err != nil { - return ok, "", common.NewError(err) + return ok, "", trust, common.NewError(err) } - return ok, partner, nil + return ok, partner, trust, nil } func (m *TokenManager) SetVerifyFunc(fn VerifyFunc) { @@ -293,9 +294,10 @@ func (m *TokenManager) ParseCpeToken(tokenStr string) (map[string]string, error) return data, nil } -func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, error) { +func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requiredCapabilities []string, vargs ...string) (bool, string, int, error) { tokenString := vargs[0] var kid string + var trust int parser := &jwt.Parser{} @@ -305,11 +307,11 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi // check kid rawkid, ok := token.Header["kid"] if !ok { - return false, "", common.NewError(fmt.Errorf("missing kid in token")) + return false, "", trust, common.NewError(fmt.Errorf("missing kid in token")) } kid, ok = rawkid.(string) if !ok { - return false, "", common.NewError(fmt.Errorf("error in reading kid from header")) + return false, "", trust, common.NewError(fmt.Errorf("error in reading kid from header")) } ok = false @@ -320,7 +322,7 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } if !ok { - return false, "", common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) + return false, "", trust, common.NewError(fmt.Errorf("token kid=%v, not in validKids=%v", kid, validKids)) } // check capabilities, if requiredCapabilities is nonempty @@ -343,21 +345,21 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi } } if !isCapable { - return false, "", common.NewError(fmt.Errorf("token without proper capabilities")) + return false, "", trust, common.NewError(fmt.Errorf("token without proper capabilities")) } } } else { - return false, "", common.NewError(err) + return false, "", trust, common.NewError(err) } decodeKey, ok := decodeKeys[kid] if !ok { - return false, "", common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) + return false, "", trust, common.NewError(fmt.Errorf("key object missing, kid=%v", kid)) } claims := jwt.MapClaims{} if _, err := jwt.ParseWithClaims(tokenString, claims, func(token *jwt.Token) (interface{}, error) { return decodeKey, nil }); err != nil { - return false, "", common.NewError(err) + return false, "", trust, common.NewError(err) } if len(vargs) > 1 { @@ -369,12 +371,12 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi if strings.ToLower(mac) == strings.ToLower(macstr) { isMatched = true } else { - return false, "", common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match mac in claims(%v)", mac, macstr)) } } } if !isMatched { - return false, "", common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) + return false, "", trust, common.NewError(fmt.Errorf("mac in token(%v) does not match claims=%v", mac, claims)) } } @@ -384,5 +386,9 @@ func VerifyToken(decodeKeys map[string]*rsa.PublicKey, validKids []string, requi partner = itf.(string) } - return true, partner, nil + if itf, ok := claims["trust"]; ok { + trust = util.ToInt(itf) + } + + return true, partner, trust, nil } diff --git a/security/token_test.go b/security/token_test.go index 5f6bab5..6f73aa6 100644 --- a/security/token_test.go +++ b/security/token_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package security import ( @@ -55,11 +55,7 @@ func TestLoadingKeyFiles(t *testing.T) { } func TestTokenValidation(t *testing.T) { - sc, err := common.GetTestServerConfig() - if err != nil { - panic(err) - } - if !sc.GetBoolean("webconfig.jwt.enabled") { + if tokenManager == nil { t.Skip("webconfig.jwt.enabled = false") } @@ -67,16 +63,26 @@ func TestTokenValidation(t *testing.T) { token := tokenManager.Generate(strings.ToLower(cpeMac), 86400) // default comcast - ok, parsedPartner, err := tokenManager.VerifyCpeToken(token, cpeMac) + ok, parsedPartner, trust, err := tokenManager.VerifyCpeToken(token, cpeMac) assert.NilError(t, err) assert.Assert(t, ok) assert.Equal(t, parsedPartner, "comcast") + assert.Equal(t, trust, 1000) // create a partner token partner1 := "cox" token1 := tokenManager.Generate(strings.ToLower(cpeMac), 86400, partner1) - ok, parsedPartner, err = tokenManager.VerifyCpeToken(token1, cpeMac) + ok, parsedPartner, trust, err = tokenManager.VerifyCpeToken(token1, cpeMac) + assert.NilError(t, err) + assert.Assert(t, ok) + assert.Equal(t, parsedPartner, partner1) + assert.Equal(t, trust, 1000) + + // create a partner token with non-default trust + token2 := tokenManager.Generate(strings.ToLower(cpeMac), 86400, partner1, 500) + ok, parsedPartner, trust, err = tokenManager.VerifyCpeToken(token2, cpeMac) assert.NilError(t, err) assert.Assert(t, ok) assert.Equal(t, parsedPartner, partner1) + assert.Equal(t, trust, 500) } From 969249e182a5949784e3bc4ae8379babbf4f7e61 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 26 Nov 2024 15:55:42 -0800 Subject: [PATCH 125/155] log details for token errors --- http/webconfig_server.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 4cb161a..9b4cb53 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -393,6 +393,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + fields := xw.Audit() authorization := r.Header.Get("Authorization") var tokenErr error if len(token) > 0 { @@ -405,7 +406,6 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true - fields := xw.Audit() fields["src_partner"] = partnerId fields["trust"] = trust @@ -426,6 +426,10 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { tokenErr = common.NewError(errors.New("CpeMiddleware() error no token")) } + if tokenErr != nil { + fields["error"] = tokenErr + } + if isValid { next.ServeHTTP(xw, r) } else { From 8200e8d3c829d1cae5f87de74dcb7a65ca988a42 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 26 Nov 2024 15:55:42 -0800 Subject: [PATCH 126/155] log details for token errors --- http/webconfig_server.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 01b4ed0..dcd7e1e 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -390,6 +390,7 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { isValid := false token := xw.Token() + fields := xw.Audit() authorization := r.Header.Get("Authorization") var tokenErr error if len(token) > 0 { @@ -402,7 +403,6 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { if ok, partnerId, trust, err := s.VerifyCpeToken(token, strings.ToLower(mac)); ok { isValid = true - fields := xw.Audit() fields["src_partner"] = partnerId fields["trust"] = trust @@ -423,6 +423,10 @@ func (s *WebconfigServer) CpeMiddleware(next http.Handler) http.Handler { tokenErr = common.NewError(errors.New("CpeMiddleware() error no token")) } + if tokenErr != nil { + fields["error"] = tokenErr + } + if isValid { next.ServeHTTP(xw, r) } else { From 884fff2d8bbac6f697fd024edc9894c29661b049 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 13 Nov 2024 17:32:40 -0800 Subject: [PATCH 127/155] cherrypick upstream profiles and merge conflicts --- config/sample_webconfig.conf | 9 +- http/supplementary_handler.go | 75 +++++++-- http/supplementary_handler_test.go | 241 ++++++++++++++++++++++++++++- http/upstream_connector.go | 37 ++++- http/webconfig_server.go | 33 ++-- http/webconfig_server_test.go | 23 ++- util/string.go | 43 ++++- util/string_test.go | 17 +- 8 files changed, 439 insertions(+), 39 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 0d0b4ed..6d2b232 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -257,12 +257,15 @@ webconfig { // this allows the root document locked if needed lock_root_document_enabled = false + // get extra profiles from upstream + upstream_profiles_enabled = false + // only valid query_params formats are accepted query_params_validation_enabled = false - // subdoc ids accepted by the validator even if they are without bitmap definition - valid_subdoc_ids = [] - // only devices using tokens with trust higher or equal can GET the full document min_trust = 0 + + // subdoc ids accepted by the validator even if they are without bitmap definition + valid_subdoc_ids = [] } diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 82f0743..20dc8fd 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -29,6 +29,10 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + notFoundProfileText = `{"profiles":[]}` +) + func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r *http.Request) { // ==== data integrity check ==== params := mux.Vars(r) @@ -50,18 +54,17 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r } // append the extra query_params if any + var rootdoc *common.RootDocument var queryParams string - if s.SupplementaryAppendingEnabled() { - rootdoc, err := s.GetRootDocument(mac) + var err error + if s.SupplementaryAppendingEnabled() || s.UpstreamProfilesEnabled() { + rootdoc, err = s.GetRootDocument(mac) if err != nil { if !s.IsDbNotFound(err) { Error(w, http.StatusInternalServerError, common.NewError(err)) return } } - if rootdoc != nil { - queryParams = rootdoc.QueryParams - } } // partner handling @@ -70,21 +73,73 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r partnerId = "" } + if s.SupplementaryAppendingEnabled() && rootdoc != nil { + queryParams = rootdoc.QueryParams + } + urlSuffix := util.GetTelemetryQueryString(r.Header, mac, queryParams, partnerId) fields["is_telemetry"] = true - rbytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + baseProfileBytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + xconfNotFound := false if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { - Error(w, rherr.StatusCode, rherr) + if rherr.StatusCode == http.StatusNotFound { + if s.UpstreamProfilesEnabled() { + xconfNotFound = true + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } + if !xconfNotFound { + Error(w, http.StatusInternalServerError, common.NewError(err)) return } - Error(w, http.StatusInternalServerError, common.NewError(err)) - return } - mpart, err := util.TelemetryBytesToMultipart(rbytes) + var profileBytes []byte + if s.UpstreamProfilesEnabled() && rootdoc != nil && len(rootdoc.QueryParams) > 0 { + // Get profiles from the second source + extraProfileBytes, _, err := s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) + if err != nil { + exitNow := true + var rherr common.RemoteHttpError + if errors.As(err, &rherr) { + if rherr.StatusCode == http.StatusNotFound { + exitNow = false + extraProfileBytes = nil + } else { + Error(w, rherr.StatusCode, rherr) + return + } + } + if exitNow { + Error(w, http.StatusInternalServerError, common.NewError(err)) + return + } + } + + if xconfNotFound { + baseProfileBytes = []byte(notFoundProfileText) + } + + // append profiles stored at webconfig + profileBytes, err = util.AppendProfiles(baseProfileBytes, extraProfileBytes) + if err != nil { + Error(w, http.StatusInternalServerError, err) + return + } + } else { + profileBytes = baseProfileBytes + } + + mpart, err := util.TelemetryBytesToMultipart(profileBytes) if err != nil { Error(w, http.StatusInternalServerError, common.NewError(err)) return diff --git a/http/supplementary_handler_test.go b/http/supplementary_handler_test.go index 54e2642..1328b9c 100644 --- a/http/supplementary_handler_test.go +++ b/http/supplementary_handler_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -890,3 +890,242 @@ func TestSupplementaryXconfReadAllErr(t *testing.T) { res.Body.Close() assert.Equal(t, res.StatusCode, http.StatusGatewayTimeout) } + +var ( + mockUpstreamProfileResponse1 = []byte(`[ + { + "name": "subname1", + "value": { + "Parameter": [ + { + "reference": "Device.X_RDK_GatewayManagement.Gateway.1.MacAddress", + "reportEmpty": true, + "type": "dataModel" + } + ] + }, + "versionHash": "977e16c4" + }, + { + "name": "subname2", + "value": { + "Parameter": [ + { + "reference": "Device.X_RDK_GatewayManagement.Gateway.2.MacAddress", + "reportEmpty": true, + "type": "dataModel" + } + ] + }, + "versionHash": "4f207ebd" + } +]`) +) + +func TestSupplementaryUpstreamProfiles(t *testing.T) { + log.SetOutput(io.Discard) + + server := NewWebconfigServer(sc, true) + router := server.GetRouter(true) + + // ==== step 1 setup mock xconf server ==== + cxbytes, err := util.CompactJson([]byte(mockProfileResponse)) + assert.NilError(t, err) + mockServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(cxbytes) + })) + defer mockServer.Close() + server.XconfConnector.SetXconfHost(mockServer.URL) + + // ==== step 2 set up upstream mock server ==== + cubytes, err := util.CompactJson(mockUpstreamProfileResponse1) + assert.NilError(t, err) + mockUpstreamServer := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(cubytes) + })) + defer mockUpstreamServer.Close() + server.SetUpstreamHost(mockUpstreamServer.URL) + targetUpstreamHost := server.UpstreamHost() + assert.Equal(t, mockUpstreamServer.URL, targetUpstreamHost) + + // ==== step 3 verify /config expect 200 with 1 mpart ==== + cpeMac := util.GenerateRandomCpeMac() + configUrl := fmt.Sprintf("/api/v1/device/%v/config", cpeMac) + req, err := http.NewRequest("GET", configUrl, nil) + assert.NilError(t, err) + + // add headers + modelName := "TG1682G" + partnerID := "comcast" + firmwareVersion := "TG1682_3.14p9s6_PROD_sey" + + req.Header.Set(common.HeaderSupplementaryService, "telemetry") + req.Header.Set(common.HeaderProfileVersion, "2.0") + req.Header.Set(common.HeaderModelName, modelName) + req.Header.Set(common.HeaderPartnerID, partnerID) + req.Header.Set(common.HeaderAccountID, "1234567890") + req.Header.Set(common.HeaderFirmwareVersion, firmwareVersion) + + res := ExecuteRequest(req, router).Result() + rbytes, err := io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err := util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok := mparts["telemetry"] + assert.Assert(t, ok) + + output := common.TR181Output{} + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes := []byte(output.Parameters[0].Value) + + var itf util.Dict + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok := itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok := profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 1) + profile1Itf := profilesJs[0] + + profile1, ok := profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok := profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok := coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + var srcItf map[string]interface{} + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 4 enable the feature flag but no query_param expect 200 with 1 mpart ==== + server.SetUpstreamProfilesEnabled(true) + defer server.SetUpstreamProfilesEnabled(false) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["telemetry"] + assert.Assert(t, ok) + + output = *new(common.TR181Output) + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes = []byte(output.Parameters[0].Value) + + itf = make(util.Dict) + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok = itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok = profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 1) + profile1Itf = profilesJs[0] + profile1, ok = profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + coreProfile1Itf, ok = profile1["value"] + assert.Assert(t, ok) + coreProfile1, ok = coreProfile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + + srcItf = make(map[string]interface{}) + err = json.Unmarshal([]byte(rawProfileStr), &srcItf) + assert.NilError(t, err) + assert.DeepEqual(t, coreProfile1, srcItf) + + // ==== step 5 set query_param in the root_document expect 200 with 1 mpart ==== + rdoc := new(common.RootDocument) + rdoc.QueryParams = "key1=val1" + rdoc.ModelName = modelName + rdoc.PartnerId = partnerID + rdoc.FirmwareVersion = firmwareVersion + err = server.SetRootDocument(cpeMac, rdoc) + assert.NilError(t, err) + + res = ExecuteRequest(req, router).Result() + rbytes, err = io.ReadAll(res.Body) + assert.NilError(t, err) + res.Body.Close() + t.Logf("%s\n", rbytes) + assert.Equal(t, res.StatusCode, http.StatusOK) + + mparts, err = util.ParseMultipart(res.Header, rbytes) + assert.NilError(t, err) + assert.Equal(t, len(mparts), 1) + mpart, ok = mparts["telemetry"] + assert.Assert(t, ok) + + output = *new(common.TR181Output) + err = msgpack.Unmarshal(mpart.Bytes, &output) + assert.NilError(t, err) + assert.Equal(t, len(output.Parameters), 1) + assert.Equal(t, output.Parameters[0].Name, common.TR181NameTelemetry) + assert.Equal(t, output.Parameters[0].DataType, common.TR181Blob) + mbytes = []byte(output.Parameters[0].Value) + + itf = make(util.Dict) + err = msgpack.Unmarshal(mbytes, &itf) + assert.NilError(t, err) + + _, err = json.Marshal(&itf) + assert.NilError(t, err) + + // assume only 1 "profile" is returned + profilesItf, ok = itf["profiles"] + assert.Assert(t, ok) + profilesJs, ok = profilesItf.([]interface{}) + assert.Assert(t, ok) + assert.Assert(t, len(profilesJs) == 3) + profile1Itf = profilesJs[0] + profile1, ok = profile1Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile1["name"].(string), "x_test_profile_001") + + profile2Itf := profilesJs[1] + profile2, ok := profile2Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile2["name"].(string), "subname1") + + profile3Itf := profilesJs[2] + profile3, ok := profile3Itf.(map[string]interface{}) + assert.Assert(t, ok) + assert.Equal(t, profile3["name"].(string), "subname2") +} diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 165aac6..d006bc5 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -22,15 +22,16 @@ import ( "fmt" "net/http" + "github.com/go-akka/configuration" "github.com/rdkcentral/webconfig/common" owcommon "github.com/rdkcentral/webconfig/common" - "github.com/go-akka/configuration" log "github.com/sirupsen/logrus" ) const ( upstreamHostDefault = "http://localhost:1234" - upstreamUrlTemplateDefault = "/api/v1/device/%v/upstream" + defaultUpstreamUrlTemplate = "/api/v1/device/%v/upstream" + defaultProfileUrlTemplate = "/api/v1/device/%v/profile?%v" ) type UpstreamConnector struct { @@ -38,6 +39,7 @@ type UpstreamConnector struct { host string serviceName string upstreamUrlTemplate string + profileUrlTemplate string } func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *UpstreamConnector { @@ -45,13 +47,16 @@ func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *Up confKey := fmt.Sprintf("webconfig.%v.host", serviceName) host := conf.GetString(confKey, upstreamHostDefault) confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) - upstreamUrlTemplate := conf.GetString(confKey, upstreamUrlTemplateDefault) + upstreamUrlTemplate := conf.GetString(confKey, defaultUpstreamUrlTemplate) + confKey = fmt.Sprintf("webconfig.%v.profile_url_template", serviceName) + profileUrlTemplate := conf.GetString(confKey, defaultProfileUrlTemplate) return &UpstreamConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, upstreamUrlTemplate: upstreamUrlTemplate, + profileUrlTemplate: profileUrlTemplate, } } @@ -90,3 +95,27 @@ func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes } return rbytes, header, nil } + +func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { + url := c.UpstreamHost() + fmt.Sprintf(c.profileUrlTemplate, mac, queryParams) + + if itf, ok := fields["audit_id"]; ok { + auditId := itf.(string) + if len(auditId) > 0 { + header.Set(common.HeaderAuditid, auditId) + } + } + + if itf, ok := fields["app_name"]; ok { + appName := itf.(string) + if len(appName) > 0 { + header.Set(common.HeaderSourceAppName, appName) + } + } + + rbytes, header, err := c.DoWithRetries("GET", url, header, nil, fields, c.ServiceName()) + if err != nil { + return rbytes, header, owcommon.NewError(err) + } + return rbytes, header, nil +} diff --git a/http/webconfig_server.go b/http/webconfig_server.go index dcd7e1e..9b4cb53 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -119,9 +119,10 @@ type WebconfigServer struct { webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string + upstreamProfilesEnabled bool queryParamsValidationEnabled bool - validSubdocIdMap map[string]int minTrust int + validSubdocIdMap map[string]int } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -290,13 +291,14 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } } + upstreamProfilesEnabled := conf.GetBoolean("webconfig.upstream_profiles_enabled") queryParamsValidationEnabled := conf.GetBoolean("webconfig.query_params_validation_enabled") + minTrust := int(conf.GetInt32("webconfig.min_trust")) validSubdocIds := conf.GetStringList("webconfig.valid_subdoc_ids") validSubdocIdMap := maps.Clone(common.SubdocBitIndexMap) for _, x := range validSubdocIds { validSubdocIdMap[x] = 1 } - minTrust := int(conf.GetInt32("webconfig.min_trust")) ws := &WebconfigServer{ Server: &http.Server{ @@ -332,9 +334,10 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, + upstreamProfilesEnabled: upstreamProfilesEnabled, queryParamsValidationEnabled: queryParamsValidationEnabled, - validSubdocIdMap: validSubdocIdMap, minTrust: minTrust, + validSubdocIdMap: validSubdocIdMap, } // Init the child poke span name ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() @@ -662,20 +665,20 @@ func (s *WebconfigServer) SetKafkaProducerTopic(x string) { s.kafkaProducerTopic = x } -func (s *WebconfigServer) QueryParamsValidationEnabled() bool { - return s.queryParamsValidationEnabled +func (s *WebconfigServer) UpstreamProfilesEnabled() bool { + return s.upstreamProfilesEnabled } -func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { - s.queryParamsValidationEnabled = enabled +func (s *WebconfigServer) SetUpstreamProfilesEnabled(enabled bool) { + s.upstreamProfilesEnabled = enabled } -func (s *WebconfigServer) ValidSubdocIdMap() map[string]int { - return s.validSubdocIdMap +func (s *WebconfigServer) QueryParamsValidationEnabled() bool { + return s.queryParamsValidationEnabled } -func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { - s.validSubdocIdMap = x +func (s *WebconfigServer) SetQueryParamsValidationEnabled(enabled bool) { + s.queryParamsValidationEnabled = enabled } func (s *WebconfigServer) MinTrust() int { @@ -686,6 +689,14 @@ func (s *WebconfigServer) SetMinTrust(trust int) { s.minTrust = trust } +func (s *WebconfigServer) ValidSubdocIdMap() map[string]int { + return s.validSubdocIdMap +} + +func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { + s.validSubdocIdMap = x +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 37a7a5f..0c028aa 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -98,6 +98,14 @@ func TestWebconfigServerSetterGetter(t *testing.T) { server.SetValidPartners(validPartners) assert.DeepEqual(t, server.ValidPartners(), validPartners) + // get profiles from upstream + enabled = true + server.SetUpstreamProfilesEnabled(enabled) + assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) + enabled = false + server.SetUpstreamProfilesEnabled(enabled) + assert.Equal(t, server.UpstreamProfilesEnabled(), enabled) + // enforce strict query parameters validation enabled = true server.SetQueryParamsValidationEnabled(enabled) @@ -106,6 +114,14 @@ func TestWebconfigServerSetterGetter(t *testing.T) { server.SetQueryParamsValidationEnabled(enabled) assert.Equal(t, server.QueryParamsValidationEnabled(), enabled) + //configure trust level + trust := 1000 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) + trust = 500 + server.SetMinTrust(trust) + assert.Equal(t, server.MinTrust(), trust) + validSubdocIdMap := map[string]int{ "red": 1, "orange": 2, @@ -115,11 +131,4 @@ func TestWebconfigServerSetterGetter(t *testing.T) { server.SetValidSubdocIdMap(validSubdocIdMap) assert.DeepEqual(t, validSubdocIdMap, server.ValidSubdocIdMap()) - //configure trust level - trust := 1000 - server.SetMinTrust(trust) - assert.Equal(t, server.MinTrust(), trust) - trust = 500 - server.SetMinTrust(trust) - assert.Equal(t, server.MinTrust(), trust) } diff --git a/util/string.go b/util/string.go index bd1eac4..064cc8e 100644 --- a/util/string.go +++ b/util/string.go @@ -14,18 +14,26 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( + "encoding/json" "fmt" "net/http" "net/url" "strconv" "strings" - "github.com/rdkcentral/webconfig/common" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" +) + +const ( + Comma = ',' + RightSquareBracket = ']' + RightCurlBracket = '}' + baseProfileStr = `{"profiles":[` ) var ( @@ -123,3 +131,34 @@ func IsValidUTF8(bbytes []byte) bool { str2 := strings.ToValidUTF8(str1, "#") return str1 == str2 } + +func AppendProfiles(pbytes, sbytes []byte) ([]byte, error) { + var builder strings.Builder + if len(pbytes) < 3 { + builder.WriteString(baseProfileStr) + } else { + s := strings.TrimSpace(string(pbytes)) + builder.WriteString(s[:len(s)-2]) + } + if len(sbytes) > 2 { + if len(pbytes) > 20 { + builder.WriteRune(',') + } + builder.Write(sbytes[1 : len(sbytes)-1]) + } + builder.WriteRune(RightSquareBracket) + builder.WriteRune(RightCurlBracket) + return []byte(builder.String()), nil +} + +func CompactJson(sbytes []byte) ([]byte, error) { + var itf interface{} + if err := json.Unmarshal(sbytes, &itf); err != nil { + return nil, common.NewError(err) + } + jsbytes, err := json.Marshal(itf) + if err != nil { + return nil, common.NewError(err) + } + return jsbytes, nil +} diff --git a/util/string_test.go b/util/string_test.go index 8fb5288..3c6df1a 100644 --- a/util/string_test.go +++ b/util/string_test.go @@ -14,10 +14,11 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( + "encoding/json" "net/http" "testing" @@ -79,3 +80,17 @@ func TestTelemetryQueryWithWanMac(t *testing.T) { expected := "env=PROD&partnerId=comcast&version=2.0&model=TG1682G&accountId=1234567890&firmwareVersion=TG1682_3.14p9s6_PROD_sey&estbMacAddress=567890ABCDEF" assert.Equal(t, qstr, expected) } + +func TestAppendProfiles(t *testing.T) { + mockedBaseProfilesResponse := `{"profiles":[{"name":"XfinityWIFI_SYNC","value":{"Description":"XfinityWIFI_SYNC to capture XWIFI info every 12 hours","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"},{"Name":"reportVersion","Reference":"Profile.Version"}],"URL":"https://stbrtl.stb.r53.xcal.tv"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"reference":"Profile.Name","type":"dataModel"},{"reference":"Profile.Version","type":"dataModel"},{"name":"Profile","reference":"Device.DeviceInfo.X_RDK_RDKProfileName","type":"dataModel"},{"name":"Time","reference":"Device.Time.X_RDK_CurrentUTCTime","type":"dataModel"},{"name":"mac","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_MAC","type":"dataModel"},{"name":"CMMAC_split","reference":"Device.DeviceInfo.X_COMCAST-COM_CM_MAC","type":"dataModel"},{"name":"erouterIpv4","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IP","type":"dataModel"},{"name":"erouterIpv6","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IPv6","type":"dataModel"},{"name":"PartnerId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_Syndication.PartnerId","type":"dataModel"},{"name":"Version","reference":"Device.DeviceInfo.SoftwareVersion","type":"dataModel"},{"name":"AccountId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.AccountInfo.AccountID","type":"dataModel"},{"name":"cpe_passpoint_enable","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_InterworkingServiceEnable","type":"dataModel"},{"name":"cpe_passpoint_inter_parameters","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_InterworkingService.Parameters","type":"dataModel"},{"name":"cpe_passpoint_parameters","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_Passpoint.Parameters","type":"dataModel"},{"name":"cpe_passpoint_rdk_enable","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_Passpoint.Enable","type":"dataModel"},{"name":"open5_bss_active","reference":"Device.WiFi.SSID.6.Enable","type":"dataModel"},{"name":"secure5_bss_active","reference":"Device.WiFi.SSID.10.Enable","type":"dataModel"},{"name":"secure5_radius_server_ip","reference":"Device.WiFi.AccessPoint.10.Security.RadiusServerIPAddr","type":"dataModel"},{"name":"wifi_enabled","reference":"Device.DeviceInfo.X_COMCAST_COM_xfinitywifiEnable","type":"dataModel"},{"name":"primary_tunnel","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.PrimaryRemoteEndpoint","type":"dataModel"},{"name":"secondary_tunnel","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.SecondaryRemoteEndpoint","type":"dataModel"},{"name":"open5_advertisement_str","reference":"Device.WiFi.AccessPoint.6.SSIDAdvertisementEnabled","type":"dataModel"},{"name":"secure5_advertisement_str","reference":"Device.WiFi.AccessPoint.10.SSIDAdvertisementEnabled","type":"dataModel"},{"name":"device_vlan_2","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.Interface.2.VLANID","type":"dataModel"},{"name":"open5_ssid_str","reference":"Device.WiFi.SSID.6.SSID","type":"dataModel"},{"name":"secure5_ssid_str","reference":"Device.WiFi.SSID.10.SSID","type":"dataModel"},{"name":"secure5_securitymode_str","reference":"Device.WiFi.AccessPoint.10.Security.ModeEnabled","type":"dataModel"},{"name":"secure5_pri_port","reference":"Device.WiFi.AccessPoint.10.Security.RadiusServerPort","type":"dataModel"},{"name":"dscp_marker","reference":"Device.X_COMCAST-COM_GRE.Tunnel.1.DSCPMarkPolicy","type":"dataModel"},{"name":"WIFI_CH_2_split","reference":"Device.WiFi.Radio.2.Channel","type":"dataModel"},{"name":"WIFI_CW_2_split","reference":"Device.WiFi.Radio.2.OperatingChannelBandwidth","type":"dataModel"},{"name":"open5_bssid_str","reference":"Device.WiFi.SSID.6.BSSID","type":"dataModel"},{"name":"secure5_bssid_str","reference":"Device.WiFi.SSID.10.BSSID","type":"dataModel"},{"name":"open5_status_str","reference":"Device.WiFi.SSID.6.Status","type":"dataModel"},{"name":"secure5_status_str","reference":"Device.WiFi.SSID.10.Status","type":"dataModel"},{"name":"open5_beaconpower_str","reference":"Device.WiFi.AccessPoint.6.X_RDKCENTRAL-COM_ManagementFramePowerControl","type":"dataModel"},{"name":"open5_beaconrate_str","reference":"Device.WiFi.AccessPoint.6.X_RDKCENTRAL-COM_BeaconRate","type":"dataModel"},{"name":"secure5_beaconpower_str","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_ManagementFramePowerControl","type":"dataModel"},{"name":"secure5_beaconrate_str","reference":"Device.WiFi.AccessPoint.10.X_RDKCENTRAL-COM_BeaconRate","type":"dataModel"},{"name":"radio5_beaconinterval","reference":"Device.WiFi.Radio.2.X_COMCAST-COM_BeaconInterval","type":"dataModel"},{"name":"secure5_encryption","reference":"Device.WiFi.AccessPoint.10.Security.X_CISCO_COM_EncryptionMethod","type":"dataModel"},{"name":"secure5_sec_radius_server_ip","reference":"Device.WiFi.AccessPoint.10.Security.SecondaryRadiusServerIPAddr","type":"dataModel"},{"name":"secure5_sec_port","reference":"Device.WiFi.AccessPoint.10.Security.SecondaryRadiusServerPort","type":"dataModel"},{"name":"UPTIME_split","reference":"Device.DeviceInfo.UpTime","type":"dataModel"},{"name":"open5_radius_server_ip","reference":"Device.WiFi.AccessPoint.6.Security.RadiusServerIPAddr","type":"dataModel"},{"name":"open5_pri_port","reference":"Device.WiFi.AccessPoint.6.Security.RadiusServerPort","type":"dataModel"},{"name":"open5_isolation_enable","reference":"Device.WiFi.AccessPoint.6.IsolationEnable","type":"dataModel"},{"name":"secure5_isolation_enable","reference":"Device.WiFi.AccessPoint.10.IsolationEnable","type":"dataModel"},{"name":"secure24_bss_active","reference":"Device.WiFi.SSID.9.Enable","type":"dataModel"},{"name":"open24_bss_active","reference":"Device.WiFi.SSID.5.Enable","type":"dataModel"}],"Protocol":"HTTP","ReportingAdjustments":{"FirstReportingInterval":300},"ReportingInterval":86400,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.6"},"versionHash":"d9c6f386"},{"name":"WIFI_MOTION_Telemetry","value":{"Description":"CSCWFM_Telemetry","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"}],"URL":"https://stbrtl-oi.stb.r53.xcal.tv"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"name":"Profile_Name","reference":"Profile.Name","type":"dataModel"},{"name":"Profile","reference":"Device.DeviceInfo.X_RDK_RDKProfileName","type":"dataModel"},{"name":"Time","reference":"Device.Time.X_RDK_CurrentUTCTime","type":"dataModel"},{"name":"mac","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_MAC","type":"dataModel"},{"name":"CMMAC_split","reference":"Device.DeviceInfo.X_COMCAST-COM_CM_MAC","type":"dataModel"},{"name":"erouterIpv4","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IP","type":"dataModel"},{"name":"erouterIpv6","reference":"Device.DeviceInfo.X_COMCAST-COM_WAN_IPv6","type":"dataModel"},{"name":"PartnerId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_Syndication.PartnerId","type":"dataModel"},{"name":"Version","reference":"Device.DeviceInfo.SoftwareVersion","type":"dataModel"},{"name":"AccountId","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.AccountInfo.AccountID","type":"dataModel"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXMrbussub_fail","name":"SYS_ERROR_WFMrbussub_fail","type":"event","use":"count"},{"component":"CSCWFMBRG","eventName":"CSCWFM_CSIpipe_restart","name":"SYS_SH_WFMpipe_restart","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_ctrlifcreate_fail","name":"SYS_ERROR_WFMifcreate_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSC_RXMrbusinit_fail","name":"SYS_ERROR_WFM_rbusinit_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_CSIsessionacquire_fail","name":"SYS_ERROR_WFMSessionAcq_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXMeventscreate_fail","name":"SYS_ERROR_WFMeventscreate_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_CSIsessionenable_fail","name":"SYS_ERROR_WFMsessionenable_fail","type":"event","use":"count"},{"component":"CSCWFMRXM","eventName":"CSCWFM_RXM_restart","name":"SYS_SH_WFMRXM_restart","type":"event","use":"count"},{"component":"CSCWFM","eventName":"CSCWFM_borg_restart","name":"SYS_SH_WFMborg_restart","type":"event","use":"count"},{"component":"CSCWFM","eventName":"CSCWFM_mqtt_restart","name":"SYS_SH_WFMmqtt_restart","type":"event","use":"count"},{"component":"CSCWFMBRG","eventName":"CSCWFM_sounding_state","name":"WFMsoundingstate_split","type":"event","use":"absolute"},{"name":"WFMEnable_split","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_RFC.Feature.CognitiveMotionDetection.Enable","type":"dataModel"},{"name":"WFMStatus_split","reference":"Device.DeviceInfo.X_RDKCENTRAL-COM_XHFW.WiFiMotionStatus","type":"dataModel"},{"logFile":"ZilkerLog.txt","marker":"wfmCogAgentCommFailCnt_split","search":"wfmCogAgentCommFailCnt:","type":"grep","use":"count"},{"logFile":"ZilkerLog.txt","marker":"wfmCogAgentConnected_split","search":"wfmCogAgentConnected:","type":"grep","use":"absolute"}],"Protocol":"HTTP","ReportingInterval":900,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.1"},"versionHash":"bf86fd16"}]}` + mockedExtraProfilesResponse := `[{"name":"james_test_profile_001","value":{"ActivationTimeout":600,"Description":"Telemetry 2.0 HSD Gateway WiFi Radio","EncodingType":"JSON","HTTP":{"Compression":"None","Method":"POST","RequestURIParameter":[{"Name":"profileName","Reference":"Profile.Name"},{"Name":"reportVersion","Reference":"Profile.Version"}],"URL":"https://rdkrtldev.stb.r53.xcal.tv/"},"JSONEncoding":{"ReportFormat":"NameValuePair","ReportTimestamp":"None"},"Parameter":[{"reference":"Profile.Name","type":"dataModel"},{"reference":"Profile.Description","type":"dataModel"},{"reference":"Profile.Version","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.MaxBitRate","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.OperatingFrequencyBand","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.ChannelsInUse","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Channel","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.AutoChannelEnable","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.OperatingChannelBandwidth","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.RadioResetCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.ErrorsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.ErrorsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.DiscardPacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.DiscardPacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.PLCPErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.FCSErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_NoiseFloor","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.Noise","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_ChannelUtilization","type":"dataModel"},{"reference":"Device.WiFi.Radio.1.Stats.X_COMCAST-COM_ActivityFactor","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.MaxBitRate","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.OperatingFrequencyBand","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.ChannelsInUse","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Channel","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.AutoChannelEnable","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.OperatingChannelBandwidth","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.RadioResetCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.ErrorsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.ErrorsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.DiscardPacketsSent","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.DiscardPacketsReceived","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.PLCPErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.FCSErrorCount","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_NoiseFloor","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.Noise","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_ChannelUtilization","type":"dataModel"},{"reference":"Device.WiFi.Radio.2.Stats.X_COMCAST-COM_ActivityFactor","type":"dataModel"}],"Protocol":"HTTP","ReportingInterval":60,"TimeReference":"0001-01-01T00:00:00Z","Version":"0.1"},"versionHash":"ed0de6ef"}]` + + appendedBytes, err := AppendProfiles([]byte(mockedBaseProfilesResponse), []byte(mockedExtraProfilesResponse)) + assert.NilError(t, err) + + var itf interface{} + err = json.Unmarshal(appendedBytes, &itf) + assert.NilError(t, err) + _, err = json.MarshalIndent(itf, "", " ") + assert.NilError(t, err) +} From eb7872f79b776dd388617f6857e853df9d58b088 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 3 Dec 2024 22:16:26 -0800 Subject: [PATCH 128/155] Add support for poke root and telemetry together --- common/const_var.go | 2 +- util/validator.go | 50 +++++++++++++----------------------------- util/validator_test.go | 4 ++-- 3 files changed, 18 insertions(+), 38 deletions(-) diff --git a/common/const_var.go b/common/const_var.go index 8e56f84..b8b8ec3 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -119,7 +119,7 @@ const ( ) var ( - SupportedPokeDocs = []string{"primary", "telemetry"} + SupportedPokeDocs = []string{"primary", "telemetry", "root"} SupportedPokeRoutes = []string{"mqtt"} ) diff --git a/util/validator.go b/util/validator.go index 754044f..2bab589 100644 --- a/util/validator.go +++ b/util/validator.go @@ -21,6 +21,7 @@ import ( "fmt" "net/http" "net/url" + "slices" "strings" "github.com/rdkcentral/webconfig/common" @@ -41,50 +42,29 @@ func ValidateMac(mac string) bool { func ValidatePokeQuery(values url.Values) (string, error) { // handle ?doc=xxx - if docQueryParamStrs, ok := values["doc"]; ok { - if len(docQueryParamStrs) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(docQueryParamStrs[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple doc parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeDocs, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) - return "", common.NewError(err) - + docStr := values.Get("doc") + if len(docStr) > 0 { + docNames := strings.Split(docStr, ",") + for _, n := range docNames { + if !slices.Contains(common.SupportedPokeDocs, n) { + err := fmt.Errorf("invalid query parameter: %v", n) + return "", common.NewError(err) + } } - return queryStr, nil + return docStr, nil } // handle ?route=xxx - if qparams, ok := values["route"]; ok { - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - qparams := strings.Split(qparams[0], ",") - if len(qparams) > 1 { - err := fmt.Errorf("multiple route parameter is not allowed") - return "", common.NewError(err) - } - - queryStr := qparams[0] - if !Contains(common.SupportedPokeRoutes, queryStr) { - err := fmt.Errorf("invalid query parameter: %v", queryStr) + routeStr := values.Get("route") + if len(routeStr) > 0 { + if !slices.Contains(common.SupportedPokeRoutes, routeStr) { + err := fmt.Errorf("invalid query parameter: %v", routeStr) return "", common.NewError(err) } - return queryStr, nil + return routeStr, nil } - // return default return "primary", nil } diff --git a/util/validator_test.go b/util/validator_test.go index 4313f03..4cfb97a 100644 --- a/util/validator_test.go +++ b/util/validator_test.go @@ -56,8 +56,8 @@ func TestValidatePokeQuery(t *testing.T) { values := url.Values{} values["doc"] = []string{ - "primary,telemetry", "hello,world", + "primary,telemetry", } _, err := ValidatePokeQuery(values) assert.Assert(t, err != nil) @@ -72,7 +72,7 @@ func TestValidatePokeQuery(t *testing.T) { "primary,telemetry", } _, err = ValidatePokeQuery(values) - assert.Assert(t, err != nil) + assert.NilError(t, err) values["doc"] = []string{ "primary", From 7e32cdd615df583b944200de43d6092e15c27418 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 9 Dec 2024 16:17:13 -0800 Subject: [PATCH 129/155] set updated_time in the case of state correction --- db/service.go | 2 ++ http/multipart_test.go | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/db/service.go b/db/service.go index 22c9b9b..169f2a0 100644 --- a/db/service.go +++ b/db/service.go @@ -188,6 +188,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } } } + updatedTime := int(time.Now().UnixMilli()) for subdocId, subdocument := range document.Items() { cloudVersion := subdocument.GetVersion() cloudState := subdocument.GetState() @@ -205,6 +206,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // update state newState := common.Deployed subdocument.SetState(&newState) + subdocument.SetUpdatedTime(&updatedTime) if cloudErrorCode > 0 { var newErrorCode int subdocument.SetErrorCode(&newErrorCode) diff --git a/http/multipart_test.go b/http/multipart_test.go index 3d26e81..ca35726 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1121,20 +1121,24 @@ func TestStateCorrectionEnabled(t *testing.T) { lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") assert.NilError(t, err) assert.Equal(t, lanSubdocument.GetState(), common.PendingDownload) + oldLanUpdatedTime := *lanSubdocument.UpdatedTime() wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") assert.NilError(t, err) assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + oldWanUpdatedTime := *wanSubdocument.UpdatedTime() meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Failure) assert.Equal(t, *meshSubdocument.ErrorCode(), meshErrorCode) assert.Equal(t, *meshSubdocument.ErrorDetails(), meshErrorDetails) + oldMeshUpdatedTime := *meshSubdocument.UpdatedTime() mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + oldMocaUpdatedTime := *mocaSubdocument.UpdatedTime() // ==== enable the state correction flag and call GET /config again with if-none-match and expect 304 ==== server.SetStateCorrectionEnabled(true) @@ -1142,6 +1146,8 @@ func TestStateCorrectionEnabled(t *testing.T) { server.SetStateCorrectionEnabled(false) }() + time.Sleep(time.Duration(100) * time.Millisecond) + req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, header1) @@ -1155,20 +1161,24 @@ func TestStateCorrectionEnabled(t *testing.T) { lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") assert.NilError(t, err) assert.Equal(t, lanSubdocument.GetState(), common.Deployed) + assert.Assert(t, *lanSubdocument.UpdatedTime() > oldLanUpdatedTime) wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") assert.NilError(t, err) assert.Equal(t, wanSubdocument.GetState(), common.Deployed) + assert.Assert(t, *wanSubdocument.UpdatedTime() > oldWanUpdatedTime) meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Deployed) assert.Equal(t, *meshSubdocument.ErrorCode(), 0) assert.Equal(t, *meshSubdocument.ErrorDetails(), "") + assert.Assert(t, *meshSubdocument.UpdatedTime() > oldMeshUpdatedTime) mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + assert.Assert(t, *mocaSubdocument.UpdatedTime() == oldMocaUpdatedTime) } func TestCorruptedEncryptedDocumentHandler(t *testing.T) { From bb27227418720fe163f365b90a75dbbb281791cb Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 9 Dec 2024 16:17:13 -0800 Subject: [PATCH 130/155] set updated_time in the case of state correction --- db/service.go | 2 ++ http/multipart_test.go | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/db/service.go b/db/service.go index 22c9b9b..169f2a0 100644 --- a/db/service.go +++ b/db/service.go @@ -188,6 +188,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } } } + updatedTime := int(time.Now().UnixMilli()) for subdocId, subdocument := range document.Items() { cloudVersion := subdocument.GetVersion() cloudState := subdocument.GetState() @@ -205,6 +206,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel // update state newState := common.Deployed subdocument.SetState(&newState) + subdocument.SetUpdatedTime(&updatedTime) if cloudErrorCode > 0 { var newErrorCode int subdocument.SetErrorCode(&newErrorCode) diff --git a/http/multipart_test.go b/http/multipart_test.go index 3d26e81..ca35726 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -1121,20 +1121,24 @@ func TestStateCorrectionEnabled(t *testing.T) { lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") assert.NilError(t, err) assert.Equal(t, lanSubdocument.GetState(), common.PendingDownload) + oldLanUpdatedTime := *lanSubdocument.UpdatedTime() wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") assert.NilError(t, err) assert.Equal(t, wanSubdocument.GetState(), common.InDeployment) + oldWanUpdatedTime := *wanSubdocument.UpdatedTime() meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Failure) assert.Equal(t, *meshSubdocument.ErrorCode(), meshErrorCode) assert.Equal(t, *meshSubdocument.ErrorDetails(), meshErrorDetails) + oldMeshUpdatedTime := *meshSubdocument.UpdatedTime() mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + oldMocaUpdatedTime := *mocaSubdocument.UpdatedTime() // ==== enable the state correction flag and call GET /config again with if-none-match and expect 304 ==== server.SetStateCorrectionEnabled(true) @@ -1142,6 +1146,8 @@ func TestStateCorrectionEnabled(t *testing.T) { server.SetStateCorrectionEnabled(false) }() + time.Sleep(time.Duration(100) * time.Millisecond) + req, err = http.NewRequest("GET", configUrl, nil) assert.NilError(t, err) req.Header.Set(common.HeaderIfNoneMatch, header1) @@ -1155,20 +1161,24 @@ func TestStateCorrectionEnabled(t *testing.T) { lanSubdocument, err = server.GetSubDocument(cpeMac, "lan") assert.NilError(t, err) assert.Equal(t, lanSubdocument.GetState(), common.Deployed) + assert.Assert(t, *lanSubdocument.UpdatedTime() > oldLanUpdatedTime) wanSubdocument, err = server.GetSubDocument(cpeMac, "wan") assert.NilError(t, err) assert.Equal(t, wanSubdocument.GetState(), common.Deployed) + assert.Assert(t, *wanSubdocument.UpdatedTime() > oldWanUpdatedTime) meshSubdocument, err = server.GetSubDocument(cpeMac, "mesh") assert.NilError(t, err) assert.Equal(t, meshSubdocument.GetState(), common.Deployed) assert.Equal(t, *meshSubdocument.ErrorCode(), 0) assert.Equal(t, *meshSubdocument.ErrorDetails(), "") + assert.Assert(t, *meshSubdocument.UpdatedTime() > oldMeshUpdatedTime) mocaSubdocument, err = server.GetSubDocument(cpeMac, "moca") assert.NilError(t, err) assert.Equal(t, mocaSubdocument.GetState(), common.PendingDownload) + assert.Assert(t, *mocaSubdocument.UpdatedTime() == oldMocaUpdatedTime) } func TestCorruptedEncryptedDocumentHandler(t *testing.T) { From 6b6de0bd83517b02f7a6d723e2f473cf058b3468 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 11 Dec 2024 22:48:58 -0800 Subject: [PATCH 131/155] propagate headers for tracing --- common/log_fields.go | 4 +--- common/log_fields_test.go | 20 ++++++++--------- http/poke_handler.go | 6 +++--- http/webconfig_server.go | 45 ++++++++++++++------------------------- http/webpa_connector.go | 17 +++------------ 5 files changed, 32 insertions(+), 60 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index 5ce267b..c14a280 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -27,8 +27,6 @@ var ( unloggedFields = []string{ "moneytrace", "token", - "out_traceparent", - "out_tracestate", } coreFields = []string{ "app_name", diff --git a/common/log_fields_test.go b/common/log_fields_test.go index 960af77..e2298cd 100644 --- a/common/log_fields_test.go +++ b/common/log_fields_test.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package common import ( @@ -49,16 +49,14 @@ func TestFilterLogFields(t *testing.T) { assert.DeepEqual(t, expected, c2) src3 := log.Fields{ - "red": "maroon", - "orange": "auburn", - "yellow": "amber", - "green": "viridian", - "blue": "turquoise", - "indigo": "sapphire", - "violet": "purple", - "out_traceparent": "foo", - "out_tracestate": "cyan", - "token": "bar", + "red": "maroon", + "orange": "auburn", + "yellow": "amber", + "green": "viridian", + "blue": "turquoise", + "indigo": "sapphire", + "violet": "purple", + "token": "bar", } c3 := FilterLogFields(src3) assert.DeepEqual(t, src, c3) diff --git a/http/poke_handler.go b/http/poke_handler.go index 4552769..cc48d30 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -25,10 +25,10 @@ import ( "sort" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" "go.opentelemetry.io/otel/attribute" ) @@ -172,7 +172,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { span.End() }() - transactionId, err := s.Poke(mac, token, pokeStr, fields) + transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 9b4cb53..fffb233 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -712,9 +712,9 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return fmt.Errorf("invalid partner") } -func (c *WebconfigServer) Poke(cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { +func (c *WebconfigServer) Poke(rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) - transactionId, err := c.Patch(cpeMac, token, []byte(body), fields) + transactionId, err := c.Patch(rHeader, cpeMac, token, []byte(body), fields) if err != nil { return "", common.NewError(err) } @@ -742,7 +742,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques token = elements[1] } - var xmTraceId, traceId, outTraceparent, outTracestate string + var xmTraceId, traceId string // extract moneytrace from the header tracePart := strings.Split(r.Header.Get("X-Moneytrace"), ";")[0] @@ -756,14 +756,11 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques traceparent := r.Header.Get(common.HeaderTraceparent) if len(traceparent) == 55 { traceId = traceparent[3:35] - outTraceparent = traceparent[:36] + s.TraceparentParentID() + traceparent[52:55] } // extract tracestate from the header tracestate := r.Header.Get(common.HeaderTracestate) - if len(tracestate) > 0 { - outTracestate = fmt.Sprintf("%v,%v=%v", tracestate, s.TracestateVendorID(), s.TraceparentParentID()) - } + // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { @@ -771,17 +768,17 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } headerMap := util.HeaderToMap(header) fields := log.Fields{ - "path": r.URL.String(), - "method": r.Method, - "audit_id": auditId, - "remote_ip": remoteIp, - "host_name": host, - "header": headerMap, - "logger": "request", - "trace_id": traceId, - "app_name": s.AppName(), - "out_traceparent": outTraceparent, - "out_tracestate": outTracestate, + "path": r.URL.String(), + "method": r.Method, + "audit_id": auditId, + "remote_ip": remoteIp, + "host_name": host, + "header": headerMap, + "logger": "request", + "trace_id": traceId, + "app_name": s.AppName(), + "traceparent": traceparent, + "tracestate": tracestate, } userAgent := r.UserAgent() @@ -992,7 +989,7 @@ func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) sc = sc.WithTraceFlags(traceFlags) } - tracestateStr := s.getTracestate(r) + tracestateStr := r.Header.Get(common.HeaderTracestate) if tracestateStr != "" { tracestate, _ := trace.ParseTraceState(tracestateStr) sc = sc.WithTraceState(tracestate) @@ -1017,16 +1014,6 @@ func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, tra return } -// extract tracestate from the header -func (s *WebconfigServer) getTracestate(r *http.Request) string { - inTracestate := r.Header.Get(common.HeaderTracestate) - var outTracestate string - if len(inTracestate) > 0 { - outTracestate = fmt.Sprintf("%v,%v=%v", inTracestate, s.TracestateVendorID(), s.TraceparentParentID()) - } - return outTracestate -} - func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { var span trace.Span diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 52378d9..41bbf9b 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -195,10 +195,10 @@ func (c *WebpaConnector) SetAsyncPokeEnabled(enabled bool) { c.asyncPokeEnabled = enabled } -func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { +func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { url := fmt.Sprintf(webpaUrlTemplate, c.WebpaHost(), c.ApiVersion(), cpeMac) - var traceId, xmTraceId, outTraceparent, outTracestate string + var traceId, xmTraceId string if itf, ok := fields["trace_id"]; ok { traceId = itf.(string) } @@ -213,22 +213,11 @@ func (c *WebpaConnector) Patch(cpeMac string, token string, bbytes []byte, field if len(traceId) == 0 { traceId = xmTraceId } - if itf, ok := fields["out_traceparent"]; ok { - outTraceparent = itf.(string) - } - if itf, ok := fields["out_tracestate"]; ok { - outTracestate = itf.(string) - } t := time.Now().UnixNano() / 1000 transactionId := fmt.Sprintf("%s_____%015x", xmTraceId, t) - xmoney := fmt.Sprintf("trace-id=%s;parent-id=0;span-id=0;span-name=%s", xmTraceId, webpaServiceName) - header := make(http.Header) - header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + header := rHeader.Clone() header.Set("X-Webpa-Transaction-Id", transactionId) - header.Set("X-Moneytrace", xmoney) - header.Set(common.HeaderTraceparent, outTraceparent) - header.Set(common.HeaderTracestate, outTracestate) method := "PATCH" _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) From 9b693066e5e8ac6009b64451b365941f3a170bde Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 12 Dec 2024 16:51:29 -0800 Subject: [PATCH 132/155] remove otel span codes --- go.mod | 19 +--- go.sum | 36 +------ http/otel.go | 208 --------------------------------------- http/poke_handler.go | 18 ---- http/router.go | 3 +- http/webconfig_server.go | 105 -------------------- http/webpa_connector.go | 10 -- main.go | 1 - 8 files changed, 7 insertions(+), 393 deletions(-) delete mode 100644 http/otel.go diff --git a/go.mod b/go.mod index 95108f9..a7cd24a 100644 --- a/go.mod +++ b/go.mod @@ -17,11 +17,6 @@ require ( github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/otel v1.27.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 - go.opentelemetry.io/otel/sdk v1.27.0 - go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 golang.org/x/sync v0.6.0 @@ -31,18 +26,14 @@ require ( require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect - github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -53,24 +44,20 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/klauspost/compress v1.16.7 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/proto/otlp v1.2.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index 74c2010..8d8c331 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,6 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= -github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -64,6 +62,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -92,11 +91,6 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= @@ -167,8 +161,6 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -228,6 +220,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -264,6 +257,7 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -303,22 +297,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= -go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= -go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= @@ -478,8 +456,6 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -581,10 +557,6 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -597,8 +569,6 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/http/otel.go b/http/otel.go deleted file mode 100644 index 468035f..0000000 --- a/http/otel.go +++ /dev/null @@ -1,208 +0,0 @@ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 -*/ -package http - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" - "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/trace" - "go.opentelemetry.io/otel/trace/noop" - - "github.com/go-akka/configuration" - log "github.com/sirupsen/logrus" -) - -// Tracing contains the core dependencies to make tracing possible across an application. -type otelTracing struct { - providerName string - envName string - appName string - opName string - tracerProvider trace.TracerProvider - propagator propagation.TextMapPropagator - tracer trace.Tracer -} - -type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) - -var ( - ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") - ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") - providersConfig = map[string]providerConstructor{ - "http": httpTraceProvider, - "stdout": stdoutTraceProvider, - "noop": noopTraceProvider, - } - - otelTracer otelTracing -) - -// DefaultTracerProvider is used when no provider is given. -// The Noop tracer provider turns all tracing related operations into -// noops essentially disabling tracing. -const defaultTracerProvider = "noop" - -// newOtel creates a structure with components that apps can use to initialize OpenTelemetry -// tracing instrumentation code. -func newOtel(conf *configuration.Config) (*otelTracing, error) { - if IsNoopTracing(conf) { - log.Debug("open telemetry tracing disabled (noop)") - } else { - log.Debug("opentelemetry tracing enabled") - } - - otelTracer.appName = conf.GetString("webconfig.app_name") - otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") - otelTracer.opName = conf.GetString("webconfig.opentelemetry.operation_name", "http.request") - tracerProvider, err := newTracerProvider(conf) - if err != nil { - return &otelTracer, err - } - otelTracer.tracerProvider = tracerProvider - otel.SetTracerProvider(tracerProvider) - - // Set up propagator. - prop := newPropagator() - otelTracer.propagator = prop - otel.SetTextMapPropagator(prop) - - otelTracer.tracer = otel.Tracer(otelTracer.appName) - return &otelTracer, nil -} - -// IsNoopTracing returns true if the provider is set to "noop" -func IsNoopTracing(conf *configuration.Config) bool { - providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - return strings.EqualFold(providerName, "noop") -} - -// TracerProvider returns the tracer provider component. By default, the noop -// tracer provider is returned. -func (t otelTracing) TracerProvider() trace.TracerProvider { - if t.tracerProvider == nil { - return noop.NewTracerProvider() - } - return t.tracerProvider -} - -// Propagator returns the component that helps propagate trace context across -// API boundaries. By default, a W3C Trace Context format propagator is returned. -func (t otelTracing) Propagator() propagation.TextMapPropagator { - if t.propagator == nil { - return propagation.TraceContext{} - } - return t.propagator -} - -func newPropagator() propagation.TextMapPropagator { - return propagation.NewCompositeTextMapPropagator( - propagation.TraceContext{}, - propagation.Baggage{}, - ) -} - -// newTracerProvider creates the TracerProvider based on config setting -// If no config setting, a noop tracerProvider will be returned. -func newTracerProvider(conf *configuration.Config) (trace.TracerProvider, error) { - providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - if len(providerName) == 0 { - providerName = defaultTracerProvider - } - // Handling camelcase of provider. - providerName = strings.ToLower(providerName) - providerConfig := providersConfig[providerName] - if providerConfig == nil { - return nil, fmt.Errorf("%w for provider %s", ErrTracerProviderNotFound, providerName) - } - - traceProvider, err := providerConfig(conf) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) - } - return traceProvider, nil -} - -func noopTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - return noop.NewTracerProvider(), nil -} - -func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - option := stdouttrace.WithPrettyPrint() - exporter, err := stdouttrace.New(option) - if err != nil { - return nil, err - } - tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), - sdktrace.WithBatcher(exporter, - // Default is 5s. Set to 1s for demonstrative purposes. - sdktrace.WithBatchTimeout(time.Second)), - sdktrace.WithResource( - resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String(otelTracer.appName), - semconv.ServiceNamespaceKey.String(otelTracer.envName), - ), - ), - ) - return tp, nil -} - -func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - // Send traces over HTTP - endpoint := conf.GetString("webconfig.opentelemetry.endpoint") - if endpoint == "" { - return nil, ErrTracerProviderBuildFailed - } - exporter, err := otlptracehttp.New(context.Background(), - otlptracehttp.WithEndpoint(endpoint), - otlptracehttp.WithInsecure(), - ) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) - } - - return sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exporter), - sdktrace.WithResource( - resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String(otelTracer.appName), - semconv.ServiceNamespaceKey.String(otelTracer.envName), - ), - ), - ), nil -} - -func (s *WebconfigServer) OtelShutdown() { - sdkTraceProvider, ok := s.otelTracer.tracerProvider.(*sdktrace.TracerProvider) - if ok && sdkTraceProvider != nil { - sdkTraceProvider.Shutdown(context.TODO()) - } -} diff --git a/http/poke_handler.go b/http/poke_handler.go index cc48d30..205fea6 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -29,7 +29,6 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "go.opentelemetry.io/otel/attribute" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -157,28 +156,12 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - statusCode := http.StatusOK - pokeSpanPath := s.WebpaConnector.PokeSpanPath(mac) - // We are not passing any qparams to webpa, so fullPath = path - fullPath := pokeSpanPath - _, span := s.newChildPokeSpan(r.Context(), s.webpaPokeSpanTemplate, pokeSpanPath, fullPath, "PATCH") - - // endSpan should reflect the real status of the webpa patch call - // not the transformed custom status - // e.g 404 from webpa patch is converted to 521, but we want to show 404 - defer func() { - statusAttr := attribute.Int("http.status_code", statusCode) - span.SetAttributes(statusAttr) - span.End() - }() - transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - statusCode = rherr.StatusCode status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 @@ -204,7 +187,6 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } - statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index f3e523f..97348c1 100644 --- a/http/router.go +++ b/http/router.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -100,7 +100,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } - sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index fffb233..c1ee3ef 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -32,11 +32,6 @@ import ( "strings" "time" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" - semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/trace" - "github.com/IBM/sarama" "github.com/go-akka/configuration" "github.com/google/uuid" @@ -115,8 +110,6 @@ type WebconfigServer struct { traceparentParentID string tracestateVendorID string supplementaryAppendingEnabled bool - otelTracer *otelTracing // For OpenTelemetry Tracing - webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string upstreamProfilesEnabled bool @@ -263,12 +256,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) - otelTracer, err := newOtel(conf) - if err != nil { - // Just log err and continue - log.Error("Could not initialize open telemetry for tracing, but continuing") - } - supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) // kafka producer @@ -330,7 +317,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer jwksEnabled: jwksEnabled, traceparentParentID: traceparentParentID, tracestateVendorID: tracestateVendorID, - otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, @@ -339,8 +325,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer minTrust: minTrust, validSubdocIdMap: validSubdocIdMap, } - // Init the child poke span name - ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() return ws } @@ -973,37 +957,6 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - spanContext := trace.SpanContextFromContext(ctx) - remote := spanContext.IsRemote() - sc := trace.SpanContext{}.WithRemote(remote) - - traceIDStr, traceFlagsStr := s.parseTraceparent(r) - if traceIDStr != "" { - traceID, _ := trace.TraceIDFromHex(traceIDStr) - sc = sc.WithTraceID(traceID) - } - if traceFlagsStr != "" { - traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) - sc = sc.WithTraceFlags(traceFlags) - } - tracestateStr := r.Header.Get(common.HeaderTracestate) - if tracestateStr != "" { - tracestate, _ := trace.ParseTraceState(tracestateStr) - sc = sc.WithTraceState(tracestate) - } - ctx = trace.ContextWithSpanContext(ctx, sc) - - ctx, span := s.newParentPokeSpan(ctx, r) - defer endSpan(span, w) - - // Pass the context with the span to the next handler - next.ServeHTTP(w, r.WithContext(ctx)) - }) -} - // extract traceparent from the header func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { inTraceparent := r.Header.Get(common.HeaderTraceparent) @@ -1014,64 +967,6 @@ func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, tra return } -func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { - var span trace.Span - - ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) - // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindServer)) - - // Feedback: Better to use the "path"/API rather than a hard coded name - route := "oswebconfig_poke_handler" - if mux.CurrentRoute(r) != nil { // This can be nil in unit tests - route, _ = mux.CurrentRoute(r).GetPathTemplate() - } - s.addAttributes(span, route, r.URL.Path, r.URL.String(), r.Method) - return ctx, span -} - -func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, path string, fullPath string, method string) (context.Context, trace.Span) { - var span trace.Span - ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) - // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) - - s.addAttributes(span, route, path, fullPath, method) - return ctx, span -} - -func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { - span.SetAttributes( - attribute.String("env", otelTracer.envName), - attribute.String("operation.name", otelTracer.opName), - attribute.String("http.url_details.path", path), - semconv.HTTPMethodKey.String(method), - semconv.HTTPRouteKey.String(route), - semconv.HTTPURLKey.String(fullPath), - ) - - log.Debug(fmt.Sprintf("added span attribute key=env, value=%s", otelTracer.envName)) - log.Debug(fmt.Sprintf("added span attribute key=operation.name, value=%s", otelTracer.opName)) - log.Debug(fmt.Sprintf("added span attribute key=http.url_details.path, value=%s", path)) - log.Debug(fmt.Sprintf("added span attribute key=http.method, value=%s", method)) - log.Debug(fmt.Sprintf("added span attribute key=http.route, value=%s", route)) - log.Debug(fmt.Sprintf("added span attribute key=http.url, value=%s", fullPath)) -} - -func endSpan(span trace.Span, w http.ResponseWriter) { - if xw, ok := w.(*XResponseWriter); ok { - statusCode := xw.Status() - statusAttr := attribute.Int("http.status_code", statusCode) - span.SetAttributes(statusAttr) - log.Debug(fmt.Sprintf("added span attribute key=http.status_code, value=%d", statusCode)) - if statusCode >= http.StatusInternalServerError { - statusText := http.StatusText(statusCode) - span.SetStatus(codes.Error, statusText) - span.SetAttributes(attribute.String("http.response.error", statusText)) - log.Debug(fmt.Sprintf("added span attribute key=http.response.error, value=%s", statusText)) - } - } - span.End() -} - func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 41bbf9b..3ad378d 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -168,16 +168,6 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } -func (c *WebpaConnector) PokeSpanTemplate() string { - // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "{mac}") -} - -// Base URL with the cpemac populated -func (c *WebpaConnector) PokeSpanPath(mac string) string { - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, mac) -} - func (c *WebpaConnector) NewQueue(capacity int) error { if c.queue != nil { err := fmt.Errorf("queue is already initialized") diff --git a/main.go b/main.go index 8a47cb9..71de68c 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,6 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) - defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From e454080090cf93f6584e954c856643a03db02845 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 17 Dec 2024 23:56:19 -0800 Subject: [PATCH 133/155] restore otel codes --- go.mod | 19 +++++-- go.sum | 36 ++++++++++++-- http/poke_handler.go | 18 +++++++ http/router.go | 1 + http/webconfig_server.go | 105 +++++++++++++++++++++++++++++++++++++++ http/webpa_connector.go | 10 ++++ main.go | 1 + 7 files changed, 184 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a7cd24a..95108f9 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,11 @@ require ( github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 go.uber.org/automaxprocs v1.5.1 go.uber.org/ratelimit v0.2.0 golang.org/x/sync v0.6.0 @@ -26,14 +31,18 @@ require ( require ( github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -44,20 +53,24 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/klauspost/compress v1.16.7 // indirect - github.com/kr/pretty v0.3.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/stretchr/testify v1.9.0 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect golang.org/x/crypto v0.23.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index 8d8c331..74c2010 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,8 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -62,7 +64,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -91,6 +92,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= @@ -161,6 +167,8 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -220,7 +228,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -257,7 +264,6 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -297,6 +303,22 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= @@ -456,6 +478,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -557,6 +581,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -569,6 +597,8 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/http/poke_handler.go b/http/poke_handler.go index 205fea6..cc48d30 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -29,6 +29,7 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" + "go.opentelemetry.io/otel/attribute" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { @@ -156,12 +157,28 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } + statusCode := http.StatusOK + pokeSpanPath := s.WebpaConnector.PokeSpanPath(mac) + // We are not passing any qparams to webpa, so fullPath = path + fullPath := pokeSpanPath + _, span := s.newChildPokeSpan(r.Context(), s.webpaPokeSpanTemplate, pokeSpanPath, fullPath, "PATCH") + + // endSpan should reflect the real status of the webpa patch call + // not the transformed custom status + // e.g 404 from webpa patch is converted to 521, but we want to show 404 + defer func() { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + span.End() + }() + transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling + statusCode = rherr.StatusCode status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 @@ -187,6 +204,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } + statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index 97348c1..26d2dbe 100644 --- a/http/router.go +++ b/http/router.go @@ -100,6 +100,7 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } + sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index c1ee3ef..fffb233 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -32,6 +32,11 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/trace" + "github.com/IBM/sarama" "github.com/go-akka/configuration" "github.com/google/uuid" @@ -110,6 +115,8 @@ type WebconfigServer struct { traceparentParentID string tracestateVendorID string supplementaryAppendingEnabled bool + otelTracer *otelTracing // For OpenTelemetry Tracing + webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string upstreamProfilesEnabled bool @@ -256,6 +263,12 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) + otelTracer, err := newOtel(conf) + if err != nil { + // Just log err and continue + log.Error("Could not initialize open telemetry for tracing, but continuing") + } + supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) // kafka producer @@ -317,6 +330,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer jwksEnabled: jwksEnabled, traceparentParentID: traceparentParentID, tracestateVendorID: tracestateVendorID, + otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, @@ -325,6 +339,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer minTrust: minTrust, validSubdocIdMap: validSubdocIdMap, } + // Init the child poke span name + ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() return ws } @@ -957,6 +973,37 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } +func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + spanContext := trace.SpanContextFromContext(ctx) + remote := spanContext.IsRemote() + sc := trace.SpanContext{}.WithRemote(remote) + + traceIDStr, traceFlagsStr := s.parseTraceparent(r) + if traceIDStr != "" { + traceID, _ := trace.TraceIDFromHex(traceIDStr) + sc = sc.WithTraceID(traceID) + } + if traceFlagsStr != "" { + traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) + sc = sc.WithTraceFlags(traceFlags) + } + tracestateStr := r.Header.Get(common.HeaderTracestate) + if tracestateStr != "" { + tracestate, _ := trace.ParseTraceState(tracestateStr) + sc = sc.WithTraceState(tracestate) + } + ctx = trace.ContextWithSpanContext(ctx, sc) + + ctx, span := s.newParentPokeSpan(ctx, r) + defer endSpan(span, w) + + // Pass the context with the span to the next handler + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + // extract traceparent from the header func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { inTraceparent := r.Header.Get(common.HeaderTraceparent) @@ -967,6 +1014,64 @@ func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, tra return } +func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { + var span trace.Span + + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindServer)) + + // Feedback: Better to use the "path"/API rather than a hard coded name + route := "oswebconfig_poke_handler" + if mux.CurrentRoute(r) != nil { // This can be nil in unit tests + route, _ = mux.CurrentRoute(r).GetPathTemplate() + } + s.addAttributes(span, route, r.URL.Path, r.URL.String(), r.Method) + return ctx, span +} + +func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, path string, fullPath string, method string) (context.Context, trace.Span) { + var span trace.Span + ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) + // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) + + s.addAttributes(span, route, path, fullPath, method) + return ctx, span +} + +func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { + span.SetAttributes( + attribute.String("env", otelTracer.envName), + attribute.String("operation.name", otelTracer.opName), + attribute.String("http.url_details.path", path), + semconv.HTTPMethodKey.String(method), + semconv.HTTPRouteKey.String(route), + semconv.HTTPURLKey.String(fullPath), + ) + + log.Debug(fmt.Sprintf("added span attribute key=env, value=%s", otelTracer.envName)) + log.Debug(fmt.Sprintf("added span attribute key=operation.name, value=%s", otelTracer.opName)) + log.Debug(fmt.Sprintf("added span attribute key=http.url_details.path, value=%s", path)) + log.Debug(fmt.Sprintf("added span attribute key=http.method, value=%s", method)) + log.Debug(fmt.Sprintf("added span attribute key=http.route, value=%s", route)) + log.Debug(fmt.Sprintf("added span attribute key=http.url, value=%s", fullPath)) +} + +func endSpan(span trace.Span, w http.ResponseWriter) { + if xw, ok := w.(*XResponseWriter); ok { + statusCode := xw.Status() + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + log.Debug(fmt.Sprintf("added span attribute key=http.status_code, value=%d", statusCode)) + if statusCode >= http.StatusInternalServerError { + statusText := http.StatusText(statusCode) + span.SetStatus(codes.Error, statusText) + span.SetAttributes(attribute.String("http.response.error", statusText)) + log.Debug(fmt.Sprintf("added span attribute key=http.response.error, value=%s", statusText)) + } + } + span.End() +} + func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 3ad378d..41bbf9b 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -168,6 +168,16 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } +func (c *WebpaConnector) PokeSpanTemplate() string { + // By convention, span name won't have the host, but only the base template + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "{mac}") +} + +// Base URL with the cpemac populated +func (c *WebpaConnector) PokeSpanPath(mac string) string { + return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, mac) +} + func (c *WebpaConnector) NewQueue(capacity int) error { if c.queue != nil { err := fmt.Errorf("queue is already initialized") diff --git a/main.go b/main.go index 71de68c..8a47cb9 100644 --- a/main.go +++ b/main.go @@ -61,6 +61,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) + defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From 751143dfdd09b3cd1030482c9f22ee0bc144c445 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 18 Dec 2024 00:04:54 -0800 Subject: [PATCH 134/155] add back a missed file --- http/otel.go | 208 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 http/otel.go diff --git a/http/otel.go b/http/otel.go new file mode 100644 index 0000000..468035f --- /dev/null +++ b/http/otel.go @@ -0,0 +1,208 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package http + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + + "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" +) + +// Tracing contains the core dependencies to make tracing possible across an application. +type otelTracing struct { + providerName string + envName string + appName string + opName string + tracerProvider trace.TracerProvider + propagator propagation.TextMapPropagator + tracer trace.Tracer +} + +type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) + +var ( + ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") + ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") + providersConfig = map[string]providerConstructor{ + "http": httpTraceProvider, + "stdout": stdoutTraceProvider, + "noop": noopTraceProvider, + } + + otelTracer otelTracing +) + +// DefaultTracerProvider is used when no provider is given. +// The Noop tracer provider turns all tracing related operations into +// noops essentially disabling tracing. +const defaultTracerProvider = "noop" + +// newOtel creates a structure with components that apps can use to initialize OpenTelemetry +// tracing instrumentation code. +func newOtel(conf *configuration.Config) (*otelTracing, error) { + if IsNoopTracing(conf) { + log.Debug("open telemetry tracing disabled (noop)") + } else { + log.Debug("opentelemetry tracing enabled") + } + + otelTracer.appName = conf.GetString("webconfig.app_name") + otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") + otelTracer.opName = conf.GetString("webconfig.opentelemetry.operation_name", "http.request") + tracerProvider, err := newTracerProvider(conf) + if err != nil { + return &otelTracer, err + } + otelTracer.tracerProvider = tracerProvider + otel.SetTracerProvider(tracerProvider) + + // Set up propagator. + prop := newPropagator() + otelTracer.propagator = prop + otel.SetTextMapPropagator(prop) + + otelTracer.tracer = otel.Tracer(otelTracer.appName) + return &otelTracer, nil +} + +// IsNoopTracing returns true if the provider is set to "noop" +func IsNoopTracing(conf *configuration.Config) bool { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + return strings.EqualFold(providerName, "noop") +} + +// TracerProvider returns the tracer provider component. By default, the noop +// tracer provider is returned. +func (t otelTracing) TracerProvider() trace.TracerProvider { + if t.tracerProvider == nil { + return noop.NewTracerProvider() + } + return t.tracerProvider +} + +// Propagator returns the component that helps propagate trace context across +// API boundaries. By default, a W3C Trace Context format propagator is returned. +func (t otelTracing) Propagator() propagation.TextMapPropagator { + if t.propagator == nil { + return propagation.TraceContext{} + } + return t.propagator +} + +func newPropagator() propagation.TextMapPropagator { + return propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) +} + +// newTracerProvider creates the TracerProvider based on config setting +// If no config setting, a noop tracerProvider will be returned. +func newTracerProvider(conf *configuration.Config) (trace.TracerProvider, error) { + providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) + if len(providerName) == 0 { + providerName = defaultTracerProvider + } + // Handling camelcase of provider. + providerName = strings.ToLower(providerName) + providerConfig := providersConfig[providerName] + if providerConfig == nil { + return nil, fmt.Errorf("%w for provider %s", ErrTracerProviderNotFound, providerName) + } + + traceProvider, err := providerConfig(conf) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + return traceProvider, nil +} + +func noopTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + return noop.NewTracerProvider(), nil +} + +func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + option := stdouttrace.WithPrettyPrint() + exporter, err := stdouttrace.New(option) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), + sdktrace.WithBatcher(exporter, + // Default is 5s. Set to 1s for demonstrative purposes. + sdktrace.WithBatchTimeout(time.Second)), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(otelTracer.appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), + ), + ), + ) + return tp, nil +} + +func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { + // Send traces over HTTP + endpoint := conf.GetString("webconfig.opentelemetry.endpoint") + if endpoint == "" { + return nil, ErrTracerProviderBuildFailed + } + exporter, err := otlptracehttp.New(context.Background(), + otlptracehttp.WithEndpoint(endpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) + } + + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(otelTracer.appName), + semconv.ServiceNamespaceKey.String(otelTracer.envName), + ), + ), + ), nil +} + +func (s *WebconfigServer) OtelShutdown() { + sdkTraceProvider, ok := s.otelTracer.tracerProvider.(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} From 6bea1ad46ba681d092440cd7af4c3c683ad280ca Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 19 Dec 2024 22:45:39 -0800 Subject: [PATCH 135/155] handle tracing propagation --- go.mod | 89 ++--- go.sum | 599 +++++--------------------------- http/http_client.go | 10 +- http/mqtt_connector.go | 9 +- http/multipart.go | 14 +- http/multipart_test.go | 19 +- http/poke_handler.go | 37 +- http/router.go | 1 - http/supplementary_handler.go | 6 +- http/upstream_connector.go | 9 +- http/upstream_connector_test.go | 6 +- http/webconfig_server.go | 101 +----- http/webpa_connector.go | 29 +- http/xconf_connector.go | 9 +- kafka/consumer.go | 7 +- main.go | 1 - 16 files changed, 224 insertions(+), 722 deletions(-) diff --git a/go.mod b/go.mod index 95108f9..6b05244 100644 --- a/go.mod +++ b/go.mod @@ -1,50 +1,52 @@ module github.com/rdkcentral/webconfig -go 1.21 +go 1.22.7 + +toolchain go1.22.10 require ( - github.com/IBM/sarama v1.42.1 + github.com/IBM/sarama v1.43.3 github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 - github.com/gocql/gocql v1.6.0 - github.com/golang-jwt/jwt/v5 v5.0.0 + github.com/gocql/gocql v1.7.0 + github.com/golang-jwt/jwt/v5 v5.2.1 github.com/google/uuid v1.6.0 - github.com/gorilla/mux v1.8.0 - github.com/mattn/go-sqlite3 v1.14.15 - github.com/prometheus/client_golang v1.13.0 - github.com/prometheus/client_model v0.2.0 - github.com/sirupsen/logrus v1.9.0 - github.com/twmb/murmur3 v1.1.6 + github.com/gorilla/mux v1.8.1 + github.com/mattn/go-sqlite3 v1.14.24 + github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/client_model v0.6.1 + github.com/sirupsen/logrus v1.9.3 + github.com/twmb/murmur3 v1.1.8 github.com/vmihailenco/msgpack v4.0.4+incompatible - github.com/vmihailenco/msgpack/v4 v4.3.12 - go.opentelemetry.io/otel v1.27.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 - go.opentelemetry.io/otel/sdk v1.27.0 - go.opentelemetry.io/otel/trace v1.27.0 - go.uber.org/automaxprocs v1.5.1 - go.uber.org/ratelimit v0.2.0 - golang.org/x/sync v0.6.0 + github.com/vmihailenco/msgpack/v4 v4.3.13 + go.opentelemetry.io/otel v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0 + go.opentelemetry.io/otel/sdk v1.33.0 + go.opentelemetry.io/otel/trace v1.33.0 + go.uber.org/automaxprocs v1.6.0 + go.uber.org/ratelimit v0.3.1 + golang.org/x/sync v0.10.0 gotest.tools v2.2.0+incompatible ) require ( - github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect + github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/eapache/go-resiliency v1.4.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -52,25 +54,28 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect - go.opentelemetry.io/otel/metric v1.27.0 // indirect - go.opentelemetry.io/proto/otlp v1.2.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/crypto v0.30.0 // indirect + golang.org/x/net v0.32.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/grpc v1.64.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.68.1 // indirect + google.golang.org/protobuf v1.35.2 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index 74c2010..d588a29 100644 --- a/go.sum +++ b/go.sum @@ -1,51 +1,9 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= -github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= +github.com/IBM/sarama v1.43.3 h1:Yj6L2IaNvb2mRBop39N7mmJAHBVY3dTPncr3qGVkxPA= +github.com/IBM/sarama v1.43.3/go.mod h1:FVIRaLrhK3Cla/9FfRF5X9Zua2KpS3SYIXxhac1H+FQ= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= @@ -54,80 +12,33 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= -github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= +github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665/go.mod h1:19bUnum2ZAeftfwwLZ/wRe7idyfoW2MfmXO464Hrfbw= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= -github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= +github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -135,52 +46,27 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -193,21 +79,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -215,431 +88,147 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= -github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= -github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= -github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= -github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/msgpack/v4 v4.3.13 h1:A2wsiTbvp63ilDaWmsk2wjx6xZdxQOvpiNlKBGKKXKI= +github.com/vmihailenco/msgpack/v4 v4.3.13/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= -go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= -go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= -go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= -go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= -go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= -go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= -go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= -go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= -go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= -go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= -go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= -go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 h1:wpMfgF8E1rkrT1Z6meFh1NDtownE9Ii3n3X2GJYjsaU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0 h1:W5AWUn/IVe8RFb5pZx1Uh9Laf/4+Qmm4kJL5zPuvR+0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0/go.mod h1:mzKxJywMNBdEX8TSJais3NnsVZUaJ+bAy6UxPTng2vk= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= +go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= -google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/http/http_client.go b/http/http_client.go index f67598c..9661cbf 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -19,6 +19,7 @@ package http import ( "bytes" + "context" "crypto/tls" "encoding/base64" "encoding/json" @@ -37,6 +38,7 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" + "go.opentelemetry.io/otel/propagation" ) const ( @@ -107,7 +109,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl } } -func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { +func (c *HttpClient) Do(ctx context.Context, method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { fields := common.FilterLogFields(auditFields) // verify a response is received @@ -206,6 +208,8 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] startTime := time.Now() // the core http call + propagator := propagation.TraceContext{} + propagator.Inject(ctx, propagation.HeaderCarrier(req.Header)) res, err := c.Client.Do(req) // err should be *url.Error @@ -354,7 +358,7 @@ func (c *HttpClient) Do(method string, url string, header http.Header, bbytes [] return rbytes, res.Header, false, nil } -func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { +func (c *HttpClient) DoWithRetries(ctx context.Context, method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { var respBytes []byte var respHeader http.Header var err error @@ -368,7 +372,7 @@ func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Heade if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - respBytes, respHeader, cont, err = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) + respBytes, respHeader, cont, err = c.Do(ctx, method, url, rHeader, cbytes, fields, loggerName, i) if !cont { break } diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 07c086b..064f2e5 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -14,18 +14,19 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( + "context" "crypto/tls" "fmt" "net/http" "time" - "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" "github.com/google/uuid" + "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) @@ -64,7 +65,7 @@ func (c *MqttConnector) ServiceName() string { return c.serviceName } -func (c *MqttConnector) PostMqtt(cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { +func (c *MqttConnector) PostMqtt(ctx context.Context, cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { url := fmt.Sprintf(mqttUrlTemplate, c.MqttHost(), cpeMac) var traceId, xmTraceId, outTraceparent, outTracestate string @@ -94,7 +95,7 @@ func (c *MqttConnector) PostMqtt(cpeMac string, bbytes []byte, fields log.Fields header.Set(common.HeaderTraceparent, outTraceparent) header.Set(common.HeaderTracestate, outTracestate) - rbytes, _, err := c.DoWithRetries("POST", url, header, bbytes, fields, c.ServiceName()) + rbytes, _, err := c.DoWithRetries(ctx, "POST", url, header, bbytes, fields, c.ServiceName()) if err != nil { return rbytes, common.NewError(err) } diff --git a/http/multipart.go b/http/multipart.go index 5e39531..a431552 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -18,6 +18,7 @@ package http import ( + "context" "errors" "fmt" "net/http" @@ -43,6 +44,7 @@ var ( ) func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() // check if this is a Supplementary service, if so, call a different handler if hd := r.Header.Get(common.HeaderSupplementaryService); len(hd) > 0 { s.MultipartSupplementaryHandler(w, r) @@ -94,7 +96,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. r.Header.Set(common.HeaderSchemaVersion, "none") } - status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(s, ctx, r.Header, common.RouteHttp, fields) switch status { case http.StatusNotFound: @@ -118,7 +120,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. _, _ = w.Write(respBytes) } -func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route string, fields log.Fields) (int, http.Header, []byte, error) { +func BuildWebconfigResponse(s *WebconfigServer, ctx context.Context, rHeader http.Header, route string, fields log.Fields) (int, http.Header, []byte, error) { fields["for_device"] = true fields["is_primary"] = true @@ -131,7 +133,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin // factory reset handling ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) if ifNoneMatch == "NONE" || ifNoneMatch == "NONE-REBOOT" { - status, respHeader, rbytes, err := BuildFactoryResetResponse(s, rHeader, fields) + status, respHeader, rbytes, err := BuildFactoryResetResponse(s, ctx, rHeader, fields) if err != nil { return status, respHeader, rbytes, common.NewError(err) } @@ -285,7 +287,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin upstreamHeader.Set(common.HeaderUpstreamOldSchemaVersion, oldRootDocument.SchemaVersion) } - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, respBytes, fields) + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(ctx, mac, upstreamHeader, respBytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -337,7 +339,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusOK, upstreamRespHeader, finalFilteredBytes, nil } -func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { +func BuildFactoryResetResponse(s *WebconfigServer, ctx context.Context, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { c := s.DatabaseClient uconn := s.GetUpstreamConnector() mac := rHeader.Get(common.HeaderDeviceId) @@ -408,7 +410,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l } // call /upstream to handle factory reset - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(ctx, mac, upstreamHeader, oldDocBytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { diff --git a/http/multipart_test.go b/http/multipart_test.go index ca35726..b20bb45 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -19,6 +19,7 @@ package http import ( "bytes" + "context" "encoding/hex" "fmt" "io" @@ -298,6 +299,7 @@ func TestCpeMiddleware(t *testing.T) { func TestVersionFiltering(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) + ctx := context.Background() cpeMac := util.GenerateRandomCpeMac() // ==== group 1 lan ==== @@ -415,7 +417,7 @@ func TestVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") fields := make(log.Fields) - status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType := respHeader.Get(common.HeaderContentType) @@ -433,7 +435,7 @@ func TestVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDocName, "root,lan,wan") kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusNotModified) assert.Equal(t, len(respBytes), 0) @@ -617,6 +619,7 @@ func TestUpstreamVersionFiltering(t *testing.T) { func TestMqttUpstreamVersionFiltering(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) + ctx := context.Background() cpeMac := util.GenerateRandomCpeMac() // ==== group 1 lan ==== @@ -677,7 +680,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader := make(http.Header) kHeader.Set(common.HeaderDeviceId, cpeMac) fields := make(log.Fields) - status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType := respHeader.Get(common.HeaderContentType) @@ -707,7 +710,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.Assert(t, err != nil) assert.Equal(t, status, http.StatusServiceUnavailable) @@ -715,7 +718,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader = make(http.Header) kHeader.Set(common.HeaderDeviceId, cpeMac) fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.Assert(t, err != nil) assert.Equal(t, status, http.StatusServiceUnavailable) @@ -741,7 +744,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType = respHeader.Get(common.HeaderContentType) @@ -765,7 +768,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusNotModified) @@ -776,7 +779,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType = respHeader.Get(common.HeaderContentType) diff --git a/http/poke_handler.go b/http/poke_handler.go index cc48d30..fb25745 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -29,10 +29,24 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { + // tracing propagation + ctx := r.Context() + + propagator := propagation.TraceContext{} + ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) + span := trace.SpanFromContext(ctx) + defer span.End() + tracer := otel.Tracer(s.otelTracer.appName) + ctx, childSpan := tracer.Start(ctx, s.otelTracer.opName) + defer childSpan.End() + + // handler params := mux.Vars(r) mac := params["mac"] mac = strings.ToUpper(mac) @@ -104,7 +118,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - _, err = s.PostMqtt(deviceId, mbytes, fields) + _, err = s.PostMqtt(ctx, deviceId, mbytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -157,28 +171,12 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - statusCode := http.StatusOK - pokeSpanPath := s.WebpaConnector.PokeSpanPath(mac) - // We are not passing any qparams to webpa, so fullPath = path - fullPath := pokeSpanPath - _, span := s.newChildPokeSpan(r.Context(), s.webpaPokeSpanTemplate, pokeSpanPath, fullPath, "PATCH") - - // endSpan should reflect the real status of the webpa patch call - // not the transformed custom status - // e.g 404 from webpa patch is converted to 521, but we want to show 404 - defer func() { - statusAttr := attribute.Int("http.status_code", statusCode) - span.SetAttributes(statusAttr) - span.End() - }() - - transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) + transactionId, err := s.Poke(ctx, r.Header, mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { // webpa error handling - statusCode = rherr.StatusCode status := rherr.StatusCode if rherr.StatusCode == http.StatusNotFound { status = 521 @@ -204,7 +202,6 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } return } - statusCode = http.StatusInternalServerError Error(w, http.StatusInternalServerError, common.NewError(err)) return } diff --git a/http/router.go b/http/router.go index 26d2dbe..97348c1 100644 --- a/http/router.go +++ b/http/router.go @@ -100,7 +100,6 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.NoAuthMiddleware) } } - sub2.Use(s.spanMiddleware) sub2.HandleFunc("", s.PokeHandler).Methods("POST") sub3 := router.Path("/api/v1/device/{mac}/rootdocument").Subrouter() diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 20dc8fd..401c4b7 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -34,6 +34,8 @@ const ( ) func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + // ==== data integrity check ==== params := mux.Vars(r) mac, ok := params["mac"] @@ -80,7 +82,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r urlSuffix := util.GetTelemetryQueryString(r.Header, mac, queryParams, partnerId) fields["is_telemetry"] = true - baseProfileBytes, resHeader, err := s.GetProfiles(urlSuffix, fields) + baseProfileBytes, resHeader, err := s.GetProfiles(ctx, urlSuffix, fields) xconfNotFound := false if err != nil { var rherr common.RemoteHttpError @@ -106,7 +108,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r var profileBytes []byte if s.UpstreamProfilesEnabled() && rootdoc != nil && len(rootdoc.QueryParams) > 0 { // Get profiles from the second source - extraProfileBytes, _, err := s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) + extraProfileBytes, _, err := s.GetUpstreamProfiles(ctx, mac, queryParams, r.Header, fields) if err != nil { exitNow := true var rherr common.RemoteHttpError diff --git a/http/upstream_connector.go b/http/upstream_connector.go index d006bc5..303f2b6 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -18,6 +18,7 @@ package http import ( + "context" "crypto/tls" "fmt" "net/http" @@ -72,7 +73,7 @@ func (c *UpstreamConnector) ServiceName() string { return c.serviceName } -func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { +func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { url := c.UpstreamHost() + fmt.Sprintf(c.upstreamUrlTemplate, mac) if itf, ok := fields["audit_id"]; ok { @@ -89,14 +90,14 @@ func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes } } - rbytes, header, err := c.DoWithRetries("POST", url, header, bbytes, fields, c.ServiceName()) + rbytes, header, err := c.DoWithRetries(ctx, "POST", url, header, bbytes, fields, c.ServiceName()) if err != nil { return rbytes, header, owcommon.NewError(err) } return rbytes, header, nil } -func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { +func (c *UpstreamConnector) GetUpstreamProfiles(ctx context.Context, mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { url := c.UpstreamHost() + fmt.Sprintf(c.profileUrlTemplate, mac, queryParams) if itf, ok := fields["audit_id"]; ok { @@ -113,7 +114,7 @@ func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header } } - rbytes, header, err := c.DoWithRetries("GET", url, header, nil, fields, c.ServiceName()) + rbytes, header, err := c.DoWithRetries(ctx, "GET", url, header, nil, fields, c.ServiceName()) if err != nil { return rbytes, header, owcommon.NewError(err) } diff --git a/http/upstream_connector_test.go b/http/upstream_connector_test.go index e9dfccb..13fddbb 100644 --- a/http/upstream_connector_test.go +++ b/http/upstream_connector_test.go @@ -14,10 +14,11 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( + "context" "net/http" "net/http/httptest" "testing" @@ -51,6 +52,7 @@ func TestUpstreamConnector(t *testing.T) { bbytes := []byte("hello world") var err error fields := log.Fields{} - _, _, err = server.PostUpstream(mac, header, bbytes, fields) + ctx := context.Background() + _, _, err = server.PostUpstream(ctx, mac, header, bbytes, fields) assert.NilError(t, err) } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index fffb233..daf331e 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -32,11 +32,6 @@ import ( "strings" "time" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" - semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/trace" - "github.com/IBM/sarama" "github.com/go-akka/configuration" "github.com/google/uuid" @@ -116,7 +111,6 @@ type WebconfigServer struct { tracestateVendorID string supplementaryAppendingEnabled bool otelTracer *otelTracing // For OpenTelemetry Tracing - webpaPokeSpanTemplate string kafkaProducerEnabled bool kafkaProducerTopic string upstreamProfilesEnabled bool @@ -339,8 +333,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer minTrust: minTrust, validSubdocIdMap: validSubdocIdMap, } - // Init the child poke span name - ws.webpaPokeSpanTemplate = ws.WebpaConnector.PokeSpanTemplate() return ws } @@ -712,9 +704,9 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return fmt.Errorf("invalid partner") } -func (c *WebconfigServer) Poke(rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { +func (c *WebconfigServer) Poke(ctx context.Context, rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) - transactionId, err := c.Patch(rHeader, cpeMac, token, []byte(body), fields) + transactionId, err := c.Patch(ctx, rHeader, cpeMac, token, []byte(body), fields) if err != nil { return "", common.NewError(err) } @@ -973,37 +965,6 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -func (s *WebconfigServer) spanMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - spanContext := trace.SpanContextFromContext(ctx) - remote := spanContext.IsRemote() - sc := trace.SpanContext{}.WithRemote(remote) - - traceIDStr, traceFlagsStr := s.parseTraceparent(r) - if traceIDStr != "" { - traceID, _ := trace.TraceIDFromHex(traceIDStr) - sc = sc.WithTraceID(traceID) - } - if traceFlagsStr != "" { - traceFlags := trace.TraceFlags(hexStringToBytes(traceFlagsStr)[0]) - sc = sc.WithTraceFlags(traceFlags) - } - tracestateStr := r.Header.Get(common.HeaderTracestate) - if tracestateStr != "" { - tracestate, _ := trace.ParseTraceState(tracestateStr) - sc = sc.WithTraceState(tracestate) - } - ctx = trace.ContextWithSpanContext(ctx, sc) - - ctx, span := s.newParentPokeSpan(ctx, r) - defer endSpan(span, w) - - // Pass the context with the span to the next handler - next.ServeHTTP(w, r.WithContext(ctx)) - }) -} - // extract traceparent from the header func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { inTraceparent := r.Header.Get(common.HeaderTraceparent) @@ -1014,64 +975,6 @@ func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, tra return } -func (s *WebconfigServer) newParentPokeSpan(ctx context.Context, r *http.Request) (context.Context, trace.Span) { - var span trace.Span - - ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) - // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindServer)) - - // Feedback: Better to use the "path"/API rather than a hard coded name - route := "oswebconfig_poke_handler" - if mux.CurrentRoute(r) != nil { // This can be nil in unit tests - route, _ = mux.CurrentRoute(r).GetPathTemplate() - } - s.addAttributes(span, route, r.URL.Path, r.URL.String(), r.Method) - return ctx, span -} - -func (s *WebconfigServer) newChildPokeSpan(ctx context.Context, route string, path string, fullPath string, method string) (context.Context, trace.Span) { - var span trace.Span - ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName) - // ctx, span = otelTracer.tracer.Start(ctx, otelTracer.opName, trace.WithSpanKind(trace.SpanKindClient)) - - s.addAttributes(span, route, path, fullPath, method) - return ctx, span -} - -func (s *WebconfigServer) addAttributes(span trace.Span, route string, path string, fullPath string, method string) { - span.SetAttributes( - attribute.String("env", otelTracer.envName), - attribute.String("operation.name", otelTracer.opName), - attribute.String("http.url_details.path", path), - semconv.HTTPMethodKey.String(method), - semconv.HTTPRouteKey.String(route), - semconv.HTTPURLKey.String(fullPath), - ) - - log.Debug(fmt.Sprintf("added span attribute key=env, value=%s", otelTracer.envName)) - log.Debug(fmt.Sprintf("added span attribute key=operation.name, value=%s", otelTracer.opName)) - log.Debug(fmt.Sprintf("added span attribute key=http.url_details.path, value=%s", path)) - log.Debug(fmt.Sprintf("added span attribute key=http.method, value=%s", method)) - log.Debug(fmt.Sprintf("added span attribute key=http.route, value=%s", route)) - log.Debug(fmt.Sprintf("added span attribute key=http.url, value=%s", fullPath)) -} - -func endSpan(span trace.Span, w http.ResponseWriter) { - if xw, ok := w.(*XResponseWriter); ok { - statusCode := xw.Status() - statusAttr := attribute.Int("http.status_code", statusCode) - span.SetAttributes(statusAttr) - log.Debug(fmt.Sprintf("added span attribute key=http.status_code, value=%d", statusCode)) - if statusCode >= http.StatusInternalServerError { - statusText := http.StatusText(statusCode) - span.SetStatus(codes.Error, statusText) - span.SetAttributes(attribute.String("http.response.error", statusText)) - log.Debug(fmt.Sprintf("added span attribute key=http.response.error, value=%s", statusText)) - } - } - span.End() -} - func hexStringToBytes(hexString string) []byte { bytes, _ := hex.DecodeString(hexString) return bytes diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 41bbf9b..f184e1f 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -18,6 +18,7 @@ package http import ( + "context" "crypto/tls" "encoding/json" "errors" @@ -168,16 +169,6 @@ func (c *WebpaConnector) SetApiVersion(apiVersion string) { c.apiVersion = apiVersion } -func (c *WebpaConnector) PokeSpanTemplate() string { - // By convention, span name won't have the host, but only the base template - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, "{mac}") -} - -// Base URL with the cpemac populated -func (c *WebpaConnector) PokeSpanPath(mac string) string { - return fmt.Sprintf(webpaUrlTemplate[2:], c.apiVersion, mac) -} - func (c *WebpaConnector) NewQueue(capacity int) error { if c.queue != nil { err := fmt.Errorf("queue is already initialized") @@ -195,7 +186,7 @@ func (c *WebpaConnector) SetAsyncPokeEnabled(enabled bool) { c.asyncPokeEnabled = enabled } -func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { +func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { url := fmt.Sprintf(webpaUrlTemplate, c.WebpaHost(), c.ApiVersion(), cpeMac) var traceId, xmTraceId string @@ -220,16 +211,16 @@ func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, header.Set("X-Webpa-Transaction-Id", transactionId) method := "PATCH" - _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) + _, _, cont, err := c.syncClient.Do(ctx, method, url, header, bbytes, fields, webpaServiceName, 0) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { if rherr.StatusCode == 524 { if c.asyncPokeEnabled { c.queue <- struct{}{} - go c.AsyncDoWithRetries(method, url, header, bbytes, fields, asyncWebpaServiceName) + go c.AsyncDoWithRetries(ctx, method, url, header, bbytes, fields, asyncWebpaServiceName) } else { - _, err := c.SyncDoWithRetries(method, url, header, bbytes, fields, webpaServiceName) + _, err := c.SyncDoWithRetries(ctx, method, url, header, bbytes, fields, webpaServiceName) if err != nil { return transactionId, common.NewError(err) } @@ -238,7 +229,7 @@ func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, } } if cont { - _, _, err := c.syncClient.DoWithRetries("PATCH", url, header, bbytes, fields, webpaServiceName) + _, _, err := c.syncClient.DoWithRetries(ctx, "PATCH", url, header, bbytes, fields, webpaServiceName) if err != nil { return transactionId, common.NewError(err) } @@ -249,7 +240,7 @@ func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, return transactionId, nil } -func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { +func (c *WebpaConnector) AsyncDoWithRetries(ctx context.Context, method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { tfields := common.FilterLogFields(fields) tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { @@ -258,7 +249,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header ht if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, cont, _ := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) + _, _, cont, _ := c.asyncClient.Do(ctx, method, url, header, cbytes, fields, loggerName, i) if !cont { msg := fmt.Sprintf("finished success after 1 retry") if i > 1 { @@ -275,7 +266,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header ht } // this has 1 less retries compared to the standard DoWithRetries() -func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, error) { +func (c *WebpaConnector) SyncDoWithRetries(ctx context.Context, method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, error) { var rbytes []byte var err error var cont bool @@ -286,7 +277,7 @@ func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header htt if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - rbytes, _, cont, err = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) + rbytes, _, cont, err = c.syncClient.Do(ctx, method, url, header, cbytes, fields, loggerName, i) if !cont { // in the case of 524/in-progress, we continue var rherr common.RemoteHttpError diff --git a/http/xconf_connector.go b/http/xconf_connector.go index d40550a..7a39974 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -14,16 +14,17 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( + "context" "crypto/tls" "fmt" "net/http" - owcommon "github.com/rdkcentral/webconfig/common" "github.com/go-akka/configuration" + owcommon "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" ) @@ -62,9 +63,9 @@ func (c *XconfConnector) ServiceName() string { return c.serviceName } -func (c *XconfConnector) GetProfiles(urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { +func (c *XconfConnector) GetProfiles(ctx context.Context, urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { url := fmt.Sprintf(xconfUrlTemplate, c.XconfHost(), urlSuffix) - rbytes, resHeader, err := c.DoWithRetries("GET", url, nil, nil, fields, c.ServiceName()) + rbytes, resHeader, err := c.DoWithRetries(ctx, "GET", url, nil, nil, fields, c.ServiceName()) if err != nil { return rbytes, resHeader, owcommon.NewError(err) } diff --git a/kafka/consumer.go b/kafka/consumer.go index 40e8ec1..51cc318 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -18,6 +18,7 @@ package kafka import ( + "context" "encoding/base64" "encoding/json" "fmt" @@ -105,6 +106,8 @@ func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common // NOTE we choose to return an EventMessage object just to pass along the metricsAgent func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common.EventMessage, error) { + ctx := context.TODO() + rHeader, _ := util.ParseHttp(inbytes) params := rHeader.Get(common.HeaderDocName) cpeMac := rHeader.Get(common.HeaderDeviceId) @@ -148,7 +151,7 @@ func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common. rHeader.Set(common.HeaderSchemaVersion, "none") } - status, respHeader, respBytes, err := wchttp.BuildWebconfigResponse(c.WebconfigServer, rHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := wchttp.BuildWebconfigResponse(c.WebconfigServer, ctx, rHeader, common.RouteMqtt, fields) if err != nil && respBytes == nil { respBytes = []byte(err.Error()) } @@ -159,7 +162,7 @@ func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common. } mqttBytes := common.BuildPayloadAsHttp(status, respHeader, respBytes) - _, err = c.PostMqtt(cpeMac, mqttBytes, fields) + _, err = c.PostMqtt(ctx, cpeMac, mqttBytes, fields) if err != nil { return &m, common.NewError(err) } diff --git a/main.go b/main.go index 8a47cb9..71de68c 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,6 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) - defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From 127cee1a2b5cd9144941210d621399f447aff0ea Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 19 Dec 2024 22:51:09 -0800 Subject: [PATCH 136/155] add back calling otelshutdown --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 71de68c..8a47cb9 100644 --- a/main.go +++ b/main.go @@ -61,6 +61,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) + defer server.OtelShutdown() // setup logging logFile := server.GetString("webconfig.log.file") From 13a9c622476ff4237a69f758f5a178ed96175f87 Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 19 Dec 2024 23:04:56 -0800 Subject: [PATCH 137/155] roll back go.mod to use 1.21 --- go.mod | 89 ++++----- go.sum | 599 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 547 insertions(+), 141 deletions(-) diff --git a/go.mod b/go.mod index 6b05244..95108f9 100644 --- a/go.mod +++ b/go.mod @@ -1,52 +1,50 @@ module github.com/rdkcentral/webconfig -go 1.22.7 - -toolchain go1.22.10 +go 1.21 require ( - github.com/IBM/sarama v1.43.3 + github.com/IBM/sarama v1.42.1 github.com/MicahParks/keyfunc/v2 v2.1.0 github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 - github.com/gocql/gocql v1.7.0 - github.com/golang-jwt/jwt/v5 v5.2.1 + github.com/gocql/gocql v1.6.0 + github.com/golang-jwt/jwt/v5 v5.0.0 github.com/google/uuid v1.6.0 - github.com/gorilla/mux v1.8.1 - github.com/mattn/go-sqlite3 v1.14.24 - github.com/prometheus/client_golang v1.20.5 - github.com/prometheus/client_model v0.6.1 - github.com/sirupsen/logrus v1.9.3 - github.com/twmb/murmur3 v1.1.8 + github.com/gorilla/mux v1.8.0 + github.com/mattn/go-sqlite3 v1.14.15 + github.com/prometheus/client_golang v1.13.0 + github.com/prometheus/client_model v0.2.0 + github.com/sirupsen/logrus v1.9.0 + github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible - github.com/vmihailenco/msgpack/v4 v4.3.13 - go.opentelemetry.io/otel v1.33.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0 - go.opentelemetry.io/otel/sdk v1.33.0 - go.opentelemetry.io/otel/trace v1.33.0 - go.uber.org/automaxprocs v1.6.0 - go.uber.org/ratelimit v0.3.1 - golang.org/x/sync v0.10.0 + github.com/vmihailenco/msgpack/v4 v4.3.12 + go.opentelemetry.io/otel v1.27.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 + go.opentelemetry.io/otel/sdk v1.27.0 + go.opentelemetry.io/otel/trace v1.27.0 + go.uber.org/automaxprocs v1.5.1 + go.uber.org/ratelimit v0.2.0 + golang.org/x/sync v0.6.0 gotest.tools v2.2.0+incompatible ) require ( - github.com/benbjohnson/clock v1.3.0 // indirect + github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/eapache/go-resiliency v1.7.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect @@ -54,28 +52,25 @@ require ( github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/common v0.55.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/vmihailenco/tagparser v0.1.2 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect - go.opentelemetry.io/otel/metric v1.33.0 // indirect - go.opentelemetry.io/proto/otlp v1.4.0 // indirect - go.uber.org/atomic v1.11.0 // indirect - golang.org/x/crypto v0.30.0 // indirect - golang.org/x/net v0.32.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/text v0.21.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.27.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/grpc v1.68.1 // indirect - google.golang.org/protobuf v1.35.2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect + google.golang.org/grpc v1.64.0 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect ) diff --git a/go.sum b/go.sum index d588a29..74c2010 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,51 @@ -github.com/IBM/sarama v1.43.3 h1:Yj6L2IaNvb2mRBop39N7mmJAHBVY3dTPncr3qGVkxPA= -github.com/IBM/sarama v1.43.3/go.mod h1:FVIRaLrhK3Cla/9FfRF5X9Zua2KpS3SYIXxhac1H+FQ= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/IBM/sarama v1.42.1 h1:wugyWa15TDEHh2kvq2gAy1IHLjEjuYOYgXz/ruC/OSQ= +github.com/IBM/sarama v1.42.1/go.mod h1:Xxho9HkHd4K/MDUo/T/sOqwtX/17D33++E9Wib6hUdQ= github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= +github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= @@ -12,33 +54,80 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= -github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-resiliency v1.4.0 h1:3OK9bWpPk5q6pbFAaYSEwD9CLUSHG8bnZuqX2yMt3B0= +github.com/eapache/go-resiliency v1.4.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 h1:Iz3aEheYgn+//VX7VisgCmF/wW3BMtXCLbvHV4jMQJA= github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665/go.mod h1:19bUnum2ZAeftfwwLZ/wRe7idyfoW2MfmXO464Hrfbw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/gocql/gocql v1.7.0 h1:O+7U7/1gSN7QTEAaMEsJc1Oq2QHXvCWoF3DFK9HDHus= -github.com/gocql/gocql v1.7.0/go.mod h1:vnlvXyFZeLBF0Wy+RS8hrOdbn0UWsWtdg07XJnFxZ+4= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= +github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -46,27 +135,52 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6 github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= -github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -79,8 +193,21 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6 github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -88,147 +215,431 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= -github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= -github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= +github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= -github.com/vmihailenco/msgpack/v4 v4.3.13 h1:A2wsiTbvp63ilDaWmsk2wjx6xZdxQOvpiNlKBGKKXKI= -github.com/vmihailenco/msgpack/v4 v4.3.13/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/msgpack/v4 v4.3.12 h1:07s4sz9IReOgdikxLTKNbBdqDMLsjPKXwvCazn8G65U= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc= github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= -go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 h1:wpMfgF8E1rkrT1Z6meFh1NDtownE9Ii3n3X2GJYjsaU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0 h1:W5AWUn/IVe8RFb5pZx1Uh9Laf/4+Qmm4kJL5zPuvR+0= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.33.0/go.mod h1:mzKxJywMNBdEX8TSJais3NnsVZUaJ+bAy6UxPTng2vk= -go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= -go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= -go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= -go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= -go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= -go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= -go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= -go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= -go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= -go.uber.org/ratelimit v0.3.1 h1:K4qVE+byfv/B3tC+4nYWP7v/6SimcO7HzHekoMNBma0= -go.uber.org/ratelimit v0.3.1/go.mod h1:6euWsTB6U/Nb3X++xEUXA8ciPJvr19Q/0h1+oDcJhRk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opentelemetry.io/otel v1.27.0 h1:9BZoF3yMK/O1AafMiQTVu0YDj5Ea4hPhxCs7sGva+cg= +go.opentelemetry.io/otel v1.27.0/go.mod h1:DMpAK8fzYRzs+bi3rS5REupisuqTheUlSZJ1WnZaPAQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0 h1:R9DE4kQ4k+YtfLI2ULwX82VtNQ2J8yZmA7ZIF/D+7Mc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.27.0/go.mod h1:OQFyQVrDlbe+R7xrEyDr/2Wr67Ol0hRUgsfA+V5A95s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 h1:QY7/0NeRPKlzusf40ZE4t1VlMKbqSNT7cJRYzWuja0s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0/go.mod h1:HVkSiDhTM9BoUJU8qE6j2eSWLLXvi1USXjyd2BXT8PY= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0 h1:/0YaXu3755A/cFbtXp+21lkXgI0QE5avTWA2HjU9/WE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.27.0/go.mod h1:m7SFxp0/7IxmJPLIY3JhOcU9CoFzDaCPL6xxQIxhA+o= +go.opentelemetry.io/otel/metric v1.27.0 h1:hvj3vdEKyeCi4YaYfNjv2NUje8FqKqUY8IlF0FxV/ik= +go.opentelemetry.io/otel/metric v1.27.0/go.mod h1:mVFgmRlhljgBiuk/MP/oKylr4hs85GZAylncepAX/ak= +go.opentelemetry.io/otel/sdk v1.27.0 h1:mlk+/Y1gLPLn84U4tI8d3GNJmGT/eXe3ZuOXN9kTWmI= +go.opentelemetry.io/otel/sdk v1.27.0/go.mod h1:Ha9vbLwJE6W86YstIywK2xFfPjbWlCuwPtMkKdz/Y4A= +go.opentelemetry.io/otel/trace v1.27.0 h1:IqYb813p7cmbHk0a5y6pD5JPakbVfftRXABGt5/Rscw= +go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39SuvvstaLBl+l4= +go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/automaxprocs v1.5.1 h1:e1YG66Lrk73dn4qhg8WFSvhF0JuFQF0ERIp4rpuV8Qk= +go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= +go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= +go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= -google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= -google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= -google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e h1:SkdGTrROJl2jRGT/Fxv5QUf9jtdKCQh4KQJXbXVLAi0= +google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e/go.mod h1:LweJcLbyVij6rCex8YunD8DYR5VDonap/jYl3ZRxcIU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From a5984fb5ce583ece505a9414798bf2c27da96acc Mon Sep 17 00:00:00 2001 From: James Chao Date: Fri, 20 Dec 2024 22:48:24 -0800 Subject: [PATCH 138/155] change the poke keyword from primary to root --- util/validator.go | 2 +- util/validator_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util/validator.go b/util/validator.go index 2bab589..bbd269e 100644 --- a/util/validator.go +++ b/util/validator.go @@ -65,7 +65,7 @@ func ValidatePokeQuery(values url.Values) (string, error) { return routeStr, nil } - return "primary", nil + return "root", nil } func ValidateQueryParams(r *http.Request, validSubdocIdMap map[string]int, fields log.Fields) error { diff --git a/util/validator_test.go b/util/validator_test.go index 4cfb97a..83ac7f2 100644 --- a/util/validator_test.go +++ b/util/validator_test.go @@ -91,7 +91,7 @@ func TestValidatePokeQuery(t *testing.T) { delete(values, "doc") s, err = ValidatePokeQuery(values) assert.NilError(t, err) - assert.Equal(t, s, "primary") + assert.Equal(t, s, "root") values["doc"] = []string{ "primary", From bf96308d50cb352dd44ddf85037a31561c78e0be Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Mon, 23 Dec 2024 00:48:59 -0800 Subject: [PATCH 139/155] Set Otel Span attribute for x-cl-expt --- http/poke_handler.go | 8 ++++++++ http/webconfig_server.go | 16 ---------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/http/poke_handler.go b/http/poke_handler.go index fb25745..24d1d69 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -30,11 +30,18 @@ import ( "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" ) +const moracideTag = "X-Cl-Experiment" + func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { + moracideTagVal := r.Header.Get(moracideTag) + if moracideTagVal == "" { + moracideTagVal = "false" + } // tracing propagation ctx := r.Context() @@ -42,6 +49,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) span := trace.SpanFromContext(ctx) defer span.End() + span.SetAttributes(attribute.String(moracideTag, moracideTagVal)) tracer := otel.Tracer(s.otelTracer.appName) ctx, childSpan := tracer.Start(ctx, s.otelTracer.opName) defer childSpan.End() diff --git a/http/webconfig_server.go b/http/webconfig_server.go index daf331e..cd33e81 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -21,7 +21,6 @@ import ( "context" "crypto/tls" "encoding/base64" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -965,21 +964,6 @@ func GetResponseLogObjs(rbytes []byte) (interface{}, string) { return itf, "" } -// extract traceparent from the header -func (s *WebconfigServer) parseTraceparent(r *http.Request) (traceID string, traceFlags string) { - inTraceparent := r.Header.Get(common.HeaderTraceparent) - if len(inTraceparent) == 55 { - traceID = inTraceparent[3:35] - traceFlags = inTraceparent[53:55] - } - return -} - -func hexStringToBytes(hexString string) []byte { - bytes, _ := hex.DecodeString(hexString) - return bytes -} - func (s *WebconfigServer) ForwardKafkaMessage(kbytes []byte, m *common.EventMessage, fields log.Fields) { tfields := common.CopyCoreLogFields(fields) From 228e9111d0cb89f772b3c960069fe0de3b565302 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Sun, 29 Dec 2024 21:45:46 -0800 Subject: [PATCH 140/155] Migrate/adapt/rewrite tracing --- config/sample_webconfig.conf | 16 ++- go.mod | 3 + http/http_client.go | 60 +++++++- http/otel.go | 208 ---------------------------- http/poke_handler.go | 19 --- http/response.go | 36 +++++ http/router.go | 5 +- http/webconfig_server.go | 58 ++++---- main.go | 2 +- tracing/ctx.go | 19 +++ tracing/homegrown.go | 230 +++++++++++++++++++++++++++++++ tracing/homegrown_test.go | 231 +++++++++++++++++++++++++++++++ tracing/otel.go | 256 +++++++++++++++++++++++++++++++++++ tracing/span.go | 190 ++++++++++++++++++++++++++ tracing/tracer.go | 99 ++++++++++++++ 15 files changed, 1168 insertions(+), 264 deletions(-) delete mode 100644 http/otel.go create mode 100644 tracing/ctx.go create mode 100644 tracing/homegrown.go create mode 100644 tracing/homegrown_test.go create mode 100644 tracing/otel.go create mode 100644 tracing/span.go create mode 100644 tracing/tracer.go diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 6d2b232..e5e93f8 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -4,17 +4,23 @@ webconfig { } panic_exit_enabled = false - traceparent_parent_id = "0000000000000001" - tracestate_vendor_id = "webconfig" - opentelemetry { + tracing { + moracide_tag_prefix = "X-Cl-Experiment" + homegrown_algorithm { + app_id = "0000000000000001" + xpc_trace_propagation = false + xpc_trace_generation = false + } + opentelemetry { + enabled = false endpoint = "127.0.0.1:4318" + operation_name = "http.request" // Allowed values: "noop", "stdout", "http" // "noop" will generate no trace // "stdout" will use stdoutTracer and output spans to stdout // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector provider = "noop" - env_name = "dev" - operation_name = "http.request" + } } // build info diff --git a/go.mod b/go.mod index 95108f9..e085f6d 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.2.0 github.com/sirupsen/logrus v1.9.0 + github.com/stretchr/testify v1.9.0 github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 @@ -56,6 +57,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -73,4 +75,5 @@ require ( google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/http/http_client.go b/http/http_client.go index 9661cbf..5ec0e72 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -36,9 +36,9 @@ import ( "github.com/go-akka/configuration" "github.com/google/uuid" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/tracing" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" - "go.opentelemetry.io/otel/propagation" ) const ( @@ -112,6 +112,13 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl func (c *HttpClient) Do(ctx context.Context, method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { fields := common.FilterLogFields(auditFields) + var respMoracideTagsFound bool + defer func(found *bool) { + if !*found { + log.Debugf("http_client: no moracide tags in response") + } + }(&respMoracideTagsFound) + // verify a response is received var req *http.Request var err error @@ -152,6 +159,7 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h req.Header.Set(common.HeaderUserAgent, c.userAgent) } + c.addMoracideTags(header, auditFields) logHeader := header.Clone() auth := logHeader.Get("Authorization") if len(auth) > 0 { @@ -207,9 +215,6 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h startTime := time.Now() - // the core http call - propagator := propagation.TraceContext{} - propagator.Inject(ctx, propagation.HeaderCarrier(req.Header)) res, err := c.Client.Do(req) // err should be *url.Error @@ -219,6 +224,31 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h delete(fields, bodyKey) + // We want to capture any errs returned by server + // i.e. timeout, 503 etc. wouldn't have a valid resp, but possible that + // the err returned by http.Do actually includes an err returned by the backend + // In which case, resp would be non-nil + moracideTagPrefix := strings.ToLower(tracing.GetMoracideTagPrefix()) + if res != nil { + for headerKey, headerVals := range res.Header { + if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { + respMoracideTagsFound = true + log.Debugf("http_client: moracide tag %s = %s found in response", headerKey, headerVals[0]) + if len(headerVals) > 1 { + log.Debugf("Tracing: moracide tag key = %s, has multiple values = %+v", headerKey, headerVals) + } + val := "false" + for _, v := range headerVals { + if v == "true" { + val = v + break + } + } + auditFields["resp_"+headerKey] = headerVals[0] + log.Debugf("Tracing: found moracide tag in response key = %s, val = %s", headerKey, val) + } + } + } var endMessage string if retry > 0 { endMessage = fmt.Sprintf("%v retry=%v ends", loggerName, retry) @@ -394,3 +424,25 @@ func (c *HttpClient) StatusHandler(status int) StatusHandlerFunc { } return nil } + +// addMoracideTags - if ctx has a moracide tag as a header, add it to the headers +// Also add traceparent, tracestate headers +func (c *HttpClient) addMoracideTags(header http.Header, fields log.Fields) { + moracideTagPrefix := strings.ToLower("req_"+tracing.GetMoracideTagPrefix()) + for key, val := range fields { + if key == "out_traceparent" { + header.Set("traceparent", val.(string)) + } + if key == "out_tracestate" { + header.Set("tracestate", val.(string)) + } + if len(key) < 5 { + // Should be of the form "req_x-cl-experiment" + continue + } + if strings.HasPrefix(strings.ToLower(key), moracideTagPrefix) { + log.Debugf("Adding moracide tag %s = %s to outgoing req", key, val) + header.Set(key[4:], val.(string)) + } + } +} diff --git a/http/otel.go b/http/otel.go deleted file mode 100644 index 468035f..0000000 --- a/http/otel.go +++ /dev/null @@ -1,208 +0,0 @@ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 -*/ -package http - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" - "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/sdk/resource" - sdktrace "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/trace" - "go.opentelemetry.io/otel/trace/noop" - - "github.com/go-akka/configuration" - log "github.com/sirupsen/logrus" -) - -// Tracing contains the core dependencies to make tracing possible across an application. -type otelTracing struct { - providerName string - envName string - appName string - opName string - tracerProvider trace.TracerProvider - propagator propagation.TextMapPropagator - tracer trace.Tracer -} - -type providerConstructor func(conf *configuration.Config) (trace.TracerProvider, error) - -var ( - ErrTracerProviderNotFound = errors.New("TracerProvider builder could not be found") - ErrTracerProviderBuildFailed = errors.New("Failed building TracerProvider") - providersConfig = map[string]providerConstructor{ - "http": httpTraceProvider, - "stdout": stdoutTraceProvider, - "noop": noopTraceProvider, - } - - otelTracer otelTracing -) - -// DefaultTracerProvider is used when no provider is given. -// The Noop tracer provider turns all tracing related operations into -// noops essentially disabling tracing. -const defaultTracerProvider = "noop" - -// newOtel creates a structure with components that apps can use to initialize OpenTelemetry -// tracing instrumentation code. -func newOtel(conf *configuration.Config) (*otelTracing, error) { - if IsNoopTracing(conf) { - log.Debug("open telemetry tracing disabled (noop)") - } else { - log.Debug("opentelemetry tracing enabled") - } - - otelTracer.appName = conf.GetString("webconfig.app_name") - otelTracer.providerName = conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - otelTracer.envName = conf.GetString("webconfig.opentelemetry.env_name", "dev") - otelTracer.opName = conf.GetString("webconfig.opentelemetry.operation_name", "http.request") - tracerProvider, err := newTracerProvider(conf) - if err != nil { - return &otelTracer, err - } - otelTracer.tracerProvider = tracerProvider - otel.SetTracerProvider(tracerProvider) - - // Set up propagator. - prop := newPropagator() - otelTracer.propagator = prop - otel.SetTextMapPropagator(prop) - - otelTracer.tracer = otel.Tracer(otelTracer.appName) - return &otelTracer, nil -} - -// IsNoopTracing returns true if the provider is set to "noop" -func IsNoopTracing(conf *configuration.Config) bool { - providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - return strings.EqualFold(providerName, "noop") -} - -// TracerProvider returns the tracer provider component. By default, the noop -// tracer provider is returned. -func (t otelTracing) TracerProvider() trace.TracerProvider { - if t.tracerProvider == nil { - return noop.NewTracerProvider() - } - return t.tracerProvider -} - -// Propagator returns the component that helps propagate trace context across -// API boundaries. By default, a W3C Trace Context format propagator is returned. -func (t otelTracing) Propagator() propagation.TextMapPropagator { - if t.propagator == nil { - return propagation.TraceContext{} - } - return t.propagator -} - -func newPropagator() propagation.TextMapPropagator { - return propagation.NewCompositeTextMapPropagator( - propagation.TraceContext{}, - propagation.Baggage{}, - ) -} - -// newTracerProvider creates the TracerProvider based on config setting -// If no config setting, a noop tracerProvider will be returned. -func newTracerProvider(conf *configuration.Config) (trace.TracerProvider, error) { - providerName := conf.GetString("webconfig.opentelemetry.provider", defaultTracerProvider) - if len(providerName) == 0 { - providerName = defaultTracerProvider - } - // Handling camelcase of provider. - providerName = strings.ToLower(providerName) - providerConfig := providersConfig[providerName] - if providerConfig == nil { - return nil, fmt.Errorf("%w for provider %s", ErrTracerProviderNotFound, providerName) - } - - traceProvider, err := providerConfig(conf) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) - } - return traceProvider, nil -} - -func noopTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - return noop.NewTracerProvider(), nil -} - -func stdoutTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - option := stdouttrace.WithPrettyPrint() - exporter, err := stdouttrace.New(option) - if err != nil { - return nil, err - } - tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), - sdktrace.WithBatcher(exporter, - // Default is 5s. Set to 1s for demonstrative purposes. - sdktrace.WithBatchTimeout(time.Second)), - sdktrace.WithResource( - resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String(otelTracer.appName), - semconv.ServiceNamespaceKey.String(otelTracer.envName), - ), - ), - ) - return tp, nil -} - -func httpTraceProvider(conf *configuration.Config) (trace.TracerProvider, error) { - // Send traces over HTTP - endpoint := conf.GetString("webconfig.opentelemetry.endpoint") - if endpoint == "" { - return nil, ErrTracerProviderBuildFailed - } - exporter, err := otlptracehttp.New(context.Background(), - otlptracehttp.WithEndpoint(endpoint), - otlptracehttp.WithInsecure(), - ) - if err != nil { - return nil, fmt.Errorf("%w: %v", ErrTracerProviderBuildFailed, err) - } - - return sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exporter), - sdktrace.WithResource( - resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String(otelTracer.appName), - semconv.ServiceNamespaceKey.String(otelTracer.envName), - ), - ), - ), nil -} - -func (s *WebconfigServer) OtelShutdown() { - sdkTraceProvider, ok := s.otelTracer.tracerProvider.(*sdktrace.TracerProvider) - if ok && sdkTraceProvider != nil { - sdkTraceProvider.Shutdown(context.TODO()) - } -} diff --git a/http/poke_handler.go b/http/poke_handler.go index 24d1d69..902441b 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -29,31 +29,12 @@ import ( "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/db" "github.com/rdkcentral/webconfig/util" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/propagation" - "go.opentelemetry.io/otel/trace" ) -const moracideTag = "X-Cl-Experiment" - func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { - moracideTagVal := r.Header.Get(moracideTag) - if moracideTagVal == "" { - moracideTagVal = "false" - } // tracing propagation ctx := r.Context() - propagator := propagation.TraceContext{} - ctx = propagator.Extract(ctx, propagation.HeaderCarrier(r.Header)) - span := trace.SpanFromContext(ctx) - defer span.End() - span.SetAttributes(attribute.String(moracideTag, moracideTagVal)) - tracer := otel.Tracer(s.otelTracer.appName) - ctx, childSpan := tracer.Start(ctx, s.otelTracer.opName) - defer childSpan.End() - // handler params := mux.Vars(r) mac := params["mac"] diff --git a/http/response.go b/http/response.go index 7107bfa..5fbd1bc 100644 --- a/http/response.go +++ b/http/response.go @@ -21,8 +21,11 @@ import ( "encoding/json" "fmt" "net/http" + "strings" + log "github.com/sirupsen/logrus" "github.com/rdkcentral/webconfig/common" + "github.com/rdkcentral/webconfig/tracing" ) const ( @@ -54,6 +57,7 @@ func WriteByMarshal(w http.ResponseWriter, status int, o interface{}) { return } w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(status) w.Write(rbytes) } @@ -107,6 +111,7 @@ func WriteOkResponseByTemplate(w http.ResponseWriter, dataStr string, state int, rbytes = []byte(fmt.Sprintf(OkResponseTemplate, s, stateText, updatedTime)) } w.Header().Set(common.HeaderContentType, common.HeaderApplicationJson) + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(http.StatusOK) w.Write(rbytes) } @@ -114,6 +119,7 @@ func WriteOkResponseByTemplate(w http.ResponseWriter, dataStr string, state int, // this is used to return default tr-181 payload while the cpe is not in the db func WriteContentTypeAndResponse(w http.ResponseWriter, rbytes []byte, version string, contentType string) { w.Header().Set(common.HeaderContentType, contentType) + addMoracideTagsAsResponseHeaders(w) w.Header().Set(common.HeaderEtag, version) w.WriteHeader(http.StatusOK) w.Write(rbytes) @@ -152,12 +158,42 @@ func WriteResponseBytes(w http.ResponseWriter, rbytes []byte, statusCode int, va if len(vargs) > 0 { w.Header().Set(common.HeaderContentType, vargs[0]) } + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(statusCode) w.Write(rbytes) } func WriteFactoryResetResponse(w http.ResponseWriter) { w.Header().Set(common.HeaderContentType, common.MultipartContentType) + addMoracideTagsAsResponseHeaders(w) w.WriteHeader(http.StatusOK) w.Write([]byte{}) } + +func addMoracideTagsAsResponseHeaders(w http.ResponseWriter) { + xw, ok := w.(*XResponseWriter) + if !ok { + return + } + + reqMoracideTagPrefix := strings.ToLower("req_"+tracing.GetMoracideTagPrefix()) + respMoracideTagPrefix := strings.ToLower("resp_"+tracing.GetMoracideTagPrefix()) + fields := xw.Audit() + moracideTags := make(map[string]string) + for key, val := range fields { + if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { + log.Debugf("Adding moracide tag from req %s = %s to response", key, val) + moracideTags[key[4:]] = val.(string) + } + if strings.HasPrefix(strings.ToLower(key), respMoracideTagPrefix) { + log.Debugf("Adding moracide tag from resp %s = %s to response", key, val) + realKey := key[5:] + if existingVal, ok := moracideTags[realKey]; !ok || (ok && existingVal != "true") { + moracideTags[realKey] = val.(string) + } + } + } + for key, val := range moracideTags { + w.Header().Set(key, val) + } +} diff --git a/http/router.go b/http/router.go index 97348c1..4969326 100644 --- a/http/router.go +++ b/http/router.go @@ -19,6 +19,7 @@ package http import ( "github.com/gorilla/mux" + "github.com/rdkcentral/webconfig/tracing" ) func (s *WebconfigServer) AddBaseRoutes(testOnly bool, router *mux.Router) { @@ -95,9 +96,9 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.TestingMiddleware) } else { if s.ServerApiTokenAuthEnabled() { - sub2.Use(s.ApiMiddleware) + sub2.Use(tracing.SpanMiddleware, s.ApiMiddleware) } else { - sub2.Use(s.NoAuthMiddleware) + sub2.Use(tracing.SpanMiddleware, s.NoAuthMiddleware) } } sub2.HandleFunc("", s.PokeHandler).Methods("POST") diff --git a/http/webconfig_server.go b/http/webconfig_server.go index cd33e81..fbc7d0c 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -40,6 +40,7 @@ import ( "github.com/rdkcentral/webconfig/db/cassandra" "github.com/rdkcentral/webconfig/db/sqlite" "github.com/rdkcentral/webconfig/security" + "github.com/rdkcentral/webconfig/tracing" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" ) @@ -93,6 +94,7 @@ type WebconfigServer struct { *MqttConnector *UpstreamConnector sarama.AsyncProducer + *tracing.XpcTracer tlsConfig *tls.Config notLoggedHeaders []string metricsEnabled bool @@ -109,7 +111,6 @@ type WebconfigServer struct { traceparentParentID string tracestateVendorID string supplementaryAppendingEnabled bool - otelTracer *otelTracing // For OpenTelemetry Tracing kafkaProducerEnabled bool kafkaProducerTopic string upstreamProfilesEnabled bool @@ -254,13 +255,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer validPartners = append(validPartners, strings.ToLower(p)) } - traceparentParentID := conf.GetString("webconfig.traceparent_parent_id", defaultTraceparentParentID) - tracestateVendorID := conf.GetString("webconfig.tracestate_vendor_id", defaultTracestateVendorID) - otelTracer, err := newOtel(conf) - if err != nil { - // Just log err and continue - log.Error("Could not initialize open telemetry for tracing, but continuing") - } + xpcTracer := tracing.NewXpcTracer(conf) supplementaryAppendingEnabled := conf.GetBoolean("webconfig.supplementary_appending_enabled", defaultSupplementaryAppendingEnabled) @@ -321,9 +316,6 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer validateMacEnabled: validateMacEnabled, validPartners: validPartners, jwksEnabled: jwksEnabled, - traceparentParentID: traceparentParentID, - tracestateVendorID: tracestateVendorID, - otelTracer: otelTracer, supplementaryAppendingEnabled: supplementaryAppendingEnabled, kafkaProducerEnabled: kafkaProducerEnabled, kafkaProducerTopic: kafkaProducerTopic, @@ -331,11 +323,16 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer queryParamsValidationEnabled: queryParamsValidationEnabled, minTrust: minTrust, validSubdocIdMap: validSubdocIdMap, + XpcTracer: xpcTracer, } return ws } +func (s *WebconfigServer) Stop() { + tracing.StopXpcTracer() +} + func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { fn := func(w http.ResponseWriter, r *http.Request) { xw := NewXResponseWriter(w) @@ -749,27 +746,34 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques traceId = traceparent[3:35] } - // extract tracestate from the header - tracestate := r.Header.Get(common.HeaderTracestate) - // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { auditId = util.GetAuditId() } + + // traceparent handling for E2E tracing + xpcTrace := tracing.NewXpcTrace(r) + headerMap := util.HeaderToMap(header) fields := log.Fields{ - "path": r.URL.String(), - "method": r.Method, - "audit_id": auditId, - "remote_ip": remoteIp, - "host_name": host, - "header": headerMap, - "logger": "request", - "trace_id": traceId, - "app_name": s.AppName(), - "traceparent": traceparent, - "tracestate": tracestate, + "path": r.URL.String(), + "method": r.Method, + "audit_id": auditId, + "remote_ip": remoteIp, + "host_name": host, + "header": headerMap, + "logger": "request", + "trace_id": traceId, + "app_name": s.AppName(), + "traceparent": xpcTrace.ReqTraceparent, + "tracestate": xpcTrace.ReqTracestate, + "out_traceparent": xpcTrace.OutTraceparent, + "out_tracestate": xpcTrace.OutTracestate, + "xpc_trace": xpcTrace, + } + for key, val := range xpcTrace.ReqMoracideTags { + fields["req_"+key] = val } userAgent := r.UserAgent() @@ -828,6 +832,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques log.WithFields(tfields).Info("request starts") } + xwriter.LogDebug(r, "tracing", fmt.Sprintf("Trace final out_traceparent %s out_traceState %s", xpcTrace.OutTraceparent, xpcTrace.OutTracestate)) return xwriter } @@ -897,6 +902,9 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { fields["duration"] = duration fields["logger"] = "request" + tracing.SetSpanStatusCode(fields) + tracing.SetSpanMoracideTags(fields) + var userAgent string if itf, ok := fields["user_agent"]; ok { userAgent = itf.(string) diff --git a/main.go b/main.go index 8a47cb9..284012c 100644 --- a/main.go +++ b/main.go @@ -61,7 +61,7 @@ func main() { panic(err) } server := wchttp.NewWebconfigServer(sc, false) - defer server.OtelShutdown() + defer server.Stop() // setup logging logFile := server.GetString("webconfig.log.file") diff --git a/tracing/ctx.go b/tracing/ctx.go new file mode 100644 index 0000000..204e084 --- /dev/null +++ b/tracing/ctx.go @@ -0,0 +1,19 @@ +package tracing + +import "context" + +type contextKey string + +func (c contextKey) String() string { + return string(c) +} + +// SetContext - setting context for logging +func SetContext(ctx context.Context, ctxName string, ctxValue interface{}) context.Context { + return context.WithValue(ctx, contextKey(ctxName), ctxValue) +} + +// GetContext - getting context value from context +func GetContext(ctx context.Context, key string) interface{} { + return ctx.Value(contextKey(key)) +} diff --git a/tracing/homegrown.go b/tracing/homegrown.go new file mode 100644 index 0000000..40f4a86 --- /dev/null +++ b/tracing/homegrown.go @@ -0,0 +1,230 @@ +/** + * @license + * Copyright Comcast. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @author RV + +/* + W3C Doc: https://www.w3.org/TR/trace-context-1 + + Usage: + 1. AppID should be set in the config + Config Template (from HC): + ------------------------------------------- + "Tracing": { + // This identifies the microservice + "AppID": "0000000000000001" + }, + ------------------------------------------- + 2. Call GetTracingHeader with a http Request, and it will return a modified + traceparent/tracestate header + 3. Set it in req.Header + 4. If the generate flag is set in GetTracingHeader and if the input req doesn't + have a traceparent header, these two headers will be created + TODO: + Also pass a Kafka message as an input to do the above + + Spec: + traceparent header has 4 sections + "---" + version is 2 digits, traceID is 32 digits, callerIF is 16, and flags is 2 + if traceparent is present in headers, replace the callerID with app's ID + + Template - hc's ID defaults to "0000000000000001" and can be changed in config + + If tracestate header exists, add "=," to the beginning of the header + else set it to "=" + + As of now, we don't generate traceparent if the header is not passed in + Any trace flag setting that tells us to regenerate the traceID is ignored. + Version, TraceID, and trace flags are passed as-is. Only CallerID is modified + TraceState will be generated if it doesn't exist + + Epic: XPC-16688 + HC Ticket: ODP-25026 + Xapproxy Ticket: XPC-17946 + + Changes: + otel implements their own traceparent/tracestate propagation and this clashes with + our home-grown propagation. If otel is enabled, suppress the home-grown propagation. + TODO: Change homegrown propagation too to a per API basis instead of per app basis +*/ + +package tracing + +import ( + "fmt" + "math/rand" + "net/http" + "strings" + // "sync" + "time" + log "github.com/sirupsen/logrus" + "github.com/rdkcentral/webconfig/common" +) + +const ( + // These are two completely internal + traceVersion = "01" + traceFlags = "00" + + defaultAppID = "0000000000000001" + + traceAppIDNotSetErr = "Tracing: AppID not set" +) + +type TraceHeader struct { + Key, Value string +} + +func GetTraceHeaders(r *http.Request, generateTrace bool) ([]TraceHeader, error) { + // This is not concurrency safe (hat tip to Jay) + // Hence it is instantiated in the goroutine that handles the API req + // Google search says this is inexpensive and acceptable + // Prefer not to use a mutex as multiple Kafka reqs can wait on this lock + + randGen := rand.NewSource(time.Now().UnixNano()) + + headers := make([]TraceHeader, 0) + traceParent := r.Header.Get("traceparent") + traceComponents := strings.Split(traceParent, "-") + if len(traceComponents) != 4 { + // traceparent header is either incorrectly formatted or doesn't exist + if !generateTrace { + // Don't create traceparent header if flag is not set + // Don't bother about tracestate header + return headers, nil + } + + if xpcTracer.appID == "" { + return headers, fmt.Errorf(traceAppIDNotSetErr) + } + // Create a new traceparent header + traceParent = fmt.Sprintf("%s-%016x%016x-%s-%s", + traceVersion, + // genInt63(randGen), genInt63(randGen), + randGen.Int63(), randGen.Int63(), + xpcTracer.appID, + traceFlags) + } else { + // Replace the appID part + if xpcTracer.appID == "" { + return headers, fmt.Errorf(traceAppIDNotSetErr) + } + // Create a new traceparent header + traceParent = fmt.Sprintf("%s-%s-%s-%s", + traceComponents[0], + traceComponents[1], + xpcTracer.appID, + traceComponents[3]) + } + headers = append(headers, TraceHeader{ + Key: "traceparent", + Value: traceParent, + }) + + traceState := r.Header.Get("tracestate") + if traceState == "" { + // Create tracestate header if it doesn't exist + traceState = fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID) + } else { + // Add current app to traceState + traceState = fmt.Sprintf("%s=%s,%s", xpcTracer.appName, xpcTracer.appID, traceState) + } + headers = append(headers, TraceHeader{ + Key: "tracestate", + Value: traceState, + }) + return headers, nil +} + +func tryHomegrownTpTs(r *http.Request, xpcTrace *XpcTrace) { + // if homegrownTracePropagation is true, we modify existing traceparent/tracestate and forward them + // but we will not generate them if they don't exist + // if homegrownTraceGeneration is true, and there is no tp/ts, we generate it + + if !xpcTracer.homegrownTracePropagation { + return + } + // ctx := r.Context() + log.Debug("Tracing: using homegrown traceparent propagation/generation") + if traceHeaders, err := GetTraceHeaders(r, xpcTracer.homegrownTraceGeneration); err == nil { + // TODO: Change homegrown propagation too to a per API basis instead of per app basis + // Reverse engineer otel basically + for _, h := range traceHeaders { + log.Debugf("Tracing: %s set to %s", h.Key, h.Value) + if h.Key == common.HeaderTraceparent { + xpcTrace.OutTraceparent = h.Value + } + if h.Key == common.HeaderTracestate { + xpcTrace.OutTracestate = h.Value + } + } + } else { + log.Errorf("Error in homegrown traceparent handling %+v", err) + } +} + +/* + // Benchmarking expt to compare mutex based random vs one source per goroutine + var ( + // This is for a benchmarking expt + global bool + globalLock sync.Mutex + globalRandGen = rand.NewSource(time.Now().UnixNano()) + + // Sample Benchmarking Results + // BenchmarkSharedRandGen-16 352419 3052 ns/op + // BenchmarkIndependentRandGen-16 670383 2755 ns/op + ) + + func setGlobal(g bool) { + global = g + } + + func genInt63(randGen rand.Source) int64 { + if global { + globalLock.Lock() + i := globalRandGen.Int63() + globalLock.Unlock() + return i + } + return randGen.Int63() + } +*/ + +/* + traceparentParentID string + tracestateVendorID string + otelEnabled bool + traceparentParentID: traceparentParentID, + tracestateVendorID: tracestateVendorID, +func (s *WebconfigServer) TraceparentParentID() string { + return s.traceparentParentID +} + +func (s *WebconfigServer) SetTraceparentParentID(x string) { + s.traceparentParentID = x +} + +func (s *WebconfigServer) TracestateVendorID() string { + return s.tracestateVendorID +} + +func (s *WebconfigServer) SetTracestateVendorID(x string) { + s.tracestateVendorID = x +} +*/ diff --git a/tracing/homegrown_test.go b/tracing/homegrown_test.go new file mode 100644 index 0000000..6e36c36 --- /dev/null +++ b/tracing/homegrown_test.go @@ -0,0 +1,231 @@ +/** + * @license + * Copyright Comcast. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @author RV + +package tracing + +import ( + "bytes" + "fmt" + "net/http" + "strings" + // "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + testTraceID = "0123456789abcdef0123456789abcdef" + testCallerID = "fedcba9876543210" + testTraceParent = traceVersion + "-" + testTraceID + "-" + testCallerID + "-" + traceFlags + + testCallerName = "test" + testTraceState = testCallerName + "=" + testCallerID + + testMyName = "whatever" + testMyID = "0123456789abcdef" +) + +func TestAppIDNotSet(t *testing.T) { + // Mimic appName/appID not set by explicitly setting them to empty strs + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = "" + xpcTracer.appName = "" + + req := makeReq(t) + req.Header.Add("traceparent", testTraceParent) + _, err := GetTraceHeaders(req, false) + require.Equal(t, traceAppIDNotSetErr, err.Error()) + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func TestNoGenerate(t *testing.T) { + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = testMyID + xpcTracer.appName = testMyName + + req := makeReq(t) + headers, _ := GetTraceHeaders(req, false) + require.Equal(t, 0, len(headers)) + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func TestOnlyParentHeader(t *testing.T) { + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = testMyID + xpcTracer.appName = testMyName + + req := makeReq(t) + req.Header.Add("traceparent", testTraceParent) + + headers, _ := GetTraceHeaders(req, false) + require.Equal(t, 2, len(headers)) + validateTraceParentHeader(t, headers) + for _, h := range headers { + if h.Key == "tracestate" { + require.Equal(t, fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID), h.Value) + } + } + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func TestOnlyStateHeader(t *testing.T) { + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = testMyID + xpcTracer.appName = testMyName + + req := makeReq(t) + req.Header.Add("tracestate", testTraceState) + headers, _ := GetTraceHeaders(req, false) + require.Equal(t, 0, len(headers)) + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func TestParentPlusStateHeaders(t *testing.T) { + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = testMyID + xpcTracer.appName = testMyName + + req := makeReq(t) + req.Header.Add("traceparent", testTraceParent) + req.Header.Add("tracestate", testTraceState) + headers, _ := GetTraceHeaders(req, false) + require.Equal(t, 2, len(headers)) + validateTraceParentHeader(t, headers) + for _, h := range headers { + if h.Key == "tracestate" { + require.Equal(t, fmt.Sprintf("%s=%s,%s", xpcTracer.appName, xpcTracer.appID,testTraceState), h.Value) + } + } + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func TestTraceHeaderGeneration(t *testing.T) { + appID := xpcTracer.appID + appName := xpcTracer.appName + xpcTracer.appID = testMyID + xpcTracer.appName = testMyName + + testTraceHeaderGeneration(t) + + xpcTracer.appID = appID + xpcTracer.appName = appName +} + +func validateTraceParentHeader(t *testing.T, headers []TraceHeader) { + for _, h := range headers { + if h.Key == "traceparent" { + traceComponents := strings.Split(h.Value, "-") + require.Equal(t, 4, len(traceComponents), 4) + require.Equal(t, traceVersion, traceComponents[0]) + require.Equal(t, testTraceID, traceComponents[1]) + require.Equal(t, xpcTracer.appID, traceComponents[2]) + require.Equal(t, traceFlags, traceComponents[3]) + } + } +} + +func makeReq(t *testing.T) *http.Request { + url := "/testurl" + var body []byte + req, err := http.NewRequest("GET", url, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + return req +} + +func testTraceHeaderGeneration(t *testing.T) { + req := makeReq(t) + headers, _ := GetTraceHeaders(req, true) + require.Equal(t, 2, len(headers)) + for _, h := range headers { + if h.Key == "traceparent" { + traceComponents := strings.Split(h.Value, "-") + require.Equal(t, 4, len(traceComponents)) + require.Equal(t, traceVersion, traceComponents[0]) + // We don't know the ID here, we can only check it is 32 digits long + require.Equal(t, 32, len(traceComponents[1])) + require.Equal(t, xpcTracer.appID, traceComponents[2]) + require.Equal(t, traceFlags, traceComponents[3]) + } + if h.Key == "tracestate" { + require.Equal(t, fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID), h.Value) + } + } +} + +/* + // These tests/benchmarks are to test mutex based random generation + // We are not going to use mutex as of now, as using one source + // per goroutine approach is faster + func TestTraceHeaderGenerationGlobal(t *testing.T) { + setGlobal(true) + testTraceHeaderGeneration(t) + setGlobal(false) + } + + func BenchmarkSharedRandGen(b *testing.B) { + setGlobal(true) + url := "/testurl" + var body []byte + req, _ := http.NewRequest("GET", url, bytes.NewReader(body)) + var wg sync.WaitGroup + for n := 0; n < b.N; n++ { + wg.Add(1) + go func() { + defer wg.Done() + GetTraceHeaders(req, true) + }() + } + wg.Wait() + setGlobal(false) + } + + func BenchmarkIndependentRandGen(b *testing.B) { + url := "/testurl" + var body []byte + req, _ := http.NewRequest("GET", url, bytes.NewReader(body)) + var wg sync.WaitGroup + + for n := 0; n < b.N; n++ { + wg.Add(1) + go func() { + defer wg.Done() + GetTraceHeaders(req, true) + }() + } + wg.Wait() + } +*/ diff --git a/tracing/otel.go b/tracing/otel.go new file mode 100644 index 0000000..22e4728 --- /dev/null +++ b/tracing/otel.go @@ -0,0 +1,256 @@ +package tracing + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/gorilla/mux" + "github.com/go-akka/configuration" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.10.0" + "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" + + log "github.com/sirupsen/logrus" +) + +type providerConstructor func() (oteltrace.TracerProvider, error) + +var ( + providerBuilders = map[string]providerConstructor{ + "http": otelHttpTraceProvider, + "stdout": otelStdoutTraceProvider, + "noop": otelNoopTraceProvider, + } +) + +// defaultOtelTracerProvider is used when no provider is given. +// The Noop tracer provider turns all tracing related operations into +// noops essentially disabling tracing. +const defaultOtelTracerProvider = "noop" + +// initOtel - initialize OpenTelemetry constructs +// tracing instrumentation code. +func otelInit(conf *configuration.Config) { + xpcTracer.OtelEnabled = conf.GetBoolean("webconfig.tracing.otel.enabled") + if !xpcTracer.OtelEnabled { + return + } + xpcTracer.otelProvider = strings.ToLower(conf.GetString("webconfig.tracing.otel.provider", defaultOtelTracerProvider)) + if xpcTracer.otelProvider == "" { + xpcTracer.otelProvider = defaultOtelTracerProvider + } + if xpcTracer.otelProvider == defaultOtelTracerProvider { + log.Debug("otel disabled, noop provider") + return + } + + log.Debug("otel enabled") + xpcTracer.otelEndpoint = conf.GetString("webconfig.tracing.otel.endpoint") + xpcTracer.otelOpName = conf.GetString("webconfig.tracing.otel.operation_name") + + if providerBuilder := providerBuilders[xpcTracer.otelProvider]; providerBuilder == nil { + log.Errorf("no builder func for otel provider %s", xpcTracer.otelProvider) + return + } else { + var err error + if xpcTracer.otelTracerProvider, err = providerBuilder(); err != nil { + log.Errorf("building otel provider for %s failed with %v", xpcTracer.otelProvider, err) + return + } + } + otel.SetTracerProvider(xpcTracer.otelTracerProvider) + + // Set up propagator. + xpcTracer.otelPropagator = propagation.NewCompositeTextMapPropagator( + propagation.TraceContext{}, + propagation.Baggage{}, + ) + otel.SetTextMapPropagator(xpcTracer.otelPropagator) + + xpcTracer.otelTracer = otel.Tracer(xpcTracer.appName) +} + +func otelNoopTraceProvider() (oteltrace.TracerProvider, error) { + return noop.NewTracerProvider(), nil +} + +func otelStdoutTraceProvider() (oteltrace.TracerProvider, error) { + option := stdouttrace.WithPrettyPrint() + exporter, err := stdouttrace.New(option) + if err != nil { + return nil, err + } + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter), + sdktrace.WithBatcher(exporter, + // Default is 5s. Set to 1s for demonstrative purposes. + sdktrace.WithBatchTimeout(time.Second)), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(xpcTracer.appName), + semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), + ), + ), + ) + return tp, nil +} + +func otelHttpTraceProvider() (oteltrace.TracerProvider, error) { + // Send traces over HTTP + if xpcTracer.otelEndpoint == "" { + return nil, fmt.Errorf("building http otel provider failure, no endpoint specified") + } + exporter, err := otlptracehttp.New(context.Background(), + otlptracehttp.WithEndpoint(xpcTracer.otelEndpoint), + otlptracehttp.WithInsecure(), + ) + if err != nil { + return nil, fmt.Errorf("building http otel provider failed with %v", err) + } + + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String(xpcTracer.appName), + semconv.ServiceNamespaceKey.String(xpcTracer.appEnv), + ), + ), + ), nil +} + +func otelShutdown() { + sdkTraceProvider, ok := xpcTracer.otelTracerProvider.(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} + +// otelOpName should return "http.request" by default +func otelOpName() string { + opName := xpcTracer.otelOpName + if opName == "" { + opName = "http.request" + } + return opName +} + +func otelNewSpan(r *http.Request) (context.Context, oteltrace.Span) { + ctx := r.Context() + var otelSpan oteltrace.Span + if !xpcTracer.OtelEnabled { + return ctx, otelSpan + } + + pathTemplate := "placeholder" + if mux.CurrentRoute(r) != nil { // This can be nil in unit tests + var err error + pathTemplate, err = mux.CurrentRoute(r).GetPathTemplate() + if err != nil { + log.Debugf("unable to get path template: %v", err) + } + } + resourceName := r.Method + " " + pathTemplate + ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.HeaderCarrier(r.Header)) + + /* + required span attribute: HTTPMethodKey = attribute.Key("http.method") + required span attribute: HTTPRouteKey = attribute.Key("http.route") + required span attribute: HTTPStatusCodeKey = attribute.Key("http.status_code") + required span attribute: HTTPURLKey = attribute.Key("http.url") + custom Comcast attribute: X-Cl-Experiment: true/false + additional: env, operation.name, http.url_details.path + */ + ctx, otelSpan = xpcTracer.otelTracer.Start(ctx, otelOpName(), + oteltrace.WithSpanKind(oteltrace.SpanKindServer), + oteltrace.WithAttributes( + attribute.String("env", xpcTracer.appEnv), + attribute.String("http.method", r.Method), + attribute.String("http.route", pathTemplate), + attribute.String("http.url", r.URL.String()), + attribute.String("http.url_details.path", r.URL.Path), + attribute.String("operation.name", otelOpName()), + ), + ) + if xpcTracer.rgn != "" { + rgnAttr := attribute.String("region", xpcTracer.rgn) + otelSpan.SetAttributes(rgnAttr) + } + + log.Debugf("span started %s", resourceName) + log.Debugf("added span attribute key = env, value = %s", xpcTracer.appEnv) + log.Debugf("added span attribute key = http.method, value = %s", r.Method) + log.Debugf("added span attribute key = http.route, value = %s", pathTemplate) + log.Debugf("added span attribute key = http.url, value = %s", r.URL.String()) + log.Debugf("added span attribute key = http.url_details.path, value = %s", r.URL.Path) + log.Debugf("added span attribute key = operation.name, value = %s", otelOpName()) + + carrier := propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctx, carrier) + for key, val := range carrier { + ctx = SetContext(ctx, "otel_"+key, val) + log.Debugf("OtelSpanCreation: otel %s = %s", key, val) + } + + ctx = SetContext(ctx, "otel_span", otelSpan) + return ctx, otelSpan +} + +func otelSetStatusCode(span oteltrace.Span, statusCode int) { + statusAttr := attribute.Int("http.status_code", statusCode) + span.SetAttributes(statusAttr) + log.Debugf("added span attribute key = http.status_code, value = %d", statusCode) + + if statusCode >= http.StatusInternalServerError { + statusText := http.StatusText(statusCode) + span.SetStatus(codes.Error, statusText) + span.SetAttributes(attribute.String("http.response.error", statusText)) + log.Debugf("added span attribute key=http.response.error, value=%s", statusText) + } +} + +func otelEndSpan(span oteltrace.Span) { + if !xpcTracer.OtelEnabled { + return + } + span.End() +} + +func otelExtractParamsFromSpan(ctx context.Context, xpcTrace *XpcTrace) { + if !xpcTracer.OtelEnabled { + return + } + if tmp := GetContext(ctx, "otel_span"); tmp != nil { + if otelSpan, ok := tmp.(oteltrace.Span); ok { + xpcTrace.otelSpan = otelSpan + } + } + if xpcTrace.otelSpan == nil { + return + } + // if otel span is found, use the extracted traceparent and tracestate from the otel span + // We store the extracted values in ctx when we created the otel span + if tmp := GetContext(ctx, "otel_traceparent"); tmp != nil { + xpcTrace.otelTraceparent = tmp.(string) + log.Debugf("Tracing: otel traceparent = %s", xpcTrace.otelTraceparent) + xpcTrace.OutTraceparent = xpcTrace.otelTraceparent + } + if tmp := GetContext(ctx, "otel_tracestate"); tmp != nil { + xpcTrace.otelTracestate = tmp.(string) + log.Debugf("Tracing: otel tracestate = %s", xpcTrace.otelTracestate) + xpcTrace.OutTracestate = xpcTrace.otelTracestate + } +} diff --git a/tracing/span.go b/tracing/span.go new file mode 100644 index 0000000..137ac8b --- /dev/null +++ b/tracing/span.go @@ -0,0 +1,190 @@ +package tracing + +import ( + "net/http" + "strings" + + log "github.com/sirupsen/logrus" + + oteltrace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/attribute" + + "github.com/rdkcentral/webconfig/common" +) + +// SpanMiddleware is a middleware that creates a new span for each incoming request. +func SpanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // For Otel, create the span explicitly + if xpcTracer.OtelEnabled { + ctx, otelSpan := otelNewSpan(r) + r = r.WithContext(ctx) + defer otelEndSpan(otelSpan) + } + next.ServeHTTP(w, r) + }) +} + +// XpcTrace is a carrier/baggage struct to extract data from spans, request headers for usage later +// Store the trace in ctx for easy retrieval. +// Ideal place to store it is ofc, xw +// But because of legacy reasons, xw is not always available in the API flow +type XpcTrace struct { + // This is a bit of overengineering, but multiple tags are possible + // e.g. X-Cl-Experiment-1, X-Cl-Experiment-xapproxy, X-Cl-Experiement-webconfig-25.1.1.1... + // For every key found in either req or resp, an explicit value of true/false will be set as an otel attribute + // or an otel span attribute + ReqMoracideTags map[string]string // These are request headers prefixed with MoracideTagPrefix + // The response moracide tags are stored in xw.audit + + // traceparent, tracestate can be set as req headers, may be extracted from otel spans + // These need to be propagated to any http calls we make + // Order of priority; use the value extracted from otel span; + // if no otel span as well, use the value in req headers (Note: this will create islands as both + // the app and its children will have the same tracestate + // If homegrown span modification is enabled in config, use it as a last resort. Otherwise, nothing + // will be passed to the child http calls, creating islands + // If any source is found, then it will be propagated to all child http calls + // TODO; also add this to Kafka headers, SNS message attributes + otelTraceparent string + otelTracestate string + ReqTraceparent string + ReqTracestate string + OutTraceparent string + OutTracestate string + + // At the end of API flow, add the status code to OtelSpan; add the Moracide tags to the spans + otelSpan oteltrace.Span + + // These are not useful as of now, just set them for the sake of completion and future + AuditID string + MoneyTrace string + ReqUserAgent string + OutUserAgent string + + TraceID string // use the value in outTraceparent, otherwise MoneyTrace +} + +// NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs +// It can also generate traceparent/state using homegrown algorithms, but avoid this as a rule +func NewXpcTrace(r *http.Request) * XpcTrace { + var xpcTrace XpcTrace + xpcTrace.ReqMoracideTags = make(map[string]string) + + extractParamsFromReq(r, &xpcTrace) + + if xpcTracer.OtelEnabled { + otelExtractParamsFromSpan(r.Context(), &xpcTrace) + } + + if xpcTrace.OutTraceparent == "" { + // No traceparent in otel spans i.e. no otel spans + // No traceparent in incoming req either + // Note: If otel root span, ts can be empty, but tp can be set, so cannot check for ts being empty + tryHomegrownTpTs(r, &xpcTrace) + } + + return &xpcTrace +} + +func SetSpanStatusCode(fields log.Fields) { + var xpcTrace *XpcTrace + if tmp, ok := fields["xpc_trace"]; ok { + xpcTrace = tmp.(*XpcTrace) + } + if xpcTrace == nil { + // Something went wrong, cannot instrument this span + log.Error("instrumentation error, no trace info") + } + if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { + if tmp, ok := fields["status"]; ok { + statusCode := tmp.(int) + otelSetStatusCode(xpcTrace.otelSpan, statusCode) + } + } +} + +func SetSpanMoracideTags(fields log.Fields) { + var xpcTrace *XpcTrace + if tmp, ok := fields["xpc_trace"]; ok { + xpcTrace = tmp.(*XpcTrace) + } + if xpcTrace == nil { + // Something went wrong, cannot instrument this span + log.Error("instrumentation error, cannot set moracide tags, no trace info") + } + + if tmp, ok := fields["xpc_trace"]; ok { + xpcTrace = tmp.(*XpcTrace) + } + moracideTags := make(map[string]string) + reqMoracideTagPrefix := strings.ToLower("req_"+xpcTracer.MoracideTagPrefix) + respMoracideTagPrefix := strings.ToLower("resp_"+xpcTracer.MoracideTagPrefix) + + for key, val := range fields { + if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { + log.Debugf("Adding moracide tag from req %s = %s to response", key, val) + moracideTags[key[4:]] = val.(string) + } + if strings.HasPrefix(strings.ToLower(key), respMoracideTagPrefix) { + log.Debugf("Adding moracide tag from resp %s = %s to response", key, val) + realKey := key[5:] + if existingVal, ok := moracideTags[realKey]; !ok || (ok && existingVal != "true") { + moracideTags[realKey] = val.(string) + } + } + } + if len(moracideTags) == 0 { + // No moracide tags in request or any response + // So set at least one span tag, x-cl-expt: false + moracideTags[xpcTracer.MoracideTagPrefix] = "false" + } + for key, val := range moracideTags { + if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { + xpcTrace.otelSpan.SetAttributes(attribute.String(key, val)) + log.Debugf("added otel span moracide tag key = %s, value = %s", key, val) + } + } +} + +func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace) { + xpcTrace.ReqTraceparent = r.Header.Get(common.HeaderTraceparent) + xpcTrace.ReqTracestate = r.Header.Get(common.HeaderTracestate) + xpcTrace.OutTraceparent = xpcTrace.ReqTraceparent + xpcTrace.OutTracestate = xpcTrace.ReqTracestate + log.Debugf("Tracing: input traceparent : %s, tracestate : %s", xpcTrace.ReqTraceparent, xpcTrace.ReqTracestate) + + // AuditID is used internally within xdp subsystem + // We will not move this to tracing module + // xpcTrace.AuditID = r.Header.Get(AuditIDHeader) + // if xpcTrace.AuditID == "" { + // xpcTrace.AuditID = util.GetAuditId() + // } + + // Moneytrace is Comcast's original solution for tracing + // Unused for now, decide whether we need to merge existing moneytrace code here + // Being replaced by traceparent/tracestate headers anyway + // xpcTrace.MoneyTrace = r.Header.Get(MoneyTraceHeader) + + xpcTrace.ReqUserAgent = r.Header.Get(UserAgentHeader) + + // In future, -H 'X-Cl-Experiment-1', -H 'X-Cl-Experiment-oswebconfig'... OR 'X-Cl-Experiment-xapproxy_25.1.1.1' are all possible + // So walk through all headers and collect any header that starts with this prefix + moracideTagPrefix := strings.ToLower(xpcTracer.MoracideTagPrefix) + for headerKey, headerVals := range r.Header { + if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { + if len(headerVals) > 1 { + log.Debugf("Tracing: moracide tag key = %s, has multiple values = %+v", headerKey, headerVals) + } + val := "false" + for _, v := range headerVals { + if v == "true" { + val = v + break + } + } + xpcTrace.ReqMoracideTags[headerKey] = val + log.Debugf("Tracing: found moracide tag key = %s, val = %s, all vals = %+v", headerKey, val, headerVals) + } + } +} diff --git a/tracing/tracer.go b/tracing/tracer.go new file mode 100644 index 0000000..80fdcc1 --- /dev/null +++ b/tracing/tracer.go @@ -0,0 +1,99 @@ +package tracing + +import ( + "strings" + "os" + + "github.com/go-akka/configuration" + + oteltrace "go.opentelemetry.io/otel/trace" + otelpropagation "go.opentelemetry.io/otel/propagation" +) + +const ( + AuditIDHeader = "X-Auditid" + UserAgentHeader = "User-Agent" + + defaultMoracideTagPrefix = "X-Cl-Experiment" +) + +// XpcTracer is a wrapper around tracer setup +type XpcTracer struct { + OtelEnabled bool + MoracideTagPrefix string // Special request header for moracide expts e.g. canary deployments + + // internal vars used by Otel + appEnv string // set this to dev for red, staging for yellow and prod for green + appName string + appVersion string + appSHA string // unused + rgn string // AWS Region e.g. us-west-2, unused, use it as a otel span attribute + siteColor string // red/yellow/green, unused, use it as a otel span attribute + + // internal otel vars + otelEndpoint string + otelOpName string + otelProvider string + otelTracerProvider oteltrace.TracerProvider + otelPropagator otelpropagation.TextMapPropagator + otelTracer oteltrace.Tracer + + // internal vars for homegrown trace generation/modification + appID string // used in the homegrown traceparent,tracestate header generation + homegrownTracePropagation bool // Use homegrown algorithm for traceparent, tracestate modification + homegrownTraceGeneration bool // Generate traceparent etc. using homegrown algorithm if not present in incoming req +} + +var xpcTracer XpcTracer // global tracer + +func NewXpcTracer(conf *configuration.Config) *XpcTracer { + initAppData(conf) + initHomegrownTracing(conf) + + otelInit(conf) + + xpcTracer.MoracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", defaultMoracideTagPrefix) + return &xpcTracer +} + +// defer this func in the main of the app +func StopXpcTracer() { + otelShutdown() +} + +// Global func to access the moracide tag +func GetMoracideTagPrefix() string { + return xpcTracer.MoracideTagPrefix +} + +func initAppData(conf *configuration.Config) { + codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") + xpcTracer.appName = codeGitCommit[0] + if len(codeGitCommit) > 1 { + xpcTracer.appVersion = codeGitCommit[1] + } + if len(codeGitCommit) > 2 { + xpcTracer.appSHA = codeGitCommit[2] + } + + // Env vars + xpcTracer.appEnv = "dev" + siteColor := os.Getenv("SITE_COLOR") + if strings.EqualFold(siteColor, "yellow") { + xpcTracer.appEnv = "staging" + } else if strings.EqualFold(siteColor, "green") { + xpcTracer.appEnv = "prod" + } + xpcTracer.rgn = os.Getenv("SITE_REGION") +} + +func initHomegrownTracing(conf *configuration.Config) { + // TODO: Marshal these tracing params from config, ok to temporarily read each separately + xpcTracer.appID = conf.GetString("webconfig.tracing.homegrown_algorithm.app_id", defaultAppID) + xpcTracer.homegrownTracePropagation = conf.GetBoolean("webconfig.tracing.homegrown_algorithm.xpc_trace_propagation") + xpcTracer.homegrownTraceGeneration = conf.GetBoolean("webconfig.tracing.homegrown_algorithm.xpc_trace_generation") +} + +func GetServiceName() string { + return xpcTracer.appName +} From 0e5f43af8d65f463ffad6209d8b4b9f9e98b7121 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Mon, 30 Dec 2024 12:23:46 -0800 Subject: [PATCH 141/155] Remove all homegrown algorithm code as it is never going to be used --- tracing/homegrown.go | 230 ------------------------------------- tracing/homegrown_test.go | 231 -------------------------------------- tracing/span.go | 11 +- tracing/tracer.go | 15 --- 4 files changed, 1 insertion(+), 486 deletions(-) delete mode 100644 tracing/homegrown.go delete mode 100644 tracing/homegrown_test.go diff --git a/tracing/homegrown.go b/tracing/homegrown.go deleted file mode 100644 index 40f4a86..0000000 --- a/tracing/homegrown.go +++ /dev/null @@ -1,230 +0,0 @@ -/** - * @license - * Copyright Comcast. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @author RV - -/* - W3C Doc: https://www.w3.org/TR/trace-context-1 - - Usage: - 1. AppID should be set in the config - Config Template (from HC): - ------------------------------------------- - "Tracing": { - // This identifies the microservice - "AppID": "0000000000000001" - }, - ------------------------------------------- - 2. Call GetTracingHeader with a http Request, and it will return a modified - traceparent/tracestate header - 3. Set it in req.Header - 4. If the generate flag is set in GetTracingHeader and if the input req doesn't - have a traceparent header, these two headers will be created - TODO: - Also pass a Kafka message as an input to do the above - - Spec: - traceparent header has 4 sections - "---" - version is 2 digits, traceID is 32 digits, callerIF is 16, and flags is 2 - if traceparent is present in headers, replace the callerID with app's ID - - Template - hc's ID defaults to "0000000000000001" and can be changed in config - - If tracestate header exists, add "=," to the beginning of the header - else set it to "=" - - As of now, we don't generate traceparent if the header is not passed in - Any trace flag setting that tells us to regenerate the traceID is ignored. - Version, TraceID, and trace flags are passed as-is. Only CallerID is modified - TraceState will be generated if it doesn't exist - - Epic: XPC-16688 - HC Ticket: ODP-25026 - Xapproxy Ticket: XPC-17946 - - Changes: - otel implements their own traceparent/tracestate propagation and this clashes with - our home-grown propagation. If otel is enabled, suppress the home-grown propagation. - TODO: Change homegrown propagation too to a per API basis instead of per app basis -*/ - -package tracing - -import ( - "fmt" - "math/rand" - "net/http" - "strings" - // "sync" - "time" - log "github.com/sirupsen/logrus" - "github.com/rdkcentral/webconfig/common" -) - -const ( - // These are two completely internal - traceVersion = "01" - traceFlags = "00" - - defaultAppID = "0000000000000001" - - traceAppIDNotSetErr = "Tracing: AppID not set" -) - -type TraceHeader struct { - Key, Value string -} - -func GetTraceHeaders(r *http.Request, generateTrace bool) ([]TraceHeader, error) { - // This is not concurrency safe (hat tip to Jay) - // Hence it is instantiated in the goroutine that handles the API req - // Google search says this is inexpensive and acceptable - // Prefer not to use a mutex as multiple Kafka reqs can wait on this lock - - randGen := rand.NewSource(time.Now().UnixNano()) - - headers := make([]TraceHeader, 0) - traceParent := r.Header.Get("traceparent") - traceComponents := strings.Split(traceParent, "-") - if len(traceComponents) != 4 { - // traceparent header is either incorrectly formatted or doesn't exist - if !generateTrace { - // Don't create traceparent header if flag is not set - // Don't bother about tracestate header - return headers, nil - } - - if xpcTracer.appID == "" { - return headers, fmt.Errorf(traceAppIDNotSetErr) - } - // Create a new traceparent header - traceParent = fmt.Sprintf("%s-%016x%016x-%s-%s", - traceVersion, - // genInt63(randGen), genInt63(randGen), - randGen.Int63(), randGen.Int63(), - xpcTracer.appID, - traceFlags) - } else { - // Replace the appID part - if xpcTracer.appID == "" { - return headers, fmt.Errorf(traceAppIDNotSetErr) - } - // Create a new traceparent header - traceParent = fmt.Sprintf("%s-%s-%s-%s", - traceComponents[0], - traceComponents[1], - xpcTracer.appID, - traceComponents[3]) - } - headers = append(headers, TraceHeader{ - Key: "traceparent", - Value: traceParent, - }) - - traceState := r.Header.Get("tracestate") - if traceState == "" { - // Create tracestate header if it doesn't exist - traceState = fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID) - } else { - // Add current app to traceState - traceState = fmt.Sprintf("%s=%s,%s", xpcTracer.appName, xpcTracer.appID, traceState) - } - headers = append(headers, TraceHeader{ - Key: "tracestate", - Value: traceState, - }) - return headers, nil -} - -func tryHomegrownTpTs(r *http.Request, xpcTrace *XpcTrace) { - // if homegrownTracePropagation is true, we modify existing traceparent/tracestate and forward them - // but we will not generate them if they don't exist - // if homegrownTraceGeneration is true, and there is no tp/ts, we generate it - - if !xpcTracer.homegrownTracePropagation { - return - } - // ctx := r.Context() - log.Debug("Tracing: using homegrown traceparent propagation/generation") - if traceHeaders, err := GetTraceHeaders(r, xpcTracer.homegrownTraceGeneration); err == nil { - // TODO: Change homegrown propagation too to a per API basis instead of per app basis - // Reverse engineer otel basically - for _, h := range traceHeaders { - log.Debugf("Tracing: %s set to %s", h.Key, h.Value) - if h.Key == common.HeaderTraceparent { - xpcTrace.OutTraceparent = h.Value - } - if h.Key == common.HeaderTracestate { - xpcTrace.OutTracestate = h.Value - } - } - } else { - log.Errorf("Error in homegrown traceparent handling %+v", err) - } -} - -/* - // Benchmarking expt to compare mutex based random vs one source per goroutine - var ( - // This is for a benchmarking expt - global bool - globalLock sync.Mutex - globalRandGen = rand.NewSource(time.Now().UnixNano()) - - // Sample Benchmarking Results - // BenchmarkSharedRandGen-16 352419 3052 ns/op - // BenchmarkIndependentRandGen-16 670383 2755 ns/op - ) - - func setGlobal(g bool) { - global = g - } - - func genInt63(randGen rand.Source) int64 { - if global { - globalLock.Lock() - i := globalRandGen.Int63() - globalLock.Unlock() - return i - } - return randGen.Int63() - } -*/ - -/* - traceparentParentID string - tracestateVendorID string - otelEnabled bool - traceparentParentID: traceparentParentID, - tracestateVendorID: tracestateVendorID, -func (s *WebconfigServer) TraceparentParentID() string { - return s.traceparentParentID -} - -func (s *WebconfigServer) SetTraceparentParentID(x string) { - s.traceparentParentID = x -} - -func (s *WebconfigServer) TracestateVendorID() string { - return s.tracestateVendorID -} - -func (s *WebconfigServer) SetTracestateVendorID(x string) { - s.tracestateVendorID = x -} -*/ diff --git a/tracing/homegrown_test.go b/tracing/homegrown_test.go deleted file mode 100644 index 6e36c36..0000000 --- a/tracing/homegrown_test.go +++ /dev/null @@ -1,231 +0,0 @@ -/** - * @license - * Copyright Comcast. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// @author RV - -package tracing - -import ( - "bytes" - "fmt" - "net/http" - "strings" - // "sync" - "testing" - - "github.com/stretchr/testify/require" -) - -const ( - testTraceID = "0123456789abcdef0123456789abcdef" - testCallerID = "fedcba9876543210" - testTraceParent = traceVersion + "-" + testTraceID + "-" + testCallerID + "-" + traceFlags - - testCallerName = "test" - testTraceState = testCallerName + "=" + testCallerID - - testMyName = "whatever" - testMyID = "0123456789abcdef" -) - -func TestAppIDNotSet(t *testing.T) { - // Mimic appName/appID not set by explicitly setting them to empty strs - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = "" - xpcTracer.appName = "" - - req := makeReq(t) - req.Header.Add("traceparent", testTraceParent) - _, err := GetTraceHeaders(req, false) - require.Equal(t, traceAppIDNotSetErr, err.Error()) - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func TestNoGenerate(t *testing.T) { - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = testMyID - xpcTracer.appName = testMyName - - req := makeReq(t) - headers, _ := GetTraceHeaders(req, false) - require.Equal(t, 0, len(headers)) - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func TestOnlyParentHeader(t *testing.T) { - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = testMyID - xpcTracer.appName = testMyName - - req := makeReq(t) - req.Header.Add("traceparent", testTraceParent) - - headers, _ := GetTraceHeaders(req, false) - require.Equal(t, 2, len(headers)) - validateTraceParentHeader(t, headers) - for _, h := range headers { - if h.Key == "tracestate" { - require.Equal(t, fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID), h.Value) - } - } - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func TestOnlyStateHeader(t *testing.T) { - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = testMyID - xpcTracer.appName = testMyName - - req := makeReq(t) - req.Header.Add("tracestate", testTraceState) - headers, _ := GetTraceHeaders(req, false) - require.Equal(t, 0, len(headers)) - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func TestParentPlusStateHeaders(t *testing.T) { - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = testMyID - xpcTracer.appName = testMyName - - req := makeReq(t) - req.Header.Add("traceparent", testTraceParent) - req.Header.Add("tracestate", testTraceState) - headers, _ := GetTraceHeaders(req, false) - require.Equal(t, 2, len(headers)) - validateTraceParentHeader(t, headers) - for _, h := range headers { - if h.Key == "tracestate" { - require.Equal(t, fmt.Sprintf("%s=%s,%s", xpcTracer.appName, xpcTracer.appID,testTraceState), h.Value) - } - } - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func TestTraceHeaderGeneration(t *testing.T) { - appID := xpcTracer.appID - appName := xpcTracer.appName - xpcTracer.appID = testMyID - xpcTracer.appName = testMyName - - testTraceHeaderGeneration(t) - - xpcTracer.appID = appID - xpcTracer.appName = appName -} - -func validateTraceParentHeader(t *testing.T, headers []TraceHeader) { - for _, h := range headers { - if h.Key == "traceparent" { - traceComponents := strings.Split(h.Value, "-") - require.Equal(t, 4, len(traceComponents), 4) - require.Equal(t, traceVersion, traceComponents[0]) - require.Equal(t, testTraceID, traceComponents[1]) - require.Equal(t, xpcTracer.appID, traceComponents[2]) - require.Equal(t, traceFlags, traceComponents[3]) - } - } -} - -func makeReq(t *testing.T) *http.Request { - url := "/testurl" - var body []byte - req, err := http.NewRequest("GET", url, bytes.NewReader(body)) - if err != nil { - t.Fatal(err) - } - return req -} - -func testTraceHeaderGeneration(t *testing.T) { - req := makeReq(t) - headers, _ := GetTraceHeaders(req, true) - require.Equal(t, 2, len(headers)) - for _, h := range headers { - if h.Key == "traceparent" { - traceComponents := strings.Split(h.Value, "-") - require.Equal(t, 4, len(traceComponents)) - require.Equal(t, traceVersion, traceComponents[0]) - // We don't know the ID here, we can only check it is 32 digits long - require.Equal(t, 32, len(traceComponents[1])) - require.Equal(t, xpcTracer.appID, traceComponents[2]) - require.Equal(t, traceFlags, traceComponents[3]) - } - if h.Key == "tracestate" { - require.Equal(t, fmt.Sprintf("%s=%s", xpcTracer.appName, xpcTracer.appID), h.Value) - } - } -} - -/* - // These tests/benchmarks are to test mutex based random generation - // We are not going to use mutex as of now, as using one source - // per goroutine approach is faster - func TestTraceHeaderGenerationGlobal(t *testing.T) { - setGlobal(true) - testTraceHeaderGeneration(t) - setGlobal(false) - } - - func BenchmarkSharedRandGen(b *testing.B) { - setGlobal(true) - url := "/testurl" - var body []byte - req, _ := http.NewRequest("GET", url, bytes.NewReader(body)) - var wg sync.WaitGroup - for n := 0; n < b.N; n++ { - wg.Add(1) - go func() { - defer wg.Done() - GetTraceHeaders(req, true) - }() - } - wg.Wait() - setGlobal(false) - } - - func BenchmarkIndependentRandGen(b *testing.B) { - url := "/testurl" - var body []byte - req, _ := http.NewRequest("GET", url, bytes.NewReader(body)) - var wg sync.WaitGroup - - for n := 0; n < b.N; n++ { - wg.Add(1) - go func() { - defer wg.Done() - GetTraceHeaders(req, true) - }() - } - wg.Wait() - } -*/ diff --git a/tracing/span.go b/tracing/span.go index 137ac8b..15002b6 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -42,8 +42,7 @@ type XpcTrace struct { // Order of priority; use the value extracted from otel span; // if no otel span as well, use the value in req headers (Note: this will create islands as both // the app and its children will have the same tracestate - // If homegrown span modification is enabled in config, use it as a last resort. Otherwise, nothing - // will be passed to the child http calls, creating islands + // Otherwise, nothing will be passed to the child http calls, creating islands // If any source is found, then it will be propagated to all child http calls // TODO; also add this to Kafka headers, SNS message attributes otelTraceparent string @@ -66,7 +65,6 @@ type XpcTrace struct { } // NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs -// It can also generate traceparent/state using homegrown algorithms, but avoid this as a rule func NewXpcTrace(r *http.Request) * XpcTrace { var xpcTrace XpcTrace xpcTrace.ReqMoracideTags = make(map[string]string) @@ -77,13 +75,6 @@ func NewXpcTrace(r *http.Request) * XpcTrace { otelExtractParamsFromSpan(r.Context(), &xpcTrace) } - if xpcTrace.OutTraceparent == "" { - // No traceparent in otel spans i.e. no otel spans - // No traceparent in incoming req either - // Note: If otel root span, ts can be empty, but tp can be set, so cannot check for ts being empty - tryHomegrownTpTs(r, &xpcTrace) - } - return &xpcTrace } diff --git a/tracing/tracer.go b/tracing/tracer.go index 80fdcc1..d447ee2 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -37,21 +37,13 @@ type XpcTracer struct { otelTracerProvider oteltrace.TracerProvider otelPropagator otelpropagation.TextMapPropagator otelTracer oteltrace.Tracer - - // internal vars for homegrown trace generation/modification - appID string // used in the homegrown traceparent,tracestate header generation - homegrownTracePropagation bool // Use homegrown algorithm for traceparent, tracestate modification - homegrownTraceGeneration bool // Generate traceparent etc. using homegrown algorithm if not present in incoming req } var xpcTracer XpcTracer // global tracer func NewXpcTracer(conf *configuration.Config) *XpcTracer { initAppData(conf) - initHomegrownTracing(conf) - otelInit(conf) - xpcTracer.MoracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", defaultMoracideTagPrefix) return &xpcTracer } @@ -87,13 +79,6 @@ func initAppData(conf *configuration.Config) { xpcTracer.rgn = os.Getenv("SITE_REGION") } -func initHomegrownTracing(conf *configuration.Config) { - // TODO: Marshal these tracing params from config, ok to temporarily read each separately - xpcTracer.appID = conf.GetString("webconfig.tracing.homegrown_algorithm.app_id", defaultAppID) - xpcTracer.homegrownTracePropagation = conf.GetBoolean("webconfig.tracing.homegrown_algorithm.xpc_trace_propagation") - xpcTracer.homegrownTraceGeneration = conf.GetBoolean("webconfig.tracing.homegrown_algorithm.xpc_trace_generation") -} - func GetServiceName() string { return xpcTracer.appName } From 4905893a4b82022c6b8dc21837e57683225d9a44 Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Thu, 2 Jan 2025 14:44:31 -0800 Subject: [PATCH 142/155] case diffs in env var e.g. use site_color instead of SITE_COLOR --- tracing/tracer.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tracing/tracer.go b/tracing/tracer.go index d447ee2..ed48374 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -5,6 +5,7 @@ import ( "os" "github.com/go-akka/configuration" + log "github.com/sirupsen/logrus" oteltrace "go.opentelemetry.io/otel/trace" otelpropagation "go.opentelemetry.io/otel/propagation" @@ -70,13 +71,17 @@ func initAppData(conf *configuration.Config) { // Env vars xpcTracer.appEnv = "dev" - siteColor := os.Getenv("SITE_COLOR") + siteColor := os.Getenv("site_color") if strings.EqualFold(siteColor, "yellow") { xpcTracer.appEnv = "staging" } else if strings.EqualFold(siteColor, "green") { xpcTracer.appEnv = "prod" } - xpcTracer.rgn = os.Getenv("SITE_REGION") + xpcTracer.rgn = os.Getenv("site_region") + if xpcTracer.rgn == "" { + xpcTracer.rgn = os.Getenv("site_region_name") + } + log.Debugf("site_color = %f, env = %s, rgn = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) } func GetServiceName() string { From a50cf7f1337f87bcc514a14dab851aa1e141881d Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Fri, 3 Jan 2025 12:44:51 -0800 Subject: [PATCH 143/155] Typo --- tracing/tracer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracing/tracer.go b/tracing/tracer.go index ed48374..8e5888a 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -81,7 +81,7 @@ func initAppData(conf *configuration.Config) { if xpcTracer.rgn == "" { xpcTracer.rgn = os.Getenv("site_region_name") } - log.Debugf("site_color = %f, env = %s, rgn = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) + log.Debugf("site_color = %s, env = %s, rgn = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) } func GetServiceName() string { From dbdee274264775ad8756dae2bf207a891717843a Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Mon, 6 Jan 2025 18:14:08 -0800 Subject: [PATCH 144/155] Standardize logs - use "message" instead of "msg" --- config/sample_webconfig.conf | 5 ----- main.go | 1 + tracing/tracer.go | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index e5e93f8..02def24 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -6,11 +6,6 @@ webconfig { panic_exit_enabled = false tracing { moracide_tag_prefix = "X-Cl-Experiment" - homegrown_algorithm { - app_id = "0000000000000001" - xpc_trace_propagation = false - xpc_trace_generation = false - } opentelemetry { enabled = false endpoint = "127.0.0.1:4318" diff --git a/main.go b/main.go index 284012c..a6e5f80 100644 --- a/main.go +++ b/main.go @@ -81,6 +81,7 @@ func main() { TimestampFormat: common.LoggingTimeFormat, FieldMap: log.FieldMap{ log.FieldKeyTime: "timestamp", + log.FieldKeyMsg: "message", }, }) diff --git a/tracing/tracer.go b/tracing/tracer.go index 8e5888a..2f78c3f 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -81,7 +81,7 @@ func initAppData(conf *configuration.Config) { if xpcTracer.rgn == "" { xpcTracer.rgn = os.Getenv("site_region_name") } - log.Debugf("site_color = %s, env = %s, rgn = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) + log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) } func GetServiceName() string { From b6bdcf2387e873d1ef26bacbc86fa4005c9e4de8 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 8 Jan 2025 14:20:01 -0800 Subject: [PATCH 145/155] fix a bug that new spanid are not included in the traceparent of outgoing headers --- go.mod | 3 --- http/http_client.go | 4 ++-- http/response.go | 6 ++--- tracing/ctx.go | 17 ++++++++++++++ tracing/otel.go | 25 ++++++++++++++++---- tracing/span.go | 56 +++++++++++++++++++++++++++++---------------- tracing/tracer.go | 23 ++++++++++++++++--- 7 files changed, 99 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index e085f6d..95108f9 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.2.0 github.com/sirupsen/logrus v1.9.0 - github.com/stretchr/testify v1.9.0 github.com/twmb/murmur3 v1.1.6 github.com/vmihailenco/msgpack v4.0.4+incompatible github.com/vmihailenco/msgpack/v4 v4.3.12 @@ -57,7 +56,6 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -75,5 +73,4 @@ require ( google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.34.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/http/http_client.go b/http/http_client.go index 5ec0e72..e9397b3 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -154,12 +154,12 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h header.Set("X-Webpa-Transaction-Id", transactionId) } + c.addMoracideTags(header, auditFields) req.Header = header.Clone() if len(c.userAgent) > 0 { req.Header.Set(common.HeaderUserAgent, c.userAgent) } - c.addMoracideTags(header, auditFields) logHeader := header.Clone() auth := logHeader.Get("Authorization") if len(auth) > 0 { @@ -428,7 +428,7 @@ func (c *HttpClient) StatusHandler(status int) StatusHandlerFunc { // addMoracideTags - if ctx has a moracide tag as a header, add it to the headers // Also add traceparent, tracestate headers func (c *HttpClient) addMoracideTags(header http.Header, fields log.Fields) { - moracideTagPrefix := strings.ToLower("req_"+tracing.GetMoracideTagPrefix()) + moracideTagPrefix := strings.ToLower("req_" + tracing.GetMoracideTagPrefix()) for key, val := range fields { if key == "out_traceparent" { header.Set("traceparent", val.(string)) diff --git a/http/response.go b/http/response.go index 5fbd1bc..6c229be 100644 --- a/http/response.go +++ b/http/response.go @@ -22,10 +22,10 @@ import ( "fmt" "net/http" "strings" - log "github.com/sirupsen/logrus" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/tracing" + log "github.com/sirupsen/logrus" ) const ( @@ -176,8 +176,8 @@ func addMoracideTagsAsResponseHeaders(w http.ResponseWriter) { return } - reqMoracideTagPrefix := strings.ToLower("req_"+tracing.GetMoracideTagPrefix()) - respMoracideTagPrefix := strings.ToLower("resp_"+tracing.GetMoracideTagPrefix()) + reqMoracideTagPrefix := strings.ToLower("req_" + tracing.GetMoracideTagPrefix()) + respMoracideTagPrefix := strings.ToLower("resp_" + tracing.GetMoracideTagPrefix()) fields := xw.Audit() moracideTags := make(map[string]string) for key, val := range fields { diff --git a/tracing/ctx.go b/tracing/ctx.go index 204e084..4a490fb 100644 --- a/tracing/ctx.go +++ b/tracing/ctx.go @@ -1,3 +1,20 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ package tracing import "context" diff --git a/tracing/otel.go b/tracing/otel.go index 22e4728..66dfb69 100644 --- a/tracing/otel.go +++ b/tracing/otel.go @@ -1,3 +1,20 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ package tracing import ( @@ -7,18 +24,18 @@ import ( "strings" "time" - "github.com/gorilla/mux" "github.com/go-akka/configuration" + "github.com/gorilla/mux" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/exporters/stdout/stdouttrace" + "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" - "go.opentelemetry.io/otel/propagation" oteltrace "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/trace/noop" @@ -29,9 +46,9 @@ type providerConstructor func() (oteltrace.TracerProvider, error) var ( providerBuilders = map[string]providerConstructor{ - "http": otelHttpTraceProvider, + "http": otelHttpTraceProvider, "stdout": otelStdoutTraceProvider, - "noop": otelNoopTraceProvider, + "noop": otelNoopTraceProvider, } ) diff --git a/tracing/span.go b/tracing/span.go index 15002b6..c79c0e3 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -1,15 +1,31 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ package tracing import ( "net/http" "strings" + "github.com/rdkcentral/webconfig/common" log "github.com/sirupsen/logrus" - oteltrace "go.opentelemetry.io/otel/trace" "go.opentelemetry.io/otel/attribute" - - "github.com/rdkcentral/webconfig/common" + oteltrace "go.opentelemetry.io/otel/trace" ) // SpanMiddleware is a middleware that creates a new span for each incoming request. @@ -34,7 +50,7 @@ type XpcTrace struct { // e.g. X-Cl-Experiment-1, X-Cl-Experiment-xapproxy, X-Cl-Experiement-webconfig-25.1.1.1... // For every key found in either req or resp, an explicit value of true/false will be set as an otel attribute // or an otel span attribute - ReqMoracideTags map[string]string // These are request headers prefixed with MoracideTagPrefix + ReqMoracideTags map[string]string // These are request headers prefixed with MoracideTagPrefix // The response moracide tags are stored in xw.audit // traceparent, tracestate can be set as req headers, may be extracted from otel spans @@ -45,27 +61,27 @@ type XpcTrace struct { // Otherwise, nothing will be passed to the child http calls, creating islands // If any source is found, then it will be propagated to all child http calls // TODO; also add this to Kafka headers, SNS message attributes - otelTraceparent string - otelTracestate string - ReqTraceparent string - ReqTracestate string - OutTraceparent string - OutTracestate string + otelTraceparent string + otelTracestate string + ReqTraceparent string + ReqTracestate string + OutTraceparent string + OutTracestate string // At the end of API flow, add the status code to OtelSpan; add the Moracide tags to the spans - otelSpan oteltrace.Span + otelSpan oteltrace.Span // These are not useful as of now, just set them for the sake of completion and future - AuditID string - MoneyTrace string - ReqUserAgent string - OutUserAgent string + AuditID string + MoneyTrace string + ReqUserAgent string + OutUserAgent string - TraceID string // use the value in outTraceparent, otherwise MoneyTrace + TraceID string // use the value in outTraceparent, otherwise MoneyTrace } // NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs -func NewXpcTrace(r *http.Request) * XpcTrace { +func NewXpcTrace(r *http.Request) *XpcTrace { var xpcTrace XpcTrace xpcTrace.ReqMoracideTags = make(map[string]string) @@ -109,8 +125,8 @@ func SetSpanMoracideTags(fields log.Fields) { xpcTrace = tmp.(*XpcTrace) } moracideTags := make(map[string]string) - reqMoracideTagPrefix := strings.ToLower("req_"+xpcTracer.MoracideTagPrefix) - respMoracideTagPrefix := strings.ToLower("resp_"+xpcTracer.MoracideTagPrefix) + reqMoracideTagPrefix := strings.ToLower("req_" + xpcTracer.MoracideTagPrefix) + respMoracideTagPrefix := strings.ToLower("resp_" + xpcTracer.MoracideTagPrefix) for key, val := range fields { if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { @@ -149,7 +165,7 @@ func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace) { // We will not move this to tracing module // xpcTrace.AuditID = r.Header.Get(AuditIDHeader) // if xpcTrace.AuditID == "" { - // xpcTrace.AuditID = util.GetAuditId() + // xpcTrace.AuditID = util.GetAuditId() // } // Moneytrace is Comcast's original solution for tracing diff --git a/tracing/tracer.go b/tracing/tracer.go index 2f78c3f..8f3be02 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -1,18 +1,35 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ package tracing import ( - "strings" "os" + "strings" "github.com/go-akka/configuration" log "github.com/sirupsen/logrus" - oteltrace "go.opentelemetry.io/otel/trace" otelpropagation "go.opentelemetry.io/otel/propagation" + oteltrace "go.opentelemetry.io/otel/trace" ) const ( - AuditIDHeader = "X-Auditid" + AuditIDHeader = "X-Auditid" UserAgentHeader = "User-Agent" defaultMoracideTagPrefix = "X-Cl-Experiment" From e9e969a40a76c03e5267b27fd8921693edb10798 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 13 Jan 2025 11:40:33 -0800 Subject: [PATCH 146/155] Make external url templates configurable --- config/sample_webconfig.conf | 36 ++++++++++++++++------------- http/http_client.go | 2 +- http/mqtt_connector.go | 20 ++++++++++++---- http/upstream_connector.go | 19 +++++++-------- http/webpa_connector.go | 45 ++++++++++++++++++------------------ http/xconf_connector.go | 20 ++++++++++++---- 6 files changed, 82 insertions(+), 60 deletions(-) diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 02def24..36c8843 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -5,17 +5,17 @@ webconfig { panic_exit_enabled = false tracing { - moracide_tag_prefix = "X-Cl-Experiment" - opentelemetry { - enabled = false - endpoint = "127.0.0.1:4318" - operation_name = "http.request" - // Allowed values: "noop", "stdout", "http" - // "noop" will generate no trace - // "stdout" will use stdoutTracer and output spans to stdout - // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector - provider = "noop" - } + moracide_tag_prefix = "X-Cl-Experiment" + otel { + enabled = false + endpoint = "127.0.0.1:4318" + operation_name = "http.request" + // Allowed values: "noop", "stdout", "http" + // "noop" will generate no trace + // "stdout" will use stdoutTracer and output spans to stdout + // "http" will use otlphttpTracer and output spans that need to be collected by otelcollector + provider = "noop" + } } // build info @@ -49,10 +49,11 @@ webconfig { read_timeout_in_secs = 142 max_idle_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "https://api.webpa.comcast.net" + host = "http://localhost:12345" async_poke_enabled = false async_poke_concurrent_calls = 100 api_version = "v2" + url_template = "%s/%s/%s" } xconf { @@ -63,7 +64,8 @@ webconfig { max_idle_conns_per_host = 100 max_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "http://qa2.xconfds.coast.xcal.tv:8080" + host = "http://localhost:12346" + url_template = "%s/%s" } mqtt { @@ -74,7 +76,8 @@ webconfig { max_idle_conns_per_host = 100 max_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "https://hcbroker.staging.us-west-2.plume.comcast.net" + host = "http://localhost:12347" + url_template = "%s/%s" } upstream { @@ -86,8 +89,9 @@ webconfig { max_idle_conns_per_host = 100 max_conns_per_host = 100 keepalive_timeout_in_secs = 30 - host = "http://localhost:9009" - url_template = "/api/v1/device/%v/upstream" + host = "http://localhost:12348" + url_template = "%s/%s" + profile_url_template = "%s/%s/%s" } http_client { diff --git a/http/http_client.go b/http/http_client.go index e9397b3..db8ca45 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -110,7 +110,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl } func (c *HttpClient) Do(ctx context.Context, method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { - fields := common.FilterLogFields(auditFields) + fields := common.FilterLogFields(auditFields, "status") var respMoracideTagsFound bool defer func(found *bool) { diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 064f2e5..034c3bb 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -31,25 +31,27 @@ import ( ) const ( - mqttHostDefault = "https://hcbroker.staging.us-west-2.plume.comcast.net" - mqttUrlTemplate = "%s/v2/mqtt/pub/x/to/%s/webconfig" + defaultMqttHost = "http://localhost:12347" + defaultMqttUrlTemplate = "%s/%s" ) type MqttConnector struct { *HttpClient host string serviceName string + urlTemplate string } func NewMqttConnector(conf *configuration.Config, tlsConfig *tls.Config) *MqttConnector { serviceName := "mqtt" - confKey := fmt.Sprintf("webconfig.%v.host", serviceName) - host := conf.GetString(confKey, mqttHostDefault) + host := conf.GetString("webconfig.mqtt.host", defaultMqttHost) + urlTemplate := conf.GetString("webconfig.mqtt.url_template", defaultMqttUrlTemplate) return &MqttConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, + urlTemplate: urlTemplate, } } @@ -61,12 +63,20 @@ func (c *MqttConnector) SetMqttHost(host string) { c.host = host } +func (c *MqttConnector) MqttUrlTemplate() string { + return c.urlTemplate +} + +func (c *MqttConnector) SetMqttUrlTemplate(x string) { + c.urlTemplate = x +} + func (c *MqttConnector) ServiceName() string { return c.serviceName } func (c *MqttConnector) PostMqtt(ctx context.Context, cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { - url := fmt.Sprintf(mqttUrlTemplate, c.MqttHost(), cpeMac) + url := fmt.Sprintf(c.MqttUrlTemplate(), c.MqttHost(), cpeMac) var traceId, xmTraceId, outTraceparent, outTracestate string if itf, ok := fields["xmoney_trace_id"]; ok { diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 303f2b6..2a3520a 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -30,9 +30,9 @@ import ( ) const ( - upstreamHostDefault = "http://localhost:1234" - defaultUpstreamUrlTemplate = "/api/v1/device/%v/upstream" - defaultProfileUrlTemplate = "/api/v1/device/%v/profile?%v" + defaultUpstreamHost = "http://localhost:12348" + defaultUpstreamUrlTemplate = "%s/%s" + defaultProfileUrlTemplate = "%s/%s/%s" ) type UpstreamConnector struct { @@ -45,12 +45,9 @@ type UpstreamConnector struct { func NewUpstreamConnector(conf *configuration.Config, tlsConfig *tls.Config) *UpstreamConnector { serviceName := "upstream" - confKey := fmt.Sprintf("webconfig.%v.host", serviceName) - host := conf.GetString(confKey, upstreamHostDefault) - confKey = fmt.Sprintf("webconfig.%v.url_template", serviceName) - upstreamUrlTemplate := conf.GetString(confKey, defaultUpstreamUrlTemplate) - confKey = fmt.Sprintf("webconfig.%v.profile_url_template", serviceName) - profileUrlTemplate := conf.GetString(confKey, defaultProfileUrlTemplate) + host := conf.GetString("webconfig.upstream.host", defaultUpstreamHost) + upstreamUrlTemplate := conf.GetString("webconfig.upstream.url_template", defaultUpstreamUrlTemplate) + profileUrlTemplate := conf.GetString("webconfig.upstream.profile_url_template", defaultProfileUrlTemplate) return &UpstreamConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), @@ -74,7 +71,7 @@ func (c *UpstreamConnector) ServiceName() string { } func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { - url := c.UpstreamHost() + fmt.Sprintf(c.upstreamUrlTemplate, mac) + url := fmt.Sprintf(c.upstreamUrlTemplate, c.UpstreamHost(), mac) if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) @@ -98,7 +95,7 @@ func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header } func (c *UpstreamConnector) GetUpstreamProfiles(ctx context.Context, mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { - url := c.UpstreamHost() + fmt.Sprintf(c.profileUrlTemplate, mac, queryParams) + url := fmt.Sprintf(c.profileUrlTemplate, c.UpstreamHost(), mac, queryParams) if itf, ok := fields["audit_id"]; ok { auditId := itf.(string) diff --git a/http/webpa_connector.go b/http/webpa_connector.go index f184e1f..014f291 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -33,14 +33,14 @@ import ( ) const ( - defaultWebpaHost = "https://api.webpa.comcast.net:8090" + defaultWebpaHost = "http://localhost:12345" defaultApiVersion = "v2" webpaServiceName = "webpa" asyncWebpaServiceName = "asyncwebpa" - webpaUrlTemplate = "%s/api/%s/device/mac:%s/config" - webpaError404 = `{"code": 521, "message": "Device not found in webpa"}` - webpaError520 = `{"code": 520, "message": "Error unsupported namespace"}` + defaultWebpaUrlTemplate = "%s/%s/%s" + webpaError404 = `{"code": 521, "message": "Device not found in webpa"}` + webpaError520 = `{"code": 520, "message": "Error unsupported namespace"}` // a new error code to indicate it is webpa 520 // but it is caused by some temporary conditions, @@ -66,6 +66,7 @@ type WebpaConnector struct { syncClient *HttpClient asyncClient *HttpClient host string + urlTemplate string queue chan struct{} retries int retryInMsecs int @@ -112,33 +113,24 @@ func asyncHandle520(rbytes []byte) ([]byte, http.Header, bool, error) { } func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *WebpaConnector { - confKey := fmt.Sprintf("webconfig.%v.host", webpaServiceName) - host := conf.GetString(confKey, defaultWebpaHost) - - confKey = fmt.Sprintf("webconfig.%v.async_poke_enabled", webpaServiceName) - asyncPokeEnabled := conf.GetBoolean(confKey, false) - - confKey = fmt.Sprintf("webconfig.%v.async_poke_concurrent_calls", webpaServiceName) - concurrentCalls := int(conf.GetInt32(confKey, 0)) + host := conf.GetString("webconfig.webpa.host", defaultWebpaHost) + asyncPokeEnabled := conf.GetBoolean("webconfig.webpa.async_poke_enabled", false) + concurrentCalls := int(conf.GetInt32("webconfig.webpa.async_poke_concurrent_calls", 0)) var queue chan struct{} if concurrentCalls > 0 { queue = make(chan struct{}, concurrentCalls) } - confKey = fmt.Sprintf("webconfig.%v.retries", webpaServiceName) - retries := int(conf.GetInt32(confKey, defaultRetries)) - - confKey = fmt.Sprintf("webconfig.%v.retry_in_msecs", webpaServiceName) - retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) + retries := int(conf.GetInt32("webconfig.webpa.retries", defaultRetries)) + retryInMsecs := int(conf.GetInt32("webconfig.webpa.retry_in_msecs", defaultRetriesInMsecs)) + apiVersion := conf.GetString("webconfig.webpa.api_version", defaultApiVersion) + urlTemplate := conf.GetString("webconfig.webpa.url_template", defaultWebpaUrlTemplate) syncClient := NewHttpClient(conf, webpaServiceName, tlsConfig) syncClient.SetStatusHandler(520, syncHandle520) asyncClient := NewHttpClient(conf, asyncWebpaServiceName, tlsConfig) asyncClient.SetStatusHandler(520, asyncHandle520) - confKey = fmt.Sprintf("webconfig.%v.api_version", webpaServiceName) - apiVersion := conf.GetString(confKey, defaultApiVersion) - connector := WebpaConnector{ syncClient: syncClient, asyncClient: asyncClient, @@ -148,6 +140,7 @@ func NewWebpaConnector(conf *configuration.Config, tlsConfig *tls.Config) *Webpa retryInMsecs: retryInMsecs, asyncPokeEnabled: asyncPokeEnabled, apiVersion: apiVersion, + urlTemplate: urlTemplate, } return &connector @@ -161,6 +154,14 @@ func (c *WebpaConnector) SetWebpaHost(host string) { c.host = host } +func (c *WebpaConnector) WebpaUrlTemplate() string { + return c.urlTemplate +} + +func (c *WebpaConnector) SetWebpaUrlTemplate(x string) { + c.urlTemplate = x +} + func (c *WebpaConnector) ApiVersion() string { return c.apiVersion } @@ -187,7 +188,7 @@ func (c *WebpaConnector) SetAsyncPokeEnabled(enabled bool) { } func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { - url := fmt.Sprintf(webpaUrlTemplate, c.WebpaHost(), c.ApiVersion(), cpeMac) + url := fmt.Sprintf(c.WebpaUrlTemplate(), c.WebpaHost(), c.ApiVersion(), cpeMac) var traceId, xmTraceId string if itf, ok := fields["trace_id"]; ok { @@ -241,7 +242,7 @@ func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac } func (c *WebpaConnector) AsyncDoWithRetries(ctx context.Context, method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { - tfields := common.FilterLogFields(fields) + tfields := common.FilterLogFields(fields, "status") tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { cbytes := make([]byte, len(bbytes)) diff --git a/http/xconf_connector.go b/http/xconf_connector.go index 7a39974..024589d 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -29,25 +29,27 @@ import ( ) const ( - xconfHostDefault = "http://qa2.xconfds.coast.xcal.tv:8080" - xconfUrlTemplate = "%s/loguploader/getTelemetryProfiles?%s" + defaultXconfHost = "http://localhost:12346" + defaultXconfUrlTemplate = "%s/%s" ) type XconfConnector struct { *HttpClient host string serviceName string + urlTemplate string } func NewXconfConnector(conf *configuration.Config, tlsConfig *tls.Config) *XconfConnector { serviceName := "xconf" - confKey := fmt.Sprintf("webconfig.%v.host", serviceName) - host := conf.GetString(confKey, xconfHostDefault) + host := conf.GetString("webconfig.xconf.host", defaultXconfHost) + urlTemplate := conf.GetString("webconfig.xconf.url_template", defaultXconfUrlTemplate) return &XconfConnector{ HttpClient: NewHttpClient(conf, serviceName, tlsConfig), host: host, serviceName: serviceName, + urlTemplate: urlTemplate, } } @@ -59,12 +61,20 @@ func (c *XconfConnector) SetXconfHost(host string) { c.host = host } +func (c *XconfConnector) XconfUrlTemplate() string { + return c.urlTemplate +} + +func (c *XconfConnector) SetXconfUrlTemplate(x string) { + c.urlTemplate = x +} + func (c *XconfConnector) ServiceName() string { return c.serviceName } func (c *XconfConnector) GetProfiles(ctx context.Context, urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { - url := fmt.Sprintf(xconfUrlTemplate, c.XconfHost(), urlSuffix) + url := fmt.Sprintf(c.XconfUrlTemplate(), c.XconfHost(), urlSuffix) rbytes, resHeader, err := c.DoWithRetries(ctx, "GET", url, nil, nil, fields, c.ServiceName()) if err != nil { return rbytes, resHeader, owcommon.NewError(err) From 51f619e2fcfcc1ef5febfce9ae7203fab8fb12e0 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 14 Jan 2025 21:29:22 -0800 Subject: [PATCH 147/155] tracing codes rearranged --- common/log_fields.go | 3 ++ http/http_client.go | 74 +++++++++++++++++++++++----------------- http/response.go | 29 +++++++--------- http/response_writer.go | 18 +++++++++- http/router.go | 5 ++- http/webconfig_server.go | 71 +++++++++++++++++++++++--------------- tracing/otel.go | 50 +++++++++++---------------- tracing/span.go | 50 +++++++-------------------- tracing/tracer.go | 60 +++++++++++++++++--------------- util/print.go | 10 +++++- 10 files changed, 192 insertions(+), 178 deletions(-) diff --git a/common/log_fields.go b/common/log_fields.go index c14a280..10fd63a 100644 --- a/common/log_fields.go +++ b/common/log_fields.go @@ -27,6 +27,9 @@ var ( unloggedFields = []string{ "moneytrace", "token", + "xpc_trace", + "req_moracide_tags", + "resp_moracide_tags", } coreFields = []string{ "app_name", diff --git a/http/http_client.go b/http/http_client.go index db8ca45..4bbbb6f 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -62,6 +62,7 @@ type HttpClient struct { retryInMsecs int statusHandlerFuncMap map[int]StatusHandlerFunc userAgent string + moracideTagPrefix string } func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tls.Config) *HttpClient { @@ -84,6 +85,8 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl retryInMsecs := int(conf.GetInt32(confKey, defaultRetriesInMsecs)) userAgent := conf.GetString("webconfig.http_client.user_agent") + moracideTagPrefix := strings.ToLower(conf.GetString("webconfig.tracing.moracide_tag_prefix", tracing.DefaultMoracideTagPrefix)) + var transport http.RoundTripper = &http.Transport{ DialContext: (&net.Dialer{ Timeout: time.Duration(connectTimeout) * time.Second, @@ -106,6 +109,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl retryInMsecs: retryInMsecs, statusHandlerFuncMap: map[int]StatusHandlerFunc{}, userAgent: userAgent, + moracideTagPrefix: moracideTagPrefix, } } @@ -228,26 +232,8 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h // i.e. timeout, 503 etc. wouldn't have a valid resp, but possible that // the err returned by http.Do actually includes an err returned by the backend // In which case, resp would be non-nil - moracideTagPrefix := strings.ToLower(tracing.GetMoracideTagPrefix()) if res != nil { - for headerKey, headerVals := range res.Header { - if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { - respMoracideTagsFound = true - log.Debugf("http_client: moracide tag %s = %s found in response", headerKey, headerVals[0]) - if len(headerVals) > 1 { - log.Debugf("Tracing: moracide tag key = %s, has multiple values = %+v", headerKey, headerVals) - } - val := "false" - for _, v := range headerVals { - if v == "true" { - val = v - break - } - } - auditFields["resp_"+headerKey] = headerVals[0] - log.Debugf("Tracing: found moracide tag in response key = %s, val = %s", headerKey, val) - } - } + respMoracideTagsFound = c.addMoracideTagsFromResponse(res.Header, auditFields) } var endMessage string if retry > 0 { @@ -428,21 +414,45 @@ func (c *HttpClient) StatusHandler(status int) StatusHandlerFunc { // addMoracideTags - if ctx has a moracide tag as a header, add it to the headers // Also add traceparent, tracestate headers func (c *HttpClient) addMoracideTags(header http.Header, fields log.Fields) { - moracideTagPrefix := strings.ToLower("req_" + tracing.GetMoracideTagPrefix()) - for key, val := range fields { - if key == "out_traceparent" { - header.Set("traceparent", val.(string)) - } - if key == "out_tracestate" { - header.Set("tracestate", val.(string)) + if itf, ok := fields["out_traceparent"]; ok { + if ss, ok := itf.(string); ok { + if len(ss) > 0 { + header.Set(common.HeaderTraceparent, ss) + } } - if len(key) < 5 { - // Should be of the form "req_x-cl-experiment" - continue + } + if itf, ok := fields["out_tracestate"]; ok { + if ss, ok := itf.(string); ok { + if len(ss) > 0 { + header.Set(common.HeaderTracestate, ss) + } } - if strings.HasPrefix(strings.ToLower(key), moracideTagPrefix) { - log.Debugf("Adding moracide tag %s = %s to outgoing req", key, val) - header.Set(key[4:], val.(string)) + } + + itf, ok := fields["req_moracide_tags"] + if !ok { + return + } + moracideTagMap := itf.(map[string]string) + + for key, val := range moracideTagMap { + log.WithFields(fields).Debugf("Adding moracide tag %s = %s to outgoing req", key, val) + header.Set(key, val) + } +} + +func (c *HttpClient) addMoracideTagsFromResponse(header http.Header, fields log.Fields) bool { + var respMoracideTagsFound bool + m := make(map[string]string) + for k := range header { + if strings.HasPrefix(strings.ToLower(k), c.moracideTagPrefix) { + respMoracideTagsFound = true + val := header.Get(k) + m[k] = val } } + if len(m) > 0 { + fields["resp_moracide_tags"] = m + } + return respMoracideTagsFound } diff --git a/http/response.go b/http/response.go index 6c229be..5b748c3 100644 --- a/http/response.go +++ b/http/response.go @@ -21,11 +21,8 @@ import ( "encoding/json" "fmt" "net/http" - "strings" "github.com/rdkcentral/webconfig/common" - "github.com/rdkcentral/webconfig/tracing" - log "github.com/sirupsen/logrus" ) const ( @@ -176,23 +173,21 @@ func addMoracideTagsAsResponseHeaders(w http.ResponseWriter) { return } - reqMoracideTagPrefix := strings.ToLower("req_" + tracing.GetMoracideTagPrefix()) - respMoracideTagPrefix := strings.ToLower("resp_" + tracing.GetMoracideTagPrefix()) - fields := xw.Audit() + reqMoracideTags := xw.ReqMoracideTags() + respMoracideTags := xw.RespMoracideTags() + moracideTags := make(map[string]string) - for key, val := range fields { - if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { - log.Debugf("Adding moracide tag from req %s = %s to response", key, val) - moracideTags[key[4:]] = val.(string) - } - if strings.HasPrefix(strings.ToLower(key), respMoracideTagPrefix) { - log.Debugf("Adding moracide tag from resp %s = %s to response", key, val) - realKey := key[5:] - if existingVal, ok := moracideTags[realKey]; !ok || (ok && existingVal != "true") { - moracideTags[realKey] = val.(string) - } + for key, val := range reqMoracideTags { + xw.LogDebug(nil, "request", fmt.Sprintf("Adding moracide tag from req %s = %s to response", key, val)) + moracideTags[key] = val + } + for key, val := range respMoracideTags { + if existingVal, ok := moracideTags[key]; !ok || (ok && existingVal != "true") { + xw.LogDebug(nil, "request", fmt.Sprintf("Adding moracide tag from resp %s = %s to response", key, val)) + moracideTags[key] = val } } + xw.LogDebug(nil, "request", fmt.Sprintf("moracideTags = %+v", moracideTags)) for key, val := range moracideTags { w.Header().Set(key, val) } diff --git a/http/response_writer.go b/http/response_writer.go index 9f0372c..ecc7e0b 100644 --- a/http/response_writer.go +++ b/http/response_writer.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -165,3 +165,19 @@ func (w *XResponseWriter) SetPartnerId(partnerId string) { w.partnerId = partnerId w.audit["partner"] = partnerId } + +func (w *XResponseWriter) ReqMoracideTags() map[string]string { + itf, ok := w.audit["req_moracide_tags"] + if !ok { + return nil + } + return itf.(map[string]string) +} + +func (w *XResponseWriter) RespMoracideTags() map[string]string { + itf, ok := w.audit["resp_moracide_tags"] + if !ok { + return nil + } + return itf.(map[string]string) +} diff --git a/http/router.go b/http/router.go index 4969326..103c77a 100644 --- a/http/router.go +++ b/http/router.go @@ -19,7 +19,6 @@ package http import ( "github.com/gorilla/mux" - "github.com/rdkcentral/webconfig/tracing" ) func (s *WebconfigServer) AddBaseRoutes(testOnly bool, router *mux.Router) { @@ -96,9 +95,9 @@ func (s *WebconfigServer) GetRouter(testOnly bool) *mux.Router { sub2.Use(s.TestingMiddleware) } else { if s.ServerApiTokenAuthEnabled() { - sub2.Use(tracing.SpanMiddleware, s.ApiMiddleware) + sub2.Use(s.SpanMiddleware, s.ApiMiddleware) } else { - sub2.Use(tracing.SpanMiddleware, s.NoAuthMiddleware) + sub2.Use(s.SpanMiddleware, s.NoAuthMiddleware) } } sub2.HandleFunc("", s.PokeHandler).Methods("POST") diff --git a/http/webconfig_server.go b/http/webconfig_server.go index fbc7d0c..67c24bc 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -43,6 +43,7 @@ import ( "github.com/rdkcentral/webconfig/tracing" "github.com/rdkcentral/webconfig/util" log "github.com/sirupsen/logrus" + sdktrace "go.opentelemetry.io/otel/sdk/trace" ) // TODO enum, probably no need @@ -330,7 +331,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer } func (s *WebconfigServer) Stop() { - tracing.StopXpcTracer() + s.StopXpcTracer() } func (s *WebconfigServer) TestingMiddleware(next http.Handler) http.Handler { @@ -730,7 +731,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques token = elements[1] } - var xmTraceId, traceId string + var xmTraceId string // extract moneytrace from the header tracePart := strings.Split(r.Header.Get("X-Moneytrace"), ";")[0] @@ -740,12 +741,6 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } } - // extract traceparent from the header - traceparent := r.Header.Get(common.HeaderTraceparent) - if len(traceparent) == 55 { - traceId = traceparent[3:35] - } - // extract auditid from the header auditId := r.Header.Get("X-Auditid") if len(auditId) == 0 { @@ -753,27 +748,29 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } // traceparent handling for E2E tracing - xpcTrace := tracing.NewXpcTrace(r) + xpcTrace := tracing.NewXpcTrace(r, s.XpcTracer) + traceId := xpcTrace.TraceID + if len(traceId) == 0 { + traceId = xmTraceId + } headerMap := util.HeaderToMap(header) fields := log.Fields{ - "path": r.URL.String(), - "method": r.Method, - "audit_id": auditId, - "remote_ip": remoteIp, - "host_name": host, - "header": headerMap, - "logger": "request", - "trace_id": traceId, - "app_name": s.AppName(), - "traceparent": xpcTrace.ReqTraceparent, - "tracestate": xpcTrace.ReqTracestate, - "out_traceparent": xpcTrace.OutTraceparent, - "out_tracestate": xpcTrace.OutTracestate, - "xpc_trace": xpcTrace, - } - for key, val := range xpcTrace.ReqMoracideTags { - fields["req_"+key] = val + "path": r.URL.String(), + "method": r.Method, + "audit_id": auditId, + "remote_ip": remoteIp, + "host_name": host, + "header": headerMap, + "logger": "request", + "trace_id": traceId, + "app_name": s.AppName(), + "traceparent": xpcTrace.ReqTraceparent, + "tracestate": xpcTrace.ReqTracestate, + "out_traceparent": xpcTrace.OutTraceparent, + "out_tracestate": xpcTrace.OutTracestate, + "req_moracide_tags": xpcTrace.ReqMoracideTags, + "xpc_trace": xpcTrace, } userAgent := r.UserAgent() @@ -902,8 +899,8 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { fields["duration"] = duration fields["logger"] = "request" - tracing.SetSpanStatusCode(fields) - tracing.SetSpanMoracideTags(fields) + tracing.SetSpanStatusCode(s.XpcTracer, fields) + tracing.SetSpanMoracideTags(s.XpcTracer, fields) var userAgent string if itf, ok := fields["user_agent"]; ok { @@ -1118,3 +1115,21 @@ func (s *WebconfigServer) HandleKafkaProducerResults() { } } } + +func (s *WebconfigServer) StopXpcTracer() { + sdkTraceProvider, ok := s.XpcTracer.OtelTracerProvider().(*sdktrace.TracerProvider) + if ok && sdkTraceProvider != nil { + sdkTraceProvider.Shutdown(context.TODO()) + } +} + +func (s *WebconfigServer) SpanMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if s.XpcTracer.OtelEnabled { + ctx, otelSpan := tracing.NewOtelSpan(r, s.XpcTracer) + r = r.WithContext(ctx) + defer tracing.EndOtelSpan(s.XpcTracer, otelSpan) + } + next.ServeHTTP(w, r) + }) +} diff --git a/tracing/otel.go b/tracing/otel.go index 66dfb69..c9dd75a 100644 --- a/tracing/otel.go +++ b/tracing/otel.go @@ -42,7 +42,7 @@ import ( log "github.com/sirupsen/logrus" ) -type providerConstructor func() (oteltrace.TracerProvider, error) +type providerConstructor func(*XpcTracer) (oteltrace.TracerProvider, error) var ( providerBuilders = map[string]providerConstructor{ @@ -59,7 +59,7 @@ const defaultOtelTracerProvider = "noop" // initOtel - initialize OpenTelemetry constructs // tracing instrumentation code. -func otelInit(conf *configuration.Config) { +func otelInit(conf *configuration.Config, xpcTracer *XpcTracer) { xpcTracer.OtelEnabled = conf.GetBoolean("webconfig.tracing.otel.enabled") if !xpcTracer.OtelEnabled { return @@ -82,7 +82,7 @@ func otelInit(conf *configuration.Config) { return } else { var err error - if xpcTracer.otelTracerProvider, err = providerBuilder(); err != nil { + if xpcTracer.otelTracerProvider, err = providerBuilder(xpcTracer); err != nil { log.Errorf("building otel provider for %s failed with %v", xpcTracer.otelProvider, err) return } @@ -99,11 +99,11 @@ func otelInit(conf *configuration.Config) { xpcTracer.otelTracer = otel.Tracer(xpcTracer.appName) } -func otelNoopTraceProvider() (oteltrace.TracerProvider, error) { +func otelNoopTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, error) { return noop.NewTracerProvider(), nil } -func otelStdoutTraceProvider() (oteltrace.TracerProvider, error) { +func otelStdoutTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, error) { option := stdouttrace.WithPrettyPrint() exporter, err := stdouttrace.New(option) if err != nil { @@ -124,7 +124,7 @@ func otelStdoutTraceProvider() (oteltrace.TracerProvider, error) { return tp, nil } -func otelHttpTraceProvider() (oteltrace.TracerProvider, error) { +func otelHttpTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, error) { // Send traces over HTTP if xpcTracer.otelEndpoint == "" { return nil, fmt.Errorf("building http otel provider failure, no endpoint specified") @@ -149,23 +149,7 @@ func otelHttpTraceProvider() (oteltrace.TracerProvider, error) { ), nil } -func otelShutdown() { - sdkTraceProvider, ok := xpcTracer.otelTracerProvider.(*sdktrace.TracerProvider) - if ok && sdkTraceProvider != nil { - sdkTraceProvider.Shutdown(context.TODO()) - } -} - -// otelOpName should return "http.request" by default -func otelOpName() string { - opName := xpcTracer.otelOpName - if opName == "" { - opName = "http.request" - } - return opName -} - -func otelNewSpan(r *http.Request) (context.Context, oteltrace.Span) { +func NewOtelSpan(r *http.Request, xpcTracer *XpcTracer) (context.Context, oteltrace.Span) { ctx := r.Context() var otelSpan oteltrace.Span if !xpcTracer.OtelEnabled { @@ -191,7 +175,7 @@ func otelNewSpan(r *http.Request) (context.Context, oteltrace.Span) { custom Comcast attribute: X-Cl-Experiment: true/false additional: env, operation.name, http.url_details.path */ - ctx, otelSpan = xpcTracer.otelTracer.Start(ctx, otelOpName(), + ctx, otelSpan = xpcTracer.otelTracer.Start(ctx, xpcTracer.OtelOpName(), oteltrace.WithSpanKind(oteltrace.SpanKindServer), oteltrace.WithAttributes( attribute.String("env", xpcTracer.appEnv), @@ -199,11 +183,11 @@ func otelNewSpan(r *http.Request) (context.Context, oteltrace.Span) { attribute.String("http.route", pathTemplate), attribute.String("http.url", r.URL.String()), attribute.String("http.url_details.path", r.URL.Path), - attribute.String("operation.name", otelOpName()), + attribute.String("operation.name", xpcTracer.OtelOpName()), ), ) - if xpcTracer.rgn != "" { - rgnAttr := attribute.String("region", xpcTracer.rgn) + if xpcTracer.region != "" { + rgnAttr := attribute.String("region", xpcTracer.region) otelSpan.SetAttributes(rgnAttr) } @@ -213,7 +197,7 @@ func otelNewSpan(r *http.Request) (context.Context, oteltrace.Span) { log.Debugf("added span attribute key = http.route, value = %s", pathTemplate) log.Debugf("added span attribute key = http.url, value = %s", r.URL.String()) log.Debugf("added span attribute key = http.url_details.path, value = %s", r.URL.Path) - log.Debugf("added span attribute key = operation.name, value = %s", otelOpName()) + log.Debugf("added span attribute key = operation.name, value = %s", xpcTracer.OtelOpName()) carrier := propagation.MapCarrier{} otel.GetTextMapPropagator().Inject(ctx, carrier) @@ -239,20 +223,26 @@ func otelSetStatusCode(span oteltrace.Span, statusCode int) { } } -func otelEndSpan(span oteltrace.Span) { +func EndOtelSpan(xpcTracer *XpcTracer, span oteltrace.Span) { if !xpcTracer.OtelEnabled { return } span.End() } -func otelExtractParamsFromSpan(ctx context.Context, xpcTrace *XpcTrace) { +func otelExtractParamsFromSpan(ctx context.Context, xpcTracer *XpcTracer, xpcTrace *XpcTrace) { if !xpcTracer.OtelEnabled { return } if tmp := GetContext(ctx, "otel_span"); tmp != nil { if otelSpan, ok := tmp.(oteltrace.Span); ok { xpcTrace.otelSpan = otelSpan + if xpcTrace.otelSpan == nil { + return + } + + spanCtx := otelSpan.SpanContext() + xpcTrace.TraceID = spanCtx.TraceID().String() } } if xpcTrace.otelSpan == nil { diff --git a/tracing/span.go b/tracing/span.go index c79c0e3..ba7edc1 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -28,19 +28,6 @@ import ( oteltrace "go.opentelemetry.io/otel/trace" ) -// SpanMiddleware is a middleware that creates a new span for each incoming request. -func SpanMiddleware(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // For Otel, create the span explicitly - if xpcTracer.OtelEnabled { - ctx, otelSpan := otelNewSpan(r) - r = r.WithContext(ctx) - defer otelEndSpan(otelSpan) - } - next.ServeHTTP(w, r) - }) -} - // XpcTrace is a carrier/baggage struct to extract data from spans, request headers for usage later // Store the trace in ctx for easy retrieval. // Ideal place to store it is ofc, xw @@ -81,20 +68,20 @@ type XpcTrace struct { } // NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs -func NewXpcTrace(r *http.Request) *XpcTrace { +func NewXpcTrace(r *http.Request, xpcTracer *XpcTracer) *XpcTrace { var xpcTrace XpcTrace xpcTrace.ReqMoracideTags = make(map[string]string) - extractParamsFromReq(r, &xpcTrace) + extractParamsFromReq(r, xpcTracer, &xpcTrace) if xpcTracer.OtelEnabled { - otelExtractParamsFromSpan(r.Context(), &xpcTrace) + otelExtractParamsFromSpan(r.Context(), xpcTracer, &xpcTrace) } return &xpcTrace } -func SetSpanStatusCode(fields log.Fields) { +func SetSpanStatusCode(xpcTracer *XpcTracer, fields log.Fields) { var xpcTrace *XpcTrace if tmp, ok := fields["xpc_trace"]; ok { xpcTrace = tmp.(*XpcTrace) @@ -102,6 +89,7 @@ func SetSpanStatusCode(fields log.Fields) { if xpcTrace == nil { // Something went wrong, cannot instrument this span log.Error("instrumentation error, no trace info") + return } if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { if tmp, ok := fields["status"]; ok { @@ -111,7 +99,7 @@ func SetSpanStatusCode(fields log.Fields) { } } -func SetSpanMoracideTags(fields log.Fields) { +func SetSpanMoracideTags(xpcTracer *XpcTracer, fields log.Fields) { var xpcTrace *XpcTrace if tmp, ok := fields["xpc_trace"]; ok { xpcTrace = tmp.(*XpcTrace) @@ -119,14 +107,12 @@ func SetSpanMoracideTags(fields log.Fields) { if xpcTrace == nil { // Something went wrong, cannot instrument this span log.Error("instrumentation error, cannot set moracide tags, no trace info") + return } - if tmp, ok := fields["xpc_trace"]; ok { - xpcTrace = tmp.(*XpcTrace) - } moracideTags := make(map[string]string) - reqMoracideTagPrefix := strings.ToLower("req_" + xpcTracer.MoracideTagPrefix) - respMoracideTagPrefix := strings.ToLower("resp_" + xpcTracer.MoracideTagPrefix) + reqMoracideTagPrefix := strings.ToLower("req_" + xpcTracer.MoracideTagPrefix()) + respMoracideTagPrefix := strings.ToLower("resp_" + xpcTracer.MoracideTagPrefix()) for key, val := range fields { if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { @@ -144,7 +130,7 @@ func SetSpanMoracideTags(fields log.Fields) { if len(moracideTags) == 0 { // No moracide tags in request or any response // So set at least one span tag, x-cl-expt: false - moracideTags[xpcTracer.MoracideTagPrefix] = "false" + moracideTags[xpcTracer.MoracideTagPrefix()] = "false" } for key, val := range moracideTags { if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { @@ -154,30 +140,18 @@ func SetSpanMoracideTags(fields log.Fields) { } } -func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace) { +func extractParamsFromReq(r *http.Request, xpcTracer *XpcTracer, xpcTrace *XpcTrace) { xpcTrace.ReqTraceparent = r.Header.Get(common.HeaderTraceparent) xpcTrace.ReqTracestate = r.Header.Get(common.HeaderTracestate) xpcTrace.OutTraceparent = xpcTrace.ReqTraceparent xpcTrace.OutTracestate = xpcTrace.ReqTracestate log.Debugf("Tracing: input traceparent : %s, tracestate : %s", xpcTrace.ReqTraceparent, xpcTrace.ReqTracestate) - // AuditID is used internally within xdp subsystem - // We will not move this to tracing module - // xpcTrace.AuditID = r.Header.Get(AuditIDHeader) - // if xpcTrace.AuditID == "" { - // xpcTrace.AuditID = util.GetAuditId() - // } - - // Moneytrace is Comcast's original solution for tracing - // Unused for now, decide whether we need to merge existing moneytrace code here - // Being replaced by traceparent/tracestate headers anyway - // xpcTrace.MoneyTrace = r.Header.Get(MoneyTraceHeader) - xpcTrace.ReqUserAgent = r.Header.Get(UserAgentHeader) // In future, -H 'X-Cl-Experiment-1', -H 'X-Cl-Experiment-oswebconfig'... OR 'X-Cl-Experiment-xapproxy_25.1.1.1' are all possible // So walk through all headers and collect any header that starts with this prefix - moracideTagPrefix := strings.ToLower(xpcTracer.MoracideTagPrefix) + moracideTagPrefix := strings.ToLower(xpcTracer.MoracideTagPrefix()) for headerKey, headerVals := range r.Header { if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { if len(headerVals) > 1 { diff --git a/tracing/tracer.go b/tracing/tracer.go index 8f3be02..fffe1f2 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -29,24 +29,23 @@ import ( ) const ( - AuditIDHeader = "X-Auditid" - UserAgentHeader = "User-Agent" - - defaultMoracideTagPrefix = "X-Cl-Experiment" + AuditIDHeader = "X-Auditid" + UserAgentHeader = "User-Agent" + DefaultMoracideTagPrefix = "X-Cl-Experiment" ) // XpcTracer is a wrapper around tracer setup type XpcTracer struct { OtelEnabled bool - MoracideTagPrefix string // Special request header for moracide expts e.g. canary deployments + moracideTagPrefix string // Special request header for moracide expts e.g. canary deployments // internal vars used by Otel appEnv string // set this to dev for red, staging for yellow and prod for green appName string appVersion string appSHA string // unused - rgn string // AWS Region e.g. us-west-2, unused, use it as a otel span attribute - siteColor string // red/yellow/green, unused, use it as a otel span attribute + region string // AWS Region e.g. us-west-2, unused, use it as a span tagattribute + siteColor string // red/yellow/green, unused, use it as a span attribute // internal otel vars otelEndpoint string @@ -57,26 +56,35 @@ type XpcTracer struct { otelTracer oteltrace.Tracer } -var xpcTracer XpcTracer // global tracer - func NewXpcTracer(conf *configuration.Config) *XpcTracer { - initAppData(conf) - otelInit(conf) - xpcTracer.MoracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", defaultMoracideTagPrefix) - return &xpcTracer + xpcTracer := new(XpcTracer) + initAppData(conf, xpcTracer) + otelInit(conf, xpcTracer) + xpcTracer.moracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", DefaultMoracideTagPrefix) + return xpcTracer } -// defer this func in the main of the app -func StopXpcTracer() { - otelShutdown() +func (t *XpcTracer) MoracideTagPrefix() string { + return t.moracideTagPrefix +} + +// otelOpName should return "http.request" by default +func (t *XpcTracer) OtelOpName() string { + if len(t.otelOpName) == 0 { + return "http.request" + } + return t.otelOpName } -// Global func to access the moracide tag -func GetMoracideTagPrefix() string { - return xpcTracer.MoracideTagPrefix +func (t *XpcTracer) OtelTracerProvider() oteltrace.TracerProvider { + return t.otelTracerProvider } -func initAppData(conf *configuration.Config) { +func (t *XpcTracer) AppName() string { + return t.appName +} + +func initAppData(conf *configuration.Config, xpcTracer *XpcTracer) { codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") xpcTracer.appName = codeGitCommit[0] if len(codeGitCommit) > 1 { @@ -94,13 +102,9 @@ func initAppData(conf *configuration.Config) { } else if strings.EqualFold(siteColor, "green") { xpcTracer.appEnv = "prod" } - xpcTracer.rgn = os.Getenv("site_region") - if xpcTracer.rgn == "" { - xpcTracer.rgn = os.Getenv("site_region_name") + xpcTracer.region = os.Getenv("site_region") + if xpcTracer.region == "" { + xpcTracer.region = os.Getenv("site_region_name") } - log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.rgn) -} - -func GetServiceName() string { - return xpcTracer.appName + log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.region) } diff --git a/util/print.go b/util/print.go index e8d55bd..afe7de4 100644 --- a/util/print.go +++ b/util/print.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package util import ( @@ -40,6 +40,14 @@ func PrettyJson(input interface{}) string { if bbytes, err := json.MarshalIndent(input, "", " "); err == nil { pretty = string(bbytes) } + case http.Header: + m := make(map[string]string) + for k := range ty { + m[k] = ty.Get(k) + } + if bbytes, err := json.MarshalIndent(m, "", " "); err == nil { + pretty = string(bbytes) + } } return pretty From 13e8d5c3ea959ba384c08838cb08fb5c401d73bc Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 15 Jan 2025 13:29:52 -0800 Subject: [PATCH 148/155] remove unused func args --- http/http_client.go | 7 +++---- http/mqtt_connector.go | 2 +- http/upstream_connector.go | 4 ++-- http/webpa_connector.go | 8 ++++---- http/xconf_connector.go | 2 +- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/http/http_client.go b/http/http_client.go index 4bbbb6f..71ff1f9 100644 --- a/http/http_client.go +++ b/http/http_client.go @@ -19,7 +19,6 @@ package http import ( "bytes" - "context" "crypto/tls" "encoding/base64" "encoding/json" @@ -113,7 +112,7 @@ func NewHttpClient(conf *configuration.Config, serviceName string, tlsConfig *tl } } -func (c *HttpClient) Do(ctx context.Context, method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { +func (c *HttpClient) Do(method string, url string, header http.Header, bbytes []byte, auditFields log.Fields, loggerName string, retry int) ([]byte, http.Header, bool, error) { fields := common.FilterLogFields(auditFields, "status") var respMoracideTagsFound bool @@ -374,7 +373,7 @@ func (c *HttpClient) Do(ctx context.Context, method string, url string, header h return rbytes, res.Header, false, nil } -func (c *HttpClient) DoWithRetries(ctx context.Context, method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { +func (c *HttpClient) DoWithRetries(method string, url string, rHeader http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, http.Header, error) { var respBytes []byte var respHeader http.Header var err error @@ -388,7 +387,7 @@ func (c *HttpClient) DoWithRetries(ctx context.Context, method string, url strin if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - respBytes, respHeader, cont, err = c.Do(ctx, method, url, rHeader, cbytes, fields, loggerName, i) + respBytes, respHeader, cont, err = c.Do(method, url, rHeader, cbytes, fields, loggerName, i) if !cont { break } diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index 034c3bb..d658cd2 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -105,7 +105,7 @@ func (c *MqttConnector) PostMqtt(ctx context.Context, cpeMac string, bbytes []by header.Set(common.HeaderTraceparent, outTraceparent) header.Set(common.HeaderTracestate, outTracestate) - rbytes, _, err := c.DoWithRetries(ctx, "POST", url, header, bbytes, fields, c.ServiceName()) + rbytes, _, err := c.DoWithRetries("POST", url, header, bbytes, fields, c.ServiceName()) if err != nil { return rbytes, common.NewError(err) } diff --git a/http/upstream_connector.go b/http/upstream_connector.go index 2a3520a..f5037d2 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -87,7 +87,7 @@ func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header } } - rbytes, header, err := c.DoWithRetries(ctx, "POST", url, header, bbytes, fields, c.ServiceName()) + rbytes, header, err := c.DoWithRetries("POST", url, header, bbytes, fields, c.ServiceName()) if err != nil { return rbytes, header, owcommon.NewError(err) } @@ -111,7 +111,7 @@ func (c *UpstreamConnector) GetUpstreamProfiles(ctx context.Context, mac, queryP } } - rbytes, header, err := c.DoWithRetries(ctx, "GET", url, header, nil, fields, c.ServiceName()) + rbytes, header, err := c.DoWithRetries("GET", url, header, nil, fields, c.ServiceName()) if err != nil { return rbytes, header, owcommon.NewError(err) } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index 014f291..d6cd7cc 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -212,7 +212,7 @@ func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac header.Set("X-Webpa-Transaction-Id", transactionId) method := "PATCH" - _, _, cont, err := c.syncClient.Do(ctx, method, url, header, bbytes, fields, webpaServiceName, 0) + _, _, cont, err := c.syncClient.Do(method, url, header, bbytes, fields, webpaServiceName, 0) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -230,7 +230,7 @@ func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac } } if cont { - _, _, err := c.syncClient.DoWithRetries(ctx, "PATCH", url, header, bbytes, fields, webpaServiceName) + _, _, err := c.syncClient.DoWithRetries("PATCH", url, header, bbytes, fields, webpaServiceName) if err != nil { return transactionId, common.NewError(err) } @@ -250,7 +250,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(ctx context.Context, method string, if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - _, _, cont, _ := c.asyncClient.Do(ctx, method, url, header, cbytes, fields, loggerName, i) + _, _, cont, _ := c.asyncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { msg := fmt.Sprintf("finished success after 1 retry") if i > 1 { @@ -278,7 +278,7 @@ func (c *WebpaConnector) SyncDoWithRetries(ctx context.Context, method string, u if i > 0 { time.Sleep(time.Duration(c.retryInMsecs) * time.Millisecond) } - rbytes, _, cont, err = c.syncClient.Do(ctx, method, url, header, cbytes, fields, loggerName, i) + rbytes, _, cont, err = c.syncClient.Do(method, url, header, cbytes, fields, loggerName, i) if !cont { // in the case of 524/in-progress, we continue var rherr common.RemoteHttpError diff --git a/http/xconf_connector.go b/http/xconf_connector.go index 024589d..5057a5e 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -75,7 +75,7 @@ func (c *XconfConnector) ServiceName() string { func (c *XconfConnector) GetProfiles(ctx context.Context, urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { url := fmt.Sprintf(c.XconfUrlTemplate(), c.XconfHost(), urlSuffix) - rbytes, resHeader, err := c.DoWithRetries(ctx, "GET", url, nil, nil, fields, c.ServiceName()) + rbytes, resHeader, err := c.DoWithRetries("GET", url, nil, nil, fields, c.ServiceName()) if err != nil { return rbytes, resHeader, owcommon.NewError(err) } From 087035a716ce648b8dcc41b901933f17cbda9ca5 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 15 Jan 2025 14:02:24 -0800 Subject: [PATCH 149/155] clean up unused contexts --- http/mqtt_connector.go | 3 +-- http/multipart.go | 4 ++-- http/poke_handler.go | 7 ++----- http/supplementary_handler.go | 6 ++---- http/upstream_connector.go | 5 ++--- http/upstream_connector_test.go | 4 +--- http/webconfig_server.go | 4 ++-- http/webpa_connector.go | 11 +++++------ http/xconf_connector.go | 3 +-- kafka/consumer.go | 2 +- 10 files changed, 19 insertions(+), 30 deletions(-) diff --git a/http/mqtt_connector.go b/http/mqtt_connector.go index d658cd2..66ac2a6 100644 --- a/http/mqtt_connector.go +++ b/http/mqtt_connector.go @@ -18,7 +18,6 @@ package http import ( - "context" "crypto/tls" "fmt" "net/http" @@ -75,7 +74,7 @@ func (c *MqttConnector) ServiceName() string { return c.serviceName } -func (c *MqttConnector) PostMqtt(ctx context.Context, cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { +func (c *MqttConnector) PostMqtt(cpeMac string, bbytes []byte, fields log.Fields) ([]byte, error) { url := fmt.Sprintf(c.MqttUrlTemplate(), c.MqttHost(), cpeMac) var traceId, xmTraceId, outTraceparent, outTracestate string diff --git a/http/multipart.go b/http/multipart.go index a431552..91f55fc 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -287,7 +287,7 @@ func BuildWebconfigResponse(s *WebconfigServer, ctx context.Context, rHeader htt upstreamHeader.Set(common.HeaderUpstreamOldSchemaVersion, oldRootDocument.SchemaVersion) } - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(ctx, mac, upstreamHeader, respBytes, fields) + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, respBytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -410,7 +410,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, ctx context.Context, rHeader } // call /upstream to handle factory reset - upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(ctx, mac, upstreamHeader, oldDocBytes, fields) + upstreamRespBytes, upstreamRespHeader, err := s.PostUpstream(mac, upstreamHeader, oldDocBytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { diff --git a/http/poke_handler.go b/http/poke_handler.go index 902441b..29e19f3 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -32,9 +32,6 @@ import ( ) func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { - // tracing propagation - ctx := r.Context() - // handler params := mux.Vars(r) mac := params["mac"] @@ -107,7 +104,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - _, err = s.PostMqtt(ctx, deviceId, mbytes, fields) + _, err = s.PostMqtt(deviceId, mbytes, fields) if err != nil { var rherr common.RemoteHttpError if errors.As(err, &rherr) { @@ -160,7 +157,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { return } - transactionId, err := s.Poke(ctx, r.Header, mac, token, pokeStr, fields) + transactionId, err := s.Poke(r.Header, mac, token, pokeStr, fields) if err != nil { var rherr common.RemoteHttpError diff --git a/http/supplementary_handler.go b/http/supplementary_handler.go index 401c4b7..20dc8fd 100644 --- a/http/supplementary_handler.go +++ b/http/supplementary_handler.go @@ -34,8 +34,6 @@ const ( ) func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - // ==== data integrity check ==== params := mux.Vars(r) mac, ok := params["mac"] @@ -82,7 +80,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r urlSuffix := util.GetTelemetryQueryString(r.Header, mac, queryParams, partnerId) fields["is_telemetry"] = true - baseProfileBytes, resHeader, err := s.GetProfiles(ctx, urlSuffix, fields) + baseProfileBytes, resHeader, err := s.GetProfiles(urlSuffix, fields) xconfNotFound := false if err != nil { var rherr common.RemoteHttpError @@ -108,7 +106,7 @@ func (s *WebconfigServer) MultipartSupplementaryHandler(w http.ResponseWriter, r var profileBytes []byte if s.UpstreamProfilesEnabled() && rootdoc != nil && len(rootdoc.QueryParams) > 0 { // Get profiles from the second source - extraProfileBytes, _, err := s.GetUpstreamProfiles(ctx, mac, queryParams, r.Header, fields) + extraProfileBytes, _, err := s.GetUpstreamProfiles(mac, queryParams, r.Header, fields) if err != nil { exitNow := true var rherr common.RemoteHttpError diff --git a/http/upstream_connector.go b/http/upstream_connector.go index f5037d2..cae0e4a 100644 --- a/http/upstream_connector.go +++ b/http/upstream_connector.go @@ -18,7 +18,6 @@ package http import ( - "context" "crypto/tls" "fmt" "net/http" @@ -70,7 +69,7 @@ func (c *UpstreamConnector) ServiceName() string { return c.serviceName } -func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { +func (c *UpstreamConnector) PostUpstream(mac string, header http.Header, bbytes []byte, fields log.Fields) ([]byte, http.Header, error) { url := fmt.Sprintf(c.upstreamUrlTemplate, c.UpstreamHost(), mac) if itf, ok := fields["audit_id"]; ok { @@ -94,7 +93,7 @@ func (c *UpstreamConnector) PostUpstream(ctx context.Context, mac string, header return rbytes, header, nil } -func (c *UpstreamConnector) GetUpstreamProfiles(ctx context.Context, mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { +func (c *UpstreamConnector) GetUpstreamProfiles(mac, queryParams string, header http.Header, fields log.Fields) ([]byte, http.Header, error) { url := fmt.Sprintf(c.profileUrlTemplate, c.UpstreamHost(), mac, queryParams) if itf, ok := fields["audit_id"]; ok { diff --git a/http/upstream_connector_test.go b/http/upstream_connector_test.go index 13fddbb..d09e1a7 100644 --- a/http/upstream_connector_test.go +++ b/http/upstream_connector_test.go @@ -18,7 +18,6 @@ package http import ( - "context" "net/http" "net/http/httptest" "testing" @@ -52,7 +51,6 @@ func TestUpstreamConnector(t *testing.T) { bbytes := []byte("hello world") var err error fields := log.Fields{} - ctx := context.Background() - _, _, err = server.PostUpstream(ctx, mac, header, bbytes, fields) + _, _, err = server.PostUpstream(mac, header, bbytes, fields) assert.NilError(t, err) } diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 67c24bc..2941f31 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -701,9 +701,9 @@ func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { return fmt.Errorf("invalid partner") } -func (c *WebconfigServer) Poke(ctx context.Context, rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { +func (c *WebconfigServer) Poke(rHeader http.Header, cpeMac string, token string, pokeStr string, fields log.Fields) (string, error) { body := fmt.Sprintf(common.PokeBodyTemplate, pokeStr) - transactionId, err := c.Patch(ctx, rHeader, cpeMac, token, []byte(body), fields) + transactionId, err := c.Patch(rHeader, cpeMac, token, []byte(body), fields) if err != nil { return "", common.NewError(err) } diff --git a/http/webpa_connector.go b/http/webpa_connector.go index d6cd7cc..95b18fa 100644 --- a/http/webpa_connector.go +++ b/http/webpa_connector.go @@ -18,7 +18,6 @@ package http import ( - "context" "crypto/tls" "encoding/json" "errors" @@ -187,7 +186,7 @@ func (c *WebpaConnector) SetAsyncPokeEnabled(enabled bool) { c.asyncPokeEnabled = enabled } -func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { +func (c *WebpaConnector) Patch(rHeader http.Header, cpeMac string, token string, bbytes []byte, fields log.Fields) (string, error) { url := fmt.Sprintf(c.WebpaUrlTemplate(), c.WebpaHost(), c.ApiVersion(), cpeMac) var traceId, xmTraceId string @@ -219,9 +218,9 @@ func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac if rherr.StatusCode == 524 { if c.asyncPokeEnabled { c.queue <- struct{}{} - go c.AsyncDoWithRetries(ctx, method, url, header, bbytes, fields, asyncWebpaServiceName) + go c.AsyncDoWithRetries(method, url, header, bbytes, fields, asyncWebpaServiceName) } else { - _, err := c.SyncDoWithRetries(ctx, method, url, header, bbytes, fields, webpaServiceName) + _, err := c.SyncDoWithRetries(method, url, header, bbytes, fields, webpaServiceName) if err != nil { return transactionId, common.NewError(err) } @@ -241,7 +240,7 @@ func (c *WebpaConnector) Patch(ctx context.Context, rHeader http.Header, cpeMac return transactionId, nil } -func (c *WebpaConnector) AsyncDoWithRetries(ctx context.Context, method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { +func (c *WebpaConnector) AsyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) { tfields := common.FilterLogFields(fields, "status") tfields["logger"] = "asyncwebpa" for i := 1; i <= c.retries; i++ { @@ -267,7 +266,7 @@ func (c *WebpaConnector) AsyncDoWithRetries(ctx context.Context, method string, } // this has 1 less retries compared to the standard DoWithRetries() -func (c *WebpaConnector) SyncDoWithRetries(ctx context.Context, method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, error) { +func (c *WebpaConnector) SyncDoWithRetries(method string, url string, header http.Header, bbytes []byte, fields log.Fields, loggerName string) ([]byte, error) { var rbytes []byte var err error var cont bool diff --git a/http/xconf_connector.go b/http/xconf_connector.go index 5057a5e..0c46ec0 100644 --- a/http/xconf_connector.go +++ b/http/xconf_connector.go @@ -18,7 +18,6 @@ package http import ( - "context" "crypto/tls" "fmt" "net/http" @@ -73,7 +72,7 @@ func (c *XconfConnector) ServiceName() string { return c.serviceName } -func (c *XconfConnector) GetProfiles(ctx context.Context, urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { +func (c *XconfConnector) GetProfiles(urlSuffix string, fields log.Fields) ([]byte, http.Header, error) { url := fmt.Sprintf(c.XconfUrlTemplate(), c.XconfHost(), urlSuffix) rbytes, resHeader, err := c.DoWithRetries("GET", url, nil, nil, fields, c.ServiceName()) if err != nil { diff --git a/kafka/consumer.go b/kafka/consumer.go index 51cc318..5f95dd5 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -162,7 +162,7 @@ func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common. } mqttBytes := common.BuildPayloadAsHttp(status, respHeader, respBytes) - _, err = c.PostMqtt(ctx, cpeMac, mqttBytes, fields) + _, err = c.PostMqtt(cpeMac, mqttBytes, fields) if err != nil { return &m, common.NewError(err) } From 719198dbfb6a12d21a3d0e19a98d1e2c810ecd3c Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 15 Jan 2025 14:34:34 -0800 Subject: [PATCH 150/155] clean up used contexts cont. --- http/multipart.go | 10 ++++------ http/multipart_test.go | 20 ++++++++------------ kafka/consumer.go | 5 +---- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/http/multipart.go b/http/multipart.go index 91f55fc..5e39531 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -18,7 +18,6 @@ package http import ( - "context" "errors" "fmt" "net/http" @@ -44,7 +43,6 @@ var ( ) func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() // check if this is a Supplementary service, if so, call a different handler if hd := r.Header.Get(common.HeaderSupplementaryService); len(hd) > 0 { s.MultipartSupplementaryHandler(w, r) @@ -96,7 +94,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. r.Header.Set(common.HeaderSchemaVersion, "none") } - status, respHeader, respBytes, err := BuildWebconfigResponse(s, ctx, r.Header, common.RouteHttp, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(s, r.Header, common.RouteHttp, fields) switch status { case http.StatusNotFound: @@ -120,7 +118,7 @@ func (s *WebconfigServer) MultipartConfigHandler(w http.ResponseWriter, r *http. _, _ = w.Write(respBytes) } -func BuildWebconfigResponse(s *WebconfigServer, ctx context.Context, rHeader http.Header, route string, fields log.Fields) (int, http.Header, []byte, error) { +func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route string, fields log.Fields) (int, http.Header, []byte, error) { fields["for_device"] = true fields["is_primary"] = true @@ -133,7 +131,7 @@ func BuildWebconfigResponse(s *WebconfigServer, ctx context.Context, rHeader htt // factory reset handling ifNoneMatch := rHeader.Get(common.HeaderIfNoneMatch) if ifNoneMatch == "NONE" || ifNoneMatch == "NONE-REBOOT" { - status, respHeader, rbytes, err := BuildFactoryResetResponse(s, ctx, rHeader, fields) + status, respHeader, rbytes, err := BuildFactoryResetResponse(s, rHeader, fields) if err != nil { return status, respHeader, rbytes, common.NewError(err) } @@ -339,7 +337,7 @@ func BuildWebconfigResponse(s *WebconfigServer, ctx context.Context, rHeader htt return http.StatusOK, upstreamRespHeader, finalFilteredBytes, nil } -func BuildFactoryResetResponse(s *WebconfigServer, ctx context.Context, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { +func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { c := s.DatabaseClient uconn := s.GetUpstreamConnector() mac := rHeader.Get(common.HeaderDeviceId) diff --git a/http/multipart_test.go b/http/multipart_test.go index b20bb45..594550d 100644 --- a/http/multipart_test.go +++ b/http/multipart_test.go @@ -19,7 +19,6 @@ package http import ( "bytes" - "context" "encoding/hex" "fmt" "io" @@ -299,7 +298,6 @@ func TestCpeMiddleware(t *testing.T) { func TestVersionFiltering(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) - ctx := context.Background() cpeMac := util.GenerateRandomCpeMac() // ==== group 1 lan ==== @@ -417,7 +415,7 @@ func TestVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") fields := make(log.Fields) - status, respHeader, respBytes, err := BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType := respHeader.Get(common.HeaderContentType) @@ -435,7 +433,7 @@ func TestVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDocName, "root,lan,wan") kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusNotModified) assert.Equal(t, len(respBytes), 0) @@ -619,8 +617,6 @@ func TestUpstreamVersionFiltering(t *testing.T) { func TestMqttUpstreamVersionFiltering(t *testing.T) { server := NewWebconfigServer(sc, true) router := server.GetRouter(true) - ctx := context.Background() - cpeMac := util.GenerateRandomCpeMac() // ==== group 1 lan ==== subdocId := "lan" @@ -680,7 +676,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader := make(http.Header) kHeader.Set(common.HeaderDeviceId, cpeMac) fields := make(log.Fields) - status, respHeader, respBytes, err := BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType := respHeader.Get(common.HeaderContentType) @@ -710,7 +706,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.Assert(t, err != nil) assert.Equal(t, status, http.StatusServiceUnavailable) @@ -718,7 +714,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader = make(http.Header) kHeader.Set(common.HeaderDeviceId, cpeMac) fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.Assert(t, err != nil) assert.Equal(t, status, http.StatusServiceUnavailable) @@ -744,7 +740,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType = respHeader.Get(common.HeaderContentType) @@ -768,7 +764,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "none") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusNotModified) @@ -779,7 +775,7 @@ func TestMqttUpstreamVersionFiltering(t *testing.T) { kHeader.Set(common.HeaderDeviceId, cpeMac) kHeader.Set(common.HeaderSchemaVersion, "33554433-1.3,33554434-1.3") fields = make(log.Fields) - status, respHeader, respBytes, err = BuildWebconfigResponse(server, ctx, kHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err = BuildWebconfigResponse(server, kHeader, common.RouteMqtt, fields) assert.NilError(t, err) assert.Equal(t, status, http.StatusOK) contentType = respHeader.Get(common.HeaderContentType) diff --git a/kafka/consumer.go b/kafka/consumer.go index 5f95dd5..40e8ec1 100644 --- a/kafka/consumer.go +++ b/kafka/consumer.go @@ -18,7 +18,6 @@ package kafka import ( - "context" "encoding/base64" "encoding/json" "fmt" @@ -106,8 +105,6 @@ func (c *Consumer) handleNotification(bbytes []byte, fields log.Fields) (*common // NOTE we choose to return an EventMessage object just to pass along the metricsAgent func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common.EventMessage, error) { - ctx := context.TODO() - rHeader, _ := util.ParseHttp(inbytes) params := rHeader.Get(common.HeaderDocName) cpeMac := rHeader.Get(common.HeaderDeviceId) @@ -151,7 +148,7 @@ func (c *Consumer) handleGetMessage(inbytes []byte, fields log.Fields) (*common. rHeader.Set(common.HeaderSchemaVersion, "none") } - status, respHeader, respBytes, err := wchttp.BuildWebconfigResponse(c.WebconfigServer, ctx, rHeader, common.RouteMqtt, fields) + status, respHeader, respBytes, err := wchttp.BuildWebconfigResponse(c.WebconfigServer, rHeader, common.RouteMqtt, fields) if err != nil && respBytes == nil { respBytes = []byte(err.Error()) } From 3bf15ca4071ffee75672cd950db86a20f136afaf Mon Sep 17 00:00:00 2001 From: James Chao Date: Thu, 16 Jan 2025 11:39:46 -0800 Subject: [PATCH 151/155] small changes in tracing context functions --- tracing/ctx.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tracing/ctx.go b/tracing/ctx.go index 4a490fb..8115737 100644 --- a/tracing/ctx.go +++ b/tracing/ctx.go @@ -19,18 +19,12 @@ package tracing import "context" -type contextKey string - -func (c contextKey) String() string { - return string(c) -} - // SetContext - setting context for logging func SetContext(ctx context.Context, ctxName string, ctxValue interface{}) context.Context { - return context.WithValue(ctx, contextKey(ctxName), ctxValue) + return context.WithValue(ctx, ctxName, ctxValue) } // GetContext - getting context value from context func GetContext(ctx context.Context, key string) interface{} { - return ctx.Value(contextKey(key)) + return ctx.Value(key) } From 85b9308c52376bf60359d17d3e73994d72c0fe80 Mon Sep 17 00:00:00 2001 From: James Chao Date: Mon, 20 Jan 2025 11:28:57 -0800 Subject: [PATCH 152/155] small changes based on review --- http/webconfig_server.go | 7 +++---- tracing/ctx.go | 6 ++++-- tracing/otel.go | 14 +++++--------- tracing/span.go | 26 +++++++++++++------------- tracing/tracer.go | 37 ++++++++++++++++++++++++++++++++++--- 5 files changed, 59 insertions(+), 31 deletions(-) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 2941f31..1f1b4ac 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -748,7 +748,7 @@ func (s *WebconfigServer) logRequestStarts(w http.ResponseWriter, r *http.Reques } // traceparent handling for E2E tracing - xpcTrace := tracing.NewXpcTrace(r, s.XpcTracer) + xpcTrace := tracing.NewXpcTrace(s.XpcTracer, r) traceId := xpcTrace.TraceID if len(traceId) == 0 { traceId = xmTraceId @@ -899,8 +899,7 @@ func (s *WebconfigServer) logRequestEnds(xw *XResponseWriter, r *http.Request) { fields["duration"] = duration fields["logger"] = "request" - tracing.SetSpanStatusCode(s.XpcTracer, fields) - tracing.SetSpanMoracideTags(s.XpcTracer, fields) + s.XpcTracer.SetSpan(fields, s.XpcTracer.MoracideTagPrefix()) var userAgent string if itf, ok := fields["user_agent"]; ok { @@ -1126,7 +1125,7 @@ func (s *WebconfigServer) StopXpcTracer() { func (s *WebconfigServer) SpanMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if s.XpcTracer.OtelEnabled { - ctx, otelSpan := tracing.NewOtelSpan(r, s.XpcTracer) + ctx, otelSpan := tracing.NewOtelSpan(s.XpcTracer, r) r = r.WithContext(ctx) defer tracing.EndOtelSpan(s.XpcTracer, otelSpan) } diff --git a/tracing/ctx.go b/tracing/ctx.go index 8115737..b106712 100644 --- a/tracing/ctx.go +++ b/tracing/ctx.go @@ -19,12 +19,14 @@ package tracing import "context" +type contextKey string + // SetContext - setting context for logging func SetContext(ctx context.Context, ctxName string, ctxValue interface{}) context.Context { - return context.WithValue(ctx, ctxName, ctxValue) + return context.WithValue(ctx, contextKey(ctxName), ctxValue) } // GetContext - getting context value from context func GetContext(ctx context.Context, key string) interface{} { - return ctx.Value(key) + return ctx.Value(contextKey(key)) } diff --git a/tracing/otel.go b/tracing/otel.go index c9dd75a..6617224 100644 --- a/tracing/otel.go +++ b/tracing/otel.go @@ -59,7 +59,7 @@ const defaultOtelTracerProvider = "noop" // initOtel - initialize OpenTelemetry constructs // tracing instrumentation code. -func otelInit(conf *configuration.Config, xpcTracer *XpcTracer) { +func otelInit(xpcTracer *XpcTracer, conf *configuration.Config) { xpcTracer.OtelEnabled = conf.GetBoolean("webconfig.tracing.otel.enabled") if !xpcTracer.OtelEnabled { return @@ -149,7 +149,7 @@ func otelHttpTraceProvider(xpcTracer *XpcTracer) (oteltrace.TracerProvider, erro ), nil } -func NewOtelSpan(r *http.Request, xpcTracer *XpcTracer) (context.Context, oteltrace.Span) { +func NewOtelSpan(xpcTracer *XpcTracer, r *http.Request) (context.Context, oteltrace.Span) { ctx := r.Context() var otelSpan oteltrace.Span if !xpcTracer.OtelEnabled { @@ -230,17 +230,13 @@ func EndOtelSpan(xpcTracer *XpcTracer, span oteltrace.Span) { span.End() } -func otelExtractParamsFromSpan(ctx context.Context, xpcTracer *XpcTracer, xpcTrace *XpcTrace) { - if !xpcTracer.OtelEnabled { - return - } +func otelExtractParamsFromSpan(ctx context.Context, xpcTrace *XpcTrace) { if tmp := GetContext(ctx, "otel_span"); tmp != nil { if otelSpan, ok := tmp.(oteltrace.Span); ok { - xpcTrace.otelSpan = otelSpan - if xpcTrace.otelSpan == nil { + if otelSpan == nil { return } - + xpcTrace.otelSpan = otelSpan spanCtx := otelSpan.SpanContext() xpcTrace.TraceID = spanCtx.TraceID().String() } diff --git a/tracing/span.go b/tracing/span.go index ba7edc1..c04fa65 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -68,20 +68,20 @@ type XpcTrace struct { } // NewXpcTrace extracts traceparent, tracestate, moracideTags from otel spans or reqs -func NewXpcTrace(r *http.Request, xpcTracer *XpcTracer) *XpcTrace { +func NewXpcTrace(xpcTracer *XpcTracer, r *http.Request) *XpcTrace { var xpcTrace XpcTrace xpcTrace.ReqMoracideTags = make(map[string]string) - extractParamsFromReq(r, xpcTracer, &xpcTrace) + extractParamsFromReq(r, &xpcTrace, xpcTracer.MoracideTagPrefix()) if xpcTracer.OtelEnabled { - otelExtractParamsFromSpan(r.Context(), xpcTracer, &xpcTrace) + otelExtractParamsFromSpan(r.Context(), &xpcTrace) } return &xpcTrace } -func SetSpanStatusCode(xpcTracer *XpcTracer, fields log.Fields) { +func SetSpanStatusCode(fields log.Fields) { var xpcTrace *XpcTrace if tmp, ok := fields["xpc_trace"]; ok { xpcTrace = tmp.(*XpcTrace) @@ -91,7 +91,7 @@ func SetSpanStatusCode(xpcTracer *XpcTracer, fields log.Fields) { log.Error("instrumentation error, no trace info") return } - if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { + if xpcTrace.otelSpan != nil { if tmp, ok := fields["status"]; ok { statusCode := tmp.(int) otelSetStatusCode(xpcTrace.otelSpan, statusCode) @@ -99,7 +99,7 @@ func SetSpanStatusCode(xpcTracer *XpcTracer, fields log.Fields) { } } -func SetSpanMoracideTags(xpcTracer *XpcTracer, fields log.Fields) { +func SetSpanMoracideTags(fields log.Fields, moracideTagPrefix string) { var xpcTrace *XpcTrace if tmp, ok := fields["xpc_trace"]; ok { xpcTrace = tmp.(*XpcTrace) @@ -111,8 +111,8 @@ func SetSpanMoracideTags(xpcTracer *XpcTracer, fields log.Fields) { } moracideTags := make(map[string]string) - reqMoracideTagPrefix := strings.ToLower("req_" + xpcTracer.MoracideTagPrefix()) - respMoracideTagPrefix := strings.ToLower("resp_" + xpcTracer.MoracideTagPrefix()) + reqMoracideTagPrefix := strings.ToLower("req_" + moracideTagPrefix) + respMoracideTagPrefix := strings.ToLower("resp_" + moracideTagPrefix) for key, val := range fields { if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { @@ -130,17 +130,17 @@ func SetSpanMoracideTags(xpcTracer *XpcTracer, fields log.Fields) { if len(moracideTags) == 0 { // No moracide tags in request or any response // So set at least one span tag, x-cl-expt: false - moracideTags[xpcTracer.MoracideTagPrefix()] = "false" + moracideTags[moracideTagPrefix] = "false" } - for key, val := range moracideTags { - if xpcTracer.OtelEnabled && xpcTrace.otelSpan != nil { + if xpcTrace.otelSpan != nil { + for key, val := range moracideTags { xpcTrace.otelSpan.SetAttributes(attribute.String(key, val)) log.Debugf("added otel span moracide tag key = %s, value = %s", key, val) } } } -func extractParamsFromReq(r *http.Request, xpcTracer *XpcTracer, xpcTrace *XpcTrace) { +func extractParamsFromReq(r *http.Request, xpcTrace *XpcTrace, moracideTagPrefix string) { xpcTrace.ReqTraceparent = r.Header.Get(common.HeaderTraceparent) xpcTrace.ReqTracestate = r.Header.Get(common.HeaderTracestate) xpcTrace.OutTraceparent = xpcTrace.ReqTraceparent @@ -151,7 +151,7 @@ func extractParamsFromReq(r *http.Request, xpcTracer *XpcTracer, xpcTrace *XpcTr // In future, -H 'X-Cl-Experiment-1', -H 'X-Cl-Experiment-oswebconfig'... OR 'X-Cl-Experiment-xapproxy_25.1.1.1' are all possible // So walk through all headers and collect any header that starts with this prefix - moracideTagPrefix := strings.ToLower(xpcTracer.MoracideTagPrefix()) + moracideTagPrefix = strings.ToLower(moracideTagPrefix) for headerKey, headerVals := range r.Header { if strings.HasPrefix(strings.ToLower(headerKey), moracideTagPrefix) { if len(headerVals) > 1 { diff --git a/tracing/tracer.go b/tracing/tracer.go index fffe1f2..a65455a 100644 --- a/tracing/tracer.go +++ b/tracing/tracer.go @@ -34,6 +34,8 @@ const ( DefaultMoracideTagPrefix = "X-Cl-Experiment" ) +type SpanSetterFunc func(log.Fields, string) + // XpcTracer is a wrapper around tracer setup type XpcTracer struct { OtelEnabled bool @@ -54,13 +56,22 @@ type XpcTracer struct { otelTracerProvider oteltrace.TracerProvider otelPropagator otelpropagation.TextMapPropagator otelTracer oteltrace.Tracer + + SetSpan SpanSetterFunc } func NewXpcTracer(conf *configuration.Config) *XpcTracer { xpcTracer := new(XpcTracer) - initAppData(conf, xpcTracer) - otelInit(conf, xpcTracer) + initAppData(xpcTracer, conf) + otelInit(xpcTracer, conf) xpcTracer.moracideTagPrefix = conf.GetString("webconfig.tracing.moracide_tag_prefix", DefaultMoracideTagPrefix) + + if xpcTracer.OtelEnabled { + xpcTracer.SetSpan = OtelSetSpan + } else { + xpcTracer.SetSpan = NoopSetSpan + } + return xpcTracer } @@ -84,7 +95,19 @@ func (t *XpcTracer) AppName() string { return t.appName } -func initAppData(conf *configuration.Config, xpcTracer *XpcTracer) { +func (t *XpcTracer) AppVersion() string { + return t.appVersion +} + +func (t *XpcTracer) AppEnv() string { + return t.appEnv +} + +func (t *XpcTracer) Region() string { + return t.region +} + +func initAppData(xpcTracer *XpcTracer, conf *configuration.Config) { codeGitCommit := strings.Split(conf.GetString("webconfig.code_git_commit"), "-") xpcTracer.appName = codeGitCommit[0] if len(codeGitCommit) > 1 { @@ -108,3 +131,11 @@ func initAppData(conf *configuration.Config, xpcTracer *XpcTracer) { } log.Debugf("site_color = %s, env = %s, region = %s", siteColor, xpcTracer.appEnv, xpcTracer.region) } + +func OtelSetSpan(fields log.Fields, tag string) { + SetSpanStatusCode(fields) + SetSpanMoracideTags(fields, tag) +} + +func NoopSetSpan(fields log.Fields, tag string) { +} From d99ae6d8443c4b2259c7896ce0e0637a354c456b Mon Sep 17 00:00:00 2001 From: "RV Subramanyan (Personal)" Date: Thu, 23 Jan 2025 15:57:55 -0800 Subject: [PATCH 153/155] Refactoring missed the tag settig part --- http/response.go | 2 +- tracing/span.go | 28 ++++++++++++---------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/http/response.go b/http/response.go index 5b748c3..c2c8f48 100644 --- a/http/response.go +++ b/http/response.go @@ -182,7 +182,7 @@ func addMoracideTagsAsResponseHeaders(w http.ResponseWriter) { moracideTags[key] = val } for key, val := range respMoracideTags { - if existingVal, ok := moracideTags[key]; !ok || (ok && existingVal != "true") { + if val == "true" { xw.LogDebug(nil, "request", fmt.Sprintf("Adding moracide tag from resp %s = %s to response", key, val)) moracideTags[key] = val } diff --git a/tracing/span.go b/tracing/span.go index c04fa65..e75e91e 100644 --- a/tracing/span.go +++ b/tracing/span.go @@ -111,27 +111,23 @@ func SetSpanMoracideTags(fields log.Fields, moracideTagPrefix string) { } moracideTags := make(map[string]string) - reqMoracideTagPrefix := strings.ToLower("req_" + moracideTagPrefix) - respMoracideTagPrefix := strings.ToLower("resp_" + moracideTagPrefix) - for key, val := range fields { - if strings.HasPrefix(strings.ToLower(key), reqMoracideTagPrefix) { - log.Debugf("Adding moracide tag from req %s = %s to response", key, val) - moracideTags[key[4:]] = val.(string) + if itf, ok := fields["req_moracide_tags"]; ok { + reqMoracideTags := itf.(map[string]string) + for key, val := range reqMoracideTags { + moracideTags[key] = val } - if strings.HasPrefix(strings.ToLower(key), respMoracideTagPrefix) { - log.Debugf("Adding moracide tag from resp %s = %s to response", key, val) - realKey := key[5:] - if existingVal, ok := moracideTags[realKey]; !ok || (ok && existingVal != "true") { - moracideTags[realKey] = val.(string) + } + + if itf, ok := fields["resp_moracide_tags"]; ok { + respMoracideTags := itf.(map[string]string) + for key, val := range respMoracideTags { + if val == "true" { + moracideTags[key] = val } } } - if len(moracideTags) == 0 { - // No moracide tags in request or any response - // So set at least one span tag, x-cl-expt: false - moracideTags[moracideTagPrefix] = "false" - } + if xpcTrace.otelSpan != nil { for key, val := range moracideTags { xpcTrace.otelSpan.SetAttributes(attribute.String(key, val)) From 514667e68fb34001485eadcb8972d617f7761d67 Mon Sep 17 00:00:00 2001 From: James Chao Date: Tue, 28 Jan 2025 22:57:48 -0800 Subject: [PATCH 154/155] filter webconfig output by bitmap --- common/const_var.go | 2 + common/document.go | 18 + common/firmware_bitmap.go | 147 +++++++ common/firmware_bitmap_test.go | 589 ++++++++++++++++++++++++++ common/root_document.go | 19 + db/service.go | 9 +- http/multipart.go | 28 +- http/rootdocument_handler_test.go | 4 +- http/supported_groups_handler.go | 6 +- http/supported_groups_handler_test.go | 10 +- http/webconfig_server.go | 12 + http/webconfig_server_test.go | 7 + util/firmware_bitmap.go | 19 +- util/firmware_bitmap_test.go | 17 + 14 files changed, 867 insertions(+), 20 deletions(-) create mode 100644 common/firmware_bitmap.go create mode 100644 common/firmware_bitmap_test.go diff --git a/common/const_var.go b/common/const_var.go index b8b8ec3..fee526c 100644 --- a/common/const_var.go +++ b/common/const_var.go @@ -112,6 +112,8 @@ const ( HeaderContentLength = "Content-Length" HeaderRefSubdocumentVersion = "X-Refsubdocument-Version" HeaderUpstreamResponse = "X-Upstream-Response" + HeaderMoneytrace = "X-Moneytrace" + HeaderWebconfigTransactionId = "X-Webconfig-Transaction-Id" ) const ( diff --git a/common/document.go b/common/document.go index 7254dc0..e216c9a 100644 --- a/common/document.go +++ b/common/document.go @@ -219,3 +219,21 @@ func (d *Document) HttpBytes(fields log.Fields) ([]byte, error) { return BuildPayloadAsHttp(http.StatusOK, header, bbytes), nil } + +func (d *Document) FilterByBitmap() *Document { + rootdoc := d.GetRootDocument() + if rootdoc == nil { + return d + } + + newdoc := NewDocument(rootdoc) + supportedMap := GetSupportedMap(rootdoc.Bitmap) + + for subdocId, subDocument := range d.docmap { + if supportedMap[subdocId] { + newdoc.SetSubDocument(subdocId, &subDocument) + } + } + + return newdoc +} diff --git a/common/firmware_bitmap.go b/common/firmware_bitmap.go new file mode 100644 index 0000000..4ddef83 --- /dev/null +++ b/common/firmware_bitmap.go @@ -0,0 +1,147 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "fmt" + "strconv" + "strings" +) + +// bitmap is used to name int variables +// bitarray is used to name string variables + +func SetBitmapByGroup(cpeBitmap *int, groupId int, groupBitmap int) error { + tuples, ok := SupportedDocsBitMaskMap[groupId] + if !ok { + return nil + } + + for _, tuple := range tuples { + mask := 1 << (tuple.GroupBit - 1) + masked := groupBitmap & mask + if masked > 0 { + (*cpeBitmap) |= 1 << (tuple.CpeBit - 1) + } + } + return nil +} + +func ParseFirmwareGroupBitarray(s string) (int, int, error) { + i, err := strconv.Atoi(s) + if err != nil { + return 0, 0, NewError(err) + } + groupId := i >> 24 + groupBitMask := 1<<24 - 1 + groupBitmap := i & groupBitMask + return groupId, groupBitmap, nil +} + +func GetCpeBitmap(rdkSupportedDocsHeaderStr string) (int, error) { + cpeBitmap := 0 + + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + for _, sid := range sids { + groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) + if err != nil { + return 0, NewError(err) + } + + err = SetBitmapByGroup(&cpeBitmap, groupId, groupBitmap) + if err != nil { + return 0, NewError(err) + } + } + + return cpeBitmap, nil +} + +func PrettyBitarray(i int) string { + x := fmt.Sprintf("%032b", i) + return fmt.Sprintf("%s %s %s %s %s %s %s %s", + x[0:4], + x[4:8], + x[8:12], + x[12:16], + x[16:20], + x[20:24], + x[24:28], + x[28:32]) +} + +func PrettyGroupBitarray(i int) string { + x := fmt.Sprintf("%032b", i) + return fmt.Sprintf("%s %s %s %s %s %s %s", + x[0:8], + x[8:12], + x[12:16], + x[16:20], + x[20:24], + x[24:28], + x[28:32]) +} + +func IsSubdocSupported(cpeBitmap int, subdocId string) bool { + index, ok := SubdocBitIndexMap[subdocId] + if !ok { + return false + } + + shift := index - 1 + bitmask := 1 << shift + masked := cpeBitmap & bitmask + return masked != 0 +} + +func GetSupportedMap(cpeBitmap int) map[string]bool { + supportedMap := map[string]bool{} + + for k, index := range SubdocBitIndexMap { + shift := index - 1 + bitmask := 1 << shift + supportedMap[k] = false + if masked := cpeBitmap & bitmask; masked > 0 { + supportedMap[k] = true + } + } + return supportedMap +} + +func BitarrayToBitmap(src string) (int, error) { + s := strings.ReplaceAll(src, " ", "") + v, err := strconv.ParseInt(s, 2, 64) + if err != nil { + return 0, NewError(err) + } + i := int(v) + return i, nil +} + +func GetBitmapFromSupportedMap(srcMap map[string]bool) int { + var bitmap int + + for k, index := range SubdocBitIndexMap { + shift := index - 1 + bitmask := 1 << shift + if srcMap[k] { + bitmap += bitmask + } + } + return bitmap +} diff --git a/common/firmware_bitmap_test.go b/common/firmware_bitmap_test.go new file mode 100644 index 0000000..85c4e0b --- /dev/null +++ b/common/firmware_bitmap_test.go @@ -0,0 +1,589 @@ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 +*/ +package common + +import ( + "fmt" + "math/rand" + "strings" + "testing" + + "gotest.tools/assert" +) + +// bitmap is used to name int variables +// bitarray is used to name string variables + +func TestPrettyBitarray(t *testing.T) { + i := 8 + bs := PrettyBitarray(i) + expected := "0000 0000 0000 0000 0000 0000 0000 1000" + assert.Equal(t, bs, expected) + + j := 16777231 + bs = PrettyGroupBitarray(j) + expected = "00000001 0000 0000 0000 0000 0000 1111" + assert.Equal(t, bs, expected) +} + +func TestParseRdkGroupBitarray(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + expectedMap := map[int]int{ + 1: 15, + 2: 3, + 3: 1, + 4: 1, + 5: 1, + 6: 1, + 7: 1, + 8: 1, + } + + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + + for _, sid := range sids { + groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) + assert.NilError(t, err) + // assert.Equal(t, groupId, 1) + + expectedBitmap, ok := expectedMap[groupId] + assert.Assert(t, ok) + assert.Equal(t, groupBitmap, expectedBitmap) + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + + // bs := PrettyGroupBitarray(cpeBitmap) + + // build expected + expectedEnabled := map[string]bool{ + "portforwarding": true, + "lan": true, + "wan": true, + "macbinding": true, + "hotspot": false, + "bridge": false, + "privatessid": true, + "homessid": true, + "radio": false, + "moca": true, + "xdns": true, + "advsecurity": true, + "mesh": true, + "aker": true, + "telemetry": true, + "statusreport": false, + "trafficreport": false, + "interfacereport": false, + "radioreport": false, + } + + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } +} + +func TestParseCustomizedGroupBitarray(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + + // build expected + expectedEnabled := map[string]bool{ + "portforwarding": true, + "lan": true, + "wan": true, + "macbinding": true, + "hotspot": false, + "bridge": false, + "privatessid": true, + "homessid": true, + "radio": false, + "moca": true, + "xdns": true, + "advsecurity": true, + "mesh": true, + "aker": true, + "telemetry": true, + "statusreport": false, + "trafficreport": false, + "interfacereport": false, + "radioreport": false, + "telcovoip": false, + "telcovoice": false, + "wanmanager": false, + "voiceservice": false, + } + + newGroup1Bitarray := "00000001 0000 0000 0000 0000 0011 0011" + group1Bitmap, err := BitarrayToBitmap(newGroup1Bitarray) + assert.NilError(t, err) + sids[0] = fmt.Sprintf("%v", group1Bitmap) + expectedEnabled["wan"] = false + expectedEnabled["macbinding"] = false + expectedEnabled["hotspot"] = true + expectedEnabled["bridge"] = true + + newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" + group2Bitmap, err := BitarrayToBitmap(newGroup2Bitarray) + assert.NilError(t, err) + sids[1] = fmt.Sprintf("%v", group2Bitmap) + expectedEnabled["privatessid"] = false + expectedEnabled["radio"] = true + + rdkSupportedDocsHeaderStr = strings.Join(sids, ",") + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for subdocId, enabled := range expectedEnabled { + parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) + assert.Equal(t, parsedEnabled, enabled) + } + + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocIds := []string{} + for k, v := range expectedEnabled { + if v { + supportedSubdocIds = append(supportedSubdocIds, k) + } + } + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseTelcovoipAndWanmanager(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549377,201326593" + sids := strings.Split(rdkSupportedDocsHeaderStr, ",") + + // build expected + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "privatessid", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "telcovoip", + "wanmanager", + } + + rdkSupportedDocsHeaderStr = strings.Join(sids, ",") + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + for _, subdocId := range supportedSubdocIds { + assert.Assert(t, IsSubdocSupported(cpeBitmap, subdocId)) + } + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestBitmapParsing(t *testing.T) { + // clearn the wan bit + newBitmap := 16777231 & ^(1 << 2) + rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,33554435,50331649,67108865,83886081,100663297,117440513,134217729", newBitmap) + + // build expected + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "macbinding", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseVoiceService(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809" + + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "voiceservice", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestManualBitmap(t *testing.T) { + for i := 0; i < 10; i++ { + bitmap := rand.Intn(40000) + parsedSupportedMap := GetSupportedMap(bitmap) + revBitmap := GetBitmapFromSupportedMap(parsedSupportedMap) + assert.Equal(t, bitmap, revBitmap) + } +} + +func TestParseSupportedDocsWithNewGroups(t *testing.T) { + cellularBitGroupId := 14 + xBitValue := (cellularBitGroupId << 24) + 1 + ss := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809,234881025" + rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", ss, xBitValue) + + // build expected + supportedSubdocIds := []string{ + "portforwarding", + "lan", + "wan", + "macbinding", + "hotspot", + "privatessid", + "homessid", + "moca", + "xdns", + "advsecurity", + "mesh", + "aker", + "telemetry", + "voiceservice", + "cellularconfig", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554435,67108865,100663297,117440513,134217729,201326595,234881025" + + // build expected + supportedSubdocIds := []string{ + "aker", + "cellularconfig", + "homessid", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549378,201326595" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "lan", + "macbinding", + "mesh", + "portforwarding", + "privatessid", + "telcovoice", + "telemetry", + "wan", + "wanfailover", + "wanmanager", + "xdns", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,251658241,268435457" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderRfc(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "defaultrfc", + "rfc", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderHcm(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} + +func TestParseSupportedDocsHeaderWebui(t *testing.T) { + rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" + + // build expected + supportedSubdocIds := []string{ + "advsecurity", + "aker", + "gwfailover", + "homessid", + "hotspot", + "lan", + "macbinding", + "mesh", + "moca", + "portforwarding", + "privatessid", + "telemetry", + "voiceservice", + "wan", + "wanfailover", + "xdns", + "prioritizedmacs", + "connectedbuilding", + "lldqoscontrol", + "clienttosteeringprofile", + "meshsteeringprofiles", + "wifistatsconfig", + "mwoconfigs", + "interference", + "xmspeedboost", + "webui", + } + + cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) + assert.NilError(t, err) + parsedSupportedMap := GetSupportedMap(cpeBitmap) + supportedSubdocMap := BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) + assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) +} diff --git a/common/root_document.go b/common/root_document.go index c662f09..54cf801 100644 --- a/common/root_document.go +++ b/common/root_document.go @@ -162,6 +162,25 @@ func (d *RootDocument) Update(r *RootDocument) { } } +func (d *RootDocument) UpdateMetadata(r *RootDocument) { + // Version and QueryParams are cloud data, so not changed + if r.Bitmap > 0 { + d.Bitmap = r.Bitmap + } + if len(r.FirmwareVersion) > 0 { + d.FirmwareVersion = r.FirmwareVersion + } + if len(r.ModelName) > 0 { + d.ModelName = r.ModelName + } + if len(r.PartnerId) > 0 { + d.PartnerId = r.PartnerId + } + if len(r.SchemaVersion) > 0 { + d.SchemaVersion = r.SchemaVersion + } +} + func (d *RootDocument) String() string { m := d.ColumnMap() bbytes, err := json.MarshalIndent(m, "", " ") diff --git a/db/service.go b/db/service.go index 169f2a0..66be7e9 100644 --- a/db/service.go +++ b/db/service.go @@ -64,7 +64,7 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel } if len(supportedDocs) > 0 { - bitmap, err = util.GetCpeBitmap(supportedDocs) + bitmap, err = common.GetCpeBitmap(supportedDocs) if err != nil { log.WithFields(fields).Warn(common.NewError(err)) } @@ -146,8 +146,11 @@ func BuildGetDocument(c DatabaseClient, inHeader http.Header, route string, fiel if err := c.SetRootDocument(mac, clonedRootDoc); err != nil { return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) } + // the returned err is dbNotFound - return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + // WARNING, this should be removed + // return nil, cloudRootDocument, deviceRootDocument, deviceVersionMap, false, nil, common.NewError(err) + cloudRootDocument = clonedRootDoc.Clone() } // ==== compare if the deviceRootDocument and cloudRootDocument are different ==== @@ -620,7 +623,7 @@ func PreprocessRootDocument(c DatabaseClient, rHeader http.Header, mac, partnerI var err error supportedDocs := rHeader.Get(common.HeaderSupportedDocs) if len(supportedDocs) > 0 { - bitmap, err = util.GetCpeBitmap(supportedDocs) + bitmap, err = common.GetCpeBitmap(supportedDocs) if err != nil { log.WithFields(fields).Warn(common.NewError(err)) } diff --git a/http/multipart.go b/http/multipart.go index 5e39531..4f27811 100644 --- a/http/multipart.go +++ b/http/multipart.go @@ -123,7 +123,6 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin fields["is_primary"] = true c := s.DatabaseClient - uconn := s.GetUpstreamConnector() mac := rHeader.Get(common.HeaderDeviceId) respHeader := make(http.Header) userAgent := rHeader.Get("User-Agent") @@ -148,7 +147,12 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin return http.StatusConflict, respHeader, nil, common.NewError(err) } - if uconn == nil { + if !s.UpstreamEnabled() { + if postUpstream { + rdoc := oldRootDocument.Clone() + rdoc.UpdateMetadata(newRootDocument) + document.SetRootDocument(rdoc) + } if err != nil { if !s.IsDbNotFound(err) { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -170,6 +174,11 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } + + if s.FilterOutputByBitmapEnabled() { + document = document.FilterByBitmap() + } + respBytes, err := document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -208,13 +217,17 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin var respBytes []byte respStatus := http.StatusNotModified if document.Length() > 0 { - if !postUpstream { document, err = db.LoadRefSubDocuments(c, document, fields) if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } } + // 3333 + if s.FilterOutputByBitmapEnabled() { + document = document.FilterByBitmap() + } + // 44444 respBytes, err = document.Bytes() if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) @@ -302,7 +315,7 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin upstreamRespEtag := upstreamRespHeader.Get(common.HeaderEtag) // filter by versionMap and filter by blockedIds - finalRootDocument := common.NewRootDocument(0, "", "", "", "", upstreamRespEtag, "") + finalRootDocument := common.NewRootDocument(newRootDocument.Bitmap, "", "", "", "", upstreamRespEtag, "") finalDocument := common.NewDocument(finalRootDocument) finalDocument.SetSubDocuments(finalMparts) @@ -320,6 +333,10 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin finalFilteredDocument.DeleteSubDocument(subdocId) } + if s.FilterOutputByBitmapEnabled() { + finalFilteredDocument = finalFilteredDocument.FilterByBitmap() + } + // 304 if finalFilteredDocument.Length() == 0 { return http.StatusNotModified, upstreamRespHeader, nil, nil @@ -339,7 +356,6 @@ func BuildWebconfigResponse(s *WebconfigServer, rHeader http.Header, route strin func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields log.Fields) (int, http.Header, []byte, error) { c := s.DatabaseClient - uconn := s.GetUpstreamConnector() mac := rHeader.Get(common.HeaderDeviceId) respHeader := make(http.Header) @@ -374,7 +390,7 @@ func BuildFactoryResetResponse(s *WebconfigServer, rHeader http.Header, fields l return http.StatusInternalServerError, respHeader, nil, common.NewError(err) } - if uconn == nil { + if !s.UpstreamEnabled() { err := c.DeleteDocument(mac) if err != nil { return http.StatusInternalServerError, respHeader, nil, common.NewError(err) diff --git a/http/rootdocument_handler_test.go b/http/rootdocument_handler_test.go index 8c4e0f6..6bc640c 100644 --- a/http/rootdocument_handler_test.go +++ b/http/rootdocument_handler_test.go @@ -79,7 +79,7 @@ func TestRootDocumentHandler(t *testing.T) { rootdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - expectedBitmap1, err := util.GetCpeBitmap(supportedDocs1) + expectedBitmap1, err := common.GetCpeBitmap(supportedDocs1) assert.NilError(t, err) expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") assert.DeepEqual(t, rootdoc, expectedRootdoc) @@ -278,7 +278,7 @@ func TestRootDocumentHandlerCorruptedHeaders(t *testing.T) { rootdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - expectedBitmap1, err := util.GetCpeBitmap(supportedDocs1) + expectedBitmap1, err := common.GetCpeBitmap(supportedDocs1) assert.NilError(t, err) expectedRootdoc := common.NewRootDocument(expectedBitmap1, firmwareVersion1, modelName1, partner1, schemaVersion1, "", "") assert.DeepEqual(t, rootdoc, expectedRootdoc) diff --git a/http/supported_groups_handler.go b/http/supported_groups_handler.go index 6eac50b..90a1a27 100644 --- a/http/supported_groups_handler.go +++ b/http/supported_groups_handler.go @@ -14,7 +14,7 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ package http import ( @@ -23,9 +23,9 @@ import ( "net/http" "strings" + "github.com/gorilla/mux" "github.com/rdkcentral/webconfig/common" "github.com/rdkcentral/webconfig/util" - "github.com/gorilla/mux" ) // The supported doc header in GET /config is parsed and stored as a bitmap @@ -54,7 +54,7 @@ func (s *WebconfigServer) GetSupportedGroupsHandler(w http.ResponseWriter, r *ht outdata := common.SupportedGroupsData{ Bitmap: rdoc.Bitmap, - Groups: util.GetSupportedMap(rdoc.Bitmap), + Groups: common.GetSupportedMap(rdoc.Bitmap), } WriteOkResponse(w, outdata) diff --git a/http/supported_groups_handler_test.go b/http/supported_groups_handler_test.go index b0c7f52..9256696 100644 --- a/http/supported_groups_handler_test.go +++ b/http/supported_groups_handler_test.go @@ -62,7 +62,7 @@ func TestSupportedGroupsHandler(t *testing.T) { rdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - bitmap, err := util.GetCpeBitmap(rdkSupportedDocsHeaderStr) + bitmap, err := common.GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) assert.Equal(t, bitmap, rdoc.Bitmap) @@ -156,7 +156,7 @@ func TestSupportedGroupsHandler(t *testing.T) { sids := strings.Split(rdkSupportedDocsHeaderStr, ",") newGroup1Bitarray := "00000001 0000 0000 0000 0000 0011 0011" - group1Bitmap, err := util.BitarrayToBitmap(newGroup1Bitarray) + group1Bitmap, err := common.BitarrayToBitmap(newGroup1Bitarray) assert.NilError(t, err) sids[0] = fmt.Sprintf("%v", group1Bitmap) supportedSubdocMap["wan"] = false @@ -165,7 +165,7 @@ func TestSupportedGroupsHandler(t *testing.T) { supportedSubdocMap["bridge"] = true newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" - group2Bitmap, err := util.BitarrayToBitmap(newGroup2Bitarray) + group2Bitmap, err := common.BitarrayToBitmap(newGroup2Bitarray) assert.NilError(t, err) sids[1] = fmt.Sprintf("%v", group2Bitmap) supportedSubdocMap["privatessid"] = false @@ -231,7 +231,7 @@ func TestSupportedGroupsHandlerTelcovoice(t *testing.T) { rdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - bitmap, err := util.GetCpeBitmap(rdkSupportedDocsHeaderStr) + bitmap, err := common.GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) assert.Equal(t, bitmap, rdoc.Bitmap) @@ -297,7 +297,7 @@ func TestSupportedGroupsHandlerWithPrioritizedmacsAndConnectedbuilding(t *testin rdoc, err := server.GetRootDocument(cpeMac) assert.NilError(t, err) - bitmap, err := util.GetCpeBitmap(rdkSupportedDocsHeaderStr) + bitmap, err := common.GetCpeBitmap(rdkSupportedDocsHeaderStr) assert.NilError(t, err) assert.Equal(t, bitmap, rdoc.Bitmap) diff --git a/http/webconfig_server.go b/http/webconfig_server.go index 1f1b4ac..6d51df6 100644 --- a/http/webconfig_server.go +++ b/http/webconfig_server.go @@ -118,6 +118,7 @@ type WebconfigServer struct { queryParamsValidationEnabled bool minTrust int validSubdocIdMap map[string]int + filterOutputByBitmapEnabled bool } func NewTlsConfig(conf *configuration.Config) (*tls.Config, error) { @@ -289,6 +290,8 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer validSubdocIdMap[x] = 1 } + filterOutputByBitmapEnabled := conf.GetBoolean("webconfig.filter_output_by_bitmap_enabled") + ws := &WebconfigServer{ Server: &http.Server{ Addr: fmt.Sprintf("%v:%v", listenHost, port), @@ -325,6 +328,7 @@ func NewWebconfigServer(sc *common.ServerConfig, testOnly bool) *WebconfigServer minTrust: minTrust, validSubdocIdMap: validSubdocIdMap, XpcTracer: xpcTracer, + filterOutputByBitmapEnabled: filterOutputByBitmapEnabled, } return ws @@ -686,6 +690,14 @@ func (s *WebconfigServer) SetValidSubdocIdMap(x map[string]int) { s.validSubdocIdMap = x } +func (s *WebconfigServer) FilterOutputByBitmapEnabled() bool { + return s.filterOutputByBitmapEnabled +} + +func (s *WebconfigServer) SetFilterOutputByBitmapEnabled(enabled bool) { + s.filterOutputByBitmapEnabled = enabled +} + func (s *WebconfigServer) ValidatePartner(parsedPartner string) error { // if no valid partners are configured, all partners are accepted/validated if len(s.validPartners) == 0 { diff --git a/http/webconfig_server_test.go b/http/webconfig_server_test.go index 0c028aa..f7d484e 100644 --- a/http/webconfig_server_test.go +++ b/http/webconfig_server_test.go @@ -122,6 +122,13 @@ func TestWebconfigServerSetterGetter(t *testing.T) { server.SetMinTrust(trust) assert.Equal(t, server.MinTrust(), trust) + x := true + server.SetFilterOutputByBitmapEnabled(x) + assert.Assert(t, server.FilterOutputByBitmapEnabled()) + x = false + server.SetFilterOutputByBitmapEnabled(x) + assert.Assert(t, !server.FilterOutputByBitmapEnabled()) + validSubdocIdMap := map[string]int{ "red": 1, "orange": 2, diff --git a/util/firmware_bitmap.go b/util/firmware_bitmap.go index ab308fc..d9fa17d 100644 --- a/util/firmware_bitmap.go +++ b/util/firmware_bitmap.go @@ -14,7 +14,24 @@ * limitations under the License. * * SPDX-License-Identifier: Apache-2.0 -*/ + */ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +* SPDX-License-Identifier: Apache-2.0 + */ package util import ( diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go index cf3d5a5..abd622e 100644 --- a/util/firmware_bitmap_test.go +++ b/util/firmware_bitmap_test.go @@ -13,6 +13,23 @@ * See the License for the specific language governing permissions and * limitations under the License. * +* SPDX-License-Identifier: Apache-2.0 + */ +/** +* Copyright 2021 Comcast Cable Communications Management, LLC +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* * SPDX-License-Identifier: Apache-2.0 */ package util From fb559f477a99cabbea959333698944323e8a4b55 Mon Sep 17 00:00:00 2001 From: James Chao Date: Wed, 29 Jan 2025 21:49:25 -0800 Subject: [PATCH 155/155] add a log field and some cleanup --- config/sample_webconfig.conf | 2 + http/poke_handler.go | 1 + util/firmware_bitmap.go | 166 ---------- util/firmware_bitmap_test.go | 606 ----------------------------------- util/string.go | 14 + 5 files changed, 17 insertions(+), 772 deletions(-) delete mode 100644 util/firmware_bitmap.go delete mode 100644 util/firmware_bitmap_test.go diff --git a/config/sample_webconfig.conf b/config/sample_webconfig.conf index 36c8843..779c0c7 100644 --- a/config/sample_webconfig.conf +++ b/config/sample_webconfig.conf @@ -273,4 +273,6 @@ webconfig { // subdoc ids accepted by the validator even if they are without bitmap definition valid_subdoc_ids = [] + + filter_output_by_bitmap_enabled = false } diff --git a/http/poke_handler.go b/http/poke_handler.go index 29e19f3..4c5093c 100644 --- a/http/poke_handler.go +++ b/http/poke_handler.go @@ -63,6 +63,7 @@ func (s *WebconfigServer) PokeHandler(w http.ResponseWriter, r *http.Request) { } fields := xw.Audit() + fields["API"] = util.GetAPIName(r.URL.RawQuery) // extract "metrics_agent" metricsAgent := "default" diff --git a/util/firmware_bitmap.go b/util/firmware_bitmap.go deleted file mode 100644 index d9fa17d..0000000 --- a/util/firmware_bitmap.go +++ /dev/null @@ -1,166 +0,0 @@ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 - */ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 - */ -package util - -import ( - "fmt" - "strconv" - "strings" - - "github.com/rdkcentral/webconfig/common" -) - -// bitmap is used to name int variables -// bitarray is used to name string variables - -func SetBitmapByGroup(cpeBitmap *int, groupId int, groupBitmap int) error { - tuples, ok := common.SupportedDocsBitMaskMap[groupId] - if !ok { - return nil - } - - for _, tuple := range tuples { - mask := 1 << (tuple.GroupBit - 1) - masked := groupBitmap & mask - if masked > 0 { - (*cpeBitmap) |= 1 << (tuple.CpeBit - 1) - } - } - return nil -} - -func ParseFirmwareGroupBitarray(s string) (int, int, error) { - i, err := strconv.Atoi(s) - if err != nil { - return 0, 0, common.NewError(err) - } - groupId := i >> 24 - groupBitMask := 1<<24 - 1 - groupBitmap := i & groupBitMask - return groupId, groupBitmap, nil -} - -func GetCpeBitmap(rdkSupportedDocsHeaderStr string) (int, error) { - cpeBitmap := 0 - - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - for _, sid := range sids { - groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) - if err != nil { - return 0, common.NewError(err) - } - - err = SetBitmapByGroup(&cpeBitmap, groupId, groupBitmap) - if err != nil { - return 0, common.NewError(err) - } - } - - return cpeBitmap, nil -} - -func PrettyBitarray(i int) string { - x := fmt.Sprintf("%032b", i) - return fmt.Sprintf("%s %s %s %s %s %s %s %s", - x[0:4], - x[4:8], - x[8:12], - x[12:16], - x[16:20], - x[20:24], - x[24:28], - x[28:32]) -} - -func PrettyGroupBitarray(i int) string { - x := fmt.Sprintf("%032b", i) - return fmt.Sprintf("%s %s %s %s %s %s %s", - x[0:8], - x[8:12], - x[12:16], - x[16:20], - x[20:24], - x[24:28], - x[28:32]) -} - -func IsSubdocSupported(cpeBitmap int, subdocId string) bool { - index, ok := common.SubdocBitIndexMap[subdocId] - if !ok { - return false - } - - shift := index - 1 - bitmask := 1 << shift - masked := cpeBitmap & bitmask - return masked != 0 -} - -func GetSupportedMap(cpeBitmap int) map[string]bool { - supportedMap := map[string]bool{} - - for k, index := range common.SubdocBitIndexMap { - shift := index - 1 - bitmask := 1 << shift - supportedMap[k] = false - if masked := cpeBitmap & bitmask; masked > 0 { - supportedMap[k] = true - } - } - return supportedMap -} - -func BitarrayToBitmap(src string) (int, error) { - s := strings.ReplaceAll(src, " ", "") - v, err := strconv.ParseInt(s, 2, 64) - if err != nil { - return 0, common.NewError(err) - } - i := int(v) - return i, nil -} - -func GetBitmapFromSupportedMap(srcMap map[string]bool) int { - var bitmap int - - for k, index := range common.SubdocBitIndexMap { - shift := index - 1 - bitmask := 1 << shift - if srcMap[k] { - bitmap += bitmask - } - } - return bitmap -} diff --git a/util/firmware_bitmap_test.go b/util/firmware_bitmap_test.go deleted file mode 100644 index abd622e..0000000 --- a/util/firmware_bitmap_test.go +++ /dev/null @@ -1,606 +0,0 @@ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 - */ -/** -* Copyright 2021 Comcast Cable Communications Management, LLC -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -* -* SPDX-License-Identifier: Apache-2.0 - */ -package util - -import ( - "fmt" - "strings" - "testing" - - "github.com/rdkcentral/webconfig/common" - "gotest.tools/assert" -) - -// bitmap is used to name int variables -// bitarray is used to name string variables - -func TestPrettyBitarray(t *testing.T) { - i := 8 - bs := PrettyBitarray(i) - expected := "0000 0000 0000 0000 0000 0000 0000 1000" - assert.Equal(t, bs, expected) - - j := 16777231 - bs = PrettyGroupBitarray(j) - expected = "00000001 0000 0000 0000 0000 0000 1111" - assert.Equal(t, bs, expected) -} - -func TestParseRdkGroupBitarray(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" - expectedMap := map[int]int{ - 1: 15, - 2: 3, - 3: 1, - 4: 1, - 5: 1, - 6: 1, - 7: 1, - 8: 1, - } - - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - - for _, sid := range sids { - groupId, groupBitmap, err := ParseFirmwareGroupBitarray(sid) - assert.NilError(t, err) - // assert.Equal(t, groupId, 1) - - expectedBitmap, ok := expectedMap[groupId] - assert.Assert(t, ok) - assert.Equal(t, groupBitmap, expectedBitmap) - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - - // bs := PrettyGroupBitarray(cpeBitmap) - - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - } - - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } -} - -func TestParseCustomizedGroupBitarray(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554435,50331649,67108865,83886081,100663297,117440513,134217729" - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - - // build expected - expectedEnabled := map[string]bool{ - "portforwarding": true, - "lan": true, - "wan": true, - "macbinding": true, - "hotspot": false, - "bridge": false, - "privatessid": true, - "homessid": true, - "radio": false, - "moca": true, - "xdns": true, - "advsecurity": true, - "mesh": true, - "aker": true, - "telemetry": true, - "statusreport": false, - "trafficreport": false, - "interfacereport": false, - "radioreport": false, - "telcovoip": false, - "telcovoice": false, - "wanmanager": false, - "voiceservice": false, - } - - newGroup1Bitarray := "00000001 0000 0000 0000 0000 0011 0011" - group1Bitmap, err := BitarrayToBitmap(newGroup1Bitarray) - assert.NilError(t, err) - sids[0] = fmt.Sprintf("%v", group1Bitmap) - expectedEnabled["wan"] = false - expectedEnabled["macbinding"] = false - expectedEnabled["hotspot"] = true - expectedEnabled["bridge"] = true - - newGroup2Bitarray := "00000010 0000 0000 0000 0000 0000 0110" - group2Bitmap, err := BitarrayToBitmap(newGroup2Bitarray) - assert.NilError(t, err) - sids[1] = fmt.Sprintf("%v", group2Bitmap) - expectedEnabled["privatessid"] = false - expectedEnabled["radio"] = true - - rdkSupportedDocsHeaderStr = strings.Join(sids, ",") - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for subdocId, enabled := range expectedEnabled { - parsedEnabled := IsSubdocSupported(cpeBitmap, subdocId) - assert.Equal(t, parsedEnabled, enabled) - } - - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocIds := []string{} - for k, v := range expectedEnabled { - if v { - supportedSubdocIds = append(supportedSubdocIds, k) - } - } - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseTelcovoipAndWanmanager(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549377,201326593" - sids := strings.Split(rdkSupportedDocsHeaderStr, ",") - - // build expected - supportedSubdocIds := []string{ - "portforwarding", - "lan", - "wan", - "macbinding", - "privatessid", - "xdns", - "advsecurity", - "mesh", - "aker", - "telemetry", - "telcovoip", - "wanmanager", - } - - rdkSupportedDocsHeaderStr = strings.Join(sids, ",") - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - for _, subdocId := range supportedSubdocIds { - assert.Assert(t, IsSubdocSupported(cpeBitmap, subdocId)) - } - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestBitmapParsing(t *testing.T) { - // clearn the wan bit - newBitmap := 16777231 & ^(1 << 2) - rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,33554435,50331649,67108865,83886081,100663297,117440513,134217729", newBitmap) - - // build expected - supportedSubdocIds := []string{ - "portforwarding", - "lan", - "macbinding", - "privatessid", - "homessid", - "moca", - "xdns", - "advsecurity", - "mesh", - "aker", - "telemetry", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseVoiceService(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809" - - supportedSubdocIds := []string{ - "portforwarding", - "lan", - "wan", - "macbinding", - "hotspot", - "privatessid", - "homessid", - "moca", - "xdns", - "advsecurity", - "mesh", - "aker", - "telemetry", - "voiceservice", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestManualBitmap(t *testing.T) { - for i := 0; i < 10; i++ { - bitmap := RandomInt(40000) - parsedSupportedMap := GetSupportedMap(bitmap) - revBitmap := GetBitmapFromSupportedMap(parsedSupportedMap) - assert.Equal(t, bitmap, revBitmap) - } -} - -func TestParseSupportedDocsWithNewGroups(t *testing.T) { - cellularBitGroupId := 14 - xBitValue := (cellularBitGroupId << 24) + 1 - ss := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,218103809,234881025" - rdkSupportedDocsHeaderStr := fmt.Sprintf("%v,%v", ss, xBitValue) - - // build expected - supportedSubdocIds := []string{ - "portforwarding", - "lan", - "wan", - "macbinding", - "hotspot", - "privatessid", - "homessid", - "moca", - "xdns", - "advsecurity", - "mesh", - "aker", - "telemetry", - "voiceservice", - "cellularconfig", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWithSomeLTEGroups(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554435,67108865,100663297,117440513,134217729,201326595,234881025" - - // build expected - supportedSubdocIds := []string{ - "aker", - "cellularconfig", - "homessid", - "lan", - "macbinding", - "mesh", - "portforwarding", - "privatessid", - "telemetry", - "wan", - "wanfailover", - "wanmanager", - "xdns", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWithTelcovoice(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777231,33554433,67108865,83886081,100663297,117440513,134217729,184549378,201326595" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "lan", - "macbinding", - "mesh", - "portforwarding", - "privatessid", - "telcovoice", - "telemetry", - "wan", - "wanfailover", - "wanmanager", - "xdns", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWithGwfailover(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWithPrioritizedMacs(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777247,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,251658241,268435457" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWithPrioritizedMacsAndConnectedbuilding(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663297,117440513,134217729,201326594,218103809,251658241,268435457,285212673" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - "connectedbuilding", - "lldqoscontrol", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderClienttosteeringprofile(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217729,201326594,218103809,251658241,268435457,285212673" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - "connectedbuilding", - "lldqoscontrol", - "clienttosteeringprofile", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderRfc(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663299,117440513,134217735,201326594,218103809,251658241,268435457,285212673" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - "connectedbuilding", - "lldqoscontrol", - "clienttosteeringprofile", - "defaultrfc", - "rfc", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderHcm(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777311,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - "connectedbuilding", - "lldqoscontrol", - "clienttosteeringprofile", - "meshsteeringprofiles", - "wifistatsconfig", - "mwoconfigs", - "interference", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} - -func TestParseSupportedDocsHeaderWebui(t *testing.T) { - rdkSupportedDocsHeaderStr := "16777695,33554435,50331649,67108865,83886081,100663359,117440513,134217729,201326594,218103809,251658241,268435457,285212673" - - // build expected - supportedSubdocIds := []string{ - "advsecurity", - "aker", - "gwfailover", - "homessid", - "hotspot", - "lan", - "macbinding", - "mesh", - "moca", - "portforwarding", - "privatessid", - "telemetry", - "voiceservice", - "wan", - "wanfailover", - "xdns", - "prioritizedmacs", - "connectedbuilding", - "lldqoscontrol", - "clienttosteeringprofile", - "meshsteeringprofiles", - "wifistatsconfig", - "mwoconfigs", - "interference", - "xmspeedboost", - "webui", - } - - cpeBitmap, err := GetCpeBitmap(rdkSupportedDocsHeaderStr) - assert.NilError(t, err) - parsedSupportedMap := GetSupportedMap(cpeBitmap) - supportedSubdocMap := common.BuildSupportedSubdocMapWithDefaults(supportedSubdocIds) - assert.DeepEqual(t, parsedSupportedMap, supportedSubdocMap) -} diff --git a/util/string.go b/util/string.go index 064cc8e..6c6cc0c 100644 --- a/util/string.go +++ b/util/string.go @@ -162,3 +162,17 @@ func CompactJson(sbytes []byte) ([]byte, error) { } return jsbytes, nil } + +func GetAPIName(queryStr string) string { + n := "poke" + if strings.Contains(queryStr, "slow") { + n = "slowpoke" + } else if strings.Contains(queryStr, "root") && strings.Contains(queryStr, "telemetry") { + n = "poke_both" + } else if strings.Contains(queryStr, "telemetry") { + n = "poke_telemetry" + } else if strings.Contains(queryStr, "mqtt") { + n = "poke_mqtt" + } + return n +}