Merge pull request #6 from lablabs/feature/use_httpAdaptiveGroups

Feature/use http adaptive groups

.github/workflows/lint-test.yaml

@@ -11,19 +11,32 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Run chart-testing (lint)
-        id: lint
-        uses: helm/[email protected]
+      - name: Set up Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: v3.4.0
+
+      - uses: actions/setup-python@v2
         with:
-          command: lint
-          config: ct.yaml
+          python-version: 3.7
+
+      - name: Set up chart-testing
+        uses: helm/[email protected]
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --target-branch master)
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
+      - name: Run chart-testing (lint)
+        run: ct lint --target-branch master
 
       - name: Create kind cluster
         uses: helm/[email protected]
-        if: steps.lint.outputs.changed == 'true'
+        if: steps.list-changed.outputs.changed == 'true'
 
       - name: Run chart-testing (install)
-        uses: helm/[email protected]
-        with:
-          command: install
-          config: ct.yaml
+        run: ct install --target-branch master

cloudflare.go

@@ -76,27 +76,15 @@ type zoneResp struct {
 			Datetime string `json:"datetime"`
 			ColoCode string `json:"coloCode"`
 		} `json:"dimensions"`
+		Count uint64 `json:"count"`
 		Sum struct {
-			Bytes          uint64 `json:"bytes"`
-			CachedBytes    uint64 `json:"cachedBytes"`
-			CachedRequests uint64 `json:"cachedRequests"`
-			Requests       uint64 `json:"requests"`
-			CountryMap     []struct {
-				ClientCountryName string `json:"clientCountryName"`
-				Requests          uint64 `json:"requests"`
-				Threats           uint64 `json:"threats"`
-			} `json:"countryMap"`
-			ResponseStatusMap []struct {
-				EdgeResponseStatus int    `json:"edgeResponseStatus"`
-				Requests           uint64 `json:"requests"`
-			} `json:"responseStatusMap"`
-			ThreatPathingMap []struct {
-				ThreatPathingName string `json:"threatPathingName"`
-				Requests          uint64 `json:"requests"`
-			} `json:"threatPathingMap"`
+			EdgeResponseBytes uint64 `json:"edgeResponseBytes"`
+			Visits uint64 `json:"visits"`
 		} `json:"sum"`
-	} `json:"httpRequests1mByColoGroups"`
-
+		Avg struct {
+			sampleInterval uint64 `json:"sampleInterval"`
+		} `json:"avg"`
+	} `json:"httpRequestsAdaptiveGroups"`
 	ZoneTag string `json:"zoneTag"`
 }
 
@@ -212,55 +200,43 @@ func fetchColoTotals(zoneIDs []string) (*cloudflareResponse, error) {
 	s := 60 * time.Second
 	now = now.Truncate(s)
 
-	http1mGroupsByColo := graphql.NewRequest(`
+	httpRequestsAdaptiveGroups := graphql.NewRequest(`
 	query ($zoneIDs: [String!], $time: Time!, $limit: Int!) {
 		viewer {
 			zones(filter: { zoneTag_in: $zoneIDs }) {
 				zoneTag
-
-				httpRequests1mByColoGroups(
+				httpRequestsAdaptiveGroups(
 					limit: $limit
 					filter: { datetime: $time }
-				) {
-					sum {
-						requests
-						bytes
-						countryMap {
-							clientCountryName
-							requests
-							threats
+					) {
+						count
+						avg {
+							sampleInterval
 						}
-						responseStatusMap {
-							edgeResponseStatus
-							requests
+						dimensions {
+							coloCode
+							datetime
 						}
-						cachedRequests
-						cachedBytes
-						threatPathingMap {
-							requests
-							threatPathingName
+						sum {
+							edgeResponseBytes
+							visits
 						}
 					}
-					dimensions {
-						coloCode
-						datetime
-					}
 				}
 			}
 		}
-	}
 `)
 
-	http1mGroupsByColo.Header.Set("X-AUTH-EMAIL", os.Getenv("CF_API_EMAIL"))
-	http1mGroupsByColo.Header.Set("X-AUTH-KEY", os.Getenv("CF_API_KEY"))
-	http1mGroupsByColo.Var("limit", 9999)
-	http1mGroupsByColo.Var("time", now)
-	http1mGroupsByColo.Var("zoneIDs", zoneIDs)
+	httpRequestsAdaptiveGroups.Header.Set("X-AUTH-EMAIL", os.Getenv("CF_API_EMAIL"))
+	httpRequestsAdaptiveGroups.Header.Set("X-AUTH-KEY", os.Getenv("CF_API_KEY"))
+	httpRequestsAdaptiveGroups.Var("limit", 9999)
+	httpRequestsAdaptiveGroups.Var("time", now)
+	httpRequestsAdaptiveGroups.Var("zoneIDs", zoneIDs)
 
 	ctx := context.Background()
 	graphqlClient := graphql.NewClient("https://api.cloudflare.com/client/v4/graphql/")
 	var resp cloudflareResponse
-	if err := graphqlClient.Run(ctx, http1mGroupsByColo, &resp); err != nil {
+	if err := graphqlClient.Run(ctx, httpRequestsAdaptiveGroups, &resp); err != nil {
 		log.Error(err)
 		return nil, err
 	}

prometheus.go

@@ -138,47 +138,17 @@ var (
 	}, []string{"zone"},
 	)
 
-	zoneColocationRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "cloudflare_zone_colocation_requests_total",
-		Help: "Total requests per colocation",
+	zoneColocationVisits = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "cloudflare_zone_colocation_visits",
+		Help: "Total visits per colocation",
 	}, []string{"zone", "colocation"},
 	)
 
-	zoneColocationRequestsCached = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "cloudflare_zone_colocation_requests_cached",
-		Help: "Total cached requests per colocation",
+	zoneColocationEdgeResponseBytes = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "cloudflare_zone_colocation_edge_response_bytes",
+		Help: "Edge response bytes per colocation",
 	}, []string{"zone", "colocation"},
 	)
-
-	zoneColocationBandwidthTotal = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "cloudflare_zone_colocation_bandwidth_total",
-		Help: "Total bandwidth per colocation",
-	}, []string{"zone", "colocation"},
-	)
-
-	zoneColocationBandwidthCached = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "cloudflare_zone_colocation_bandwidth_cached",
-		Help: "Total cached bandwidth per colocation",
-	}, []string{"zone", "colocation"},
-	)
-
-	zoneColocationResponseStatus = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "cloudflare_zone_colocation_response_status",
-		Help: "HTTP response status per colocation",
-	}, []string{"zone", "colocation", "status"},
-	)
-
-	zoneColocationRequestsByCountry = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "cloudflare_zone_colocation_requests_country",
-		Help: "Requests per colocation per country",
-	}, []string{"zone", "colocation", "country", "region"},
-	)
-
-	zoneColocationThreatsByCountry = promauto.NewCounterVec(prometheus.CounterOpts{
-		Name: "cloudflare_zone_colocation_threats_country",
-		Help: "Threats per colocation per country",
-	}, []string{"zone", "colocation", "country", "region"},
-	)
 )
 
 func fetchZoneColocationAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) {
@@ -195,23 +165,9 @@ func fetchZoneColocationAnalytics(zones []cloudflare.Zone, wg *sync.WaitGroup) {
 
 		cg := z.ColoGroups
 		name := findZoneName(zones, z.ZoneTag)
-
 		for _, c := range cg {
-			zoneColocationRequestsTotal.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode}).Add(float64(c.Sum.Requests))
-			zoneColocationRequestsCached.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode}).Add(float64(c.Sum.CachedRequests))
-			zoneColocationBandwidthTotal.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode}).Add(float64(c.Sum.Bytes))
-			zoneColocationBandwidthCached.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode}).Add(float64(c.Sum.CachedBytes))
-
-			for _, s := range c.Sum.ResponseStatusMap {
-				zoneColocationResponseStatus.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode, "status": strconv.Itoa(s.EdgeResponseStatus)}).Add(float64(s.Requests))
-			}
-
-			for _, country := range c.Sum.CountryMap {
-				region := countries.ByName(country.ClientCountryName).Info().Region.Info().Name
-
-				zoneColocationRequestsByCountry.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode, "country": country.ClientCountryName, "region": region}).Add(float64(country.Requests))
-				zoneColocationRequestsByCountry.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode, "country": country.ClientCountryName, "region": region}).Add(float64(country.Threats))
-			}
+			zoneColocationVisits.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode}).Add(float64(c.Sum.Visits))
+			zoneColocationEdgeResponseBytes.With(prometheus.Labels{"zone": name, "colocation": c.Dimensions.ColoCode}).Add(float64(c.Sum.EdgeResponseBytes))
 		}
 	}