Skip to content

Commit

Permalink
Add support /select/logsql/hits for log histograms (#146)
Browse files Browse the repository at this point in the history 
* add support `/select/logsql/hits` for log histograms

* issue-142: add hits endpoint

* issue-142: add hits endpoint

* issue-142: add hits endpoint

* issue-142: add hits endpoint

* issue-142: fix labels

* issue-142: add field to the query

* issue-142: fix value labels

* issue-142: add field config

* issue-142: fix display name from ds

* issue-142: add hits endpoint

issue-142: add hits endpoint

issue-142: add hits endpoint

issue-142: add hits endpoint

issue-142: fix labels

issue-142: add field to the query

issue-142: fix value labels

issue-142: add field config

issue-142: fix display name from ds

issue-142: fix display hits

* add step param to `/select/logsql/hits`

* issue-142: add tests for hits url

* issue-142: add response tests

* issue-142: fix comments

* issue-142: fix tests

---------

Co-authored-by: dmitryk-dk <[email protected]>
  • Loading branch information
@Loori-R @dmitryk-dk
Loori-R and dmitryk-dk authored Dec 10, 2024
1 parent 62c508f commit 49f21ee
Show file tree
Hide file tree
Showing 8 changed files with 491 additions and 1 deletion.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

* FEATURE: add tooltips and info messages for query types. Now, plugin will warn about correct usage of `stats` panels and will provide more info about different query types.
* FEATURE: automatically add `_time` field if it s not present in the query for the `stats` [API call](https://docs.victoriametrics.com/victorialogs/querying/#querying-log-stats). See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/142).
* FEATURE: add support for `/select/logs/hits` to display precise logs volume on the Explore page. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/15).

* BUGFIX: fix bug with incomplete rendering of time series panels when selecting bigger time intervals.
* BUGFIX: fix a bug where the time range was reset when using query variables. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/118).
Expand Down
2 changes: 2 additions & 0 deletions pkg/plugin/datasource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,8 @@ func (d *Datasource) query(ctx context.Context, _ backend.PluginContext, q *Quer
return parseStatsResponse(r, q)
case QueryTypeStatsRange:
return parseStatsResponse(r, q)
case QueryTypeHits:
return parseHitsResponse(r)
default:
return parseInstantResponse(r)
}
Expand Down
46 changes: 46 additions & 0 deletions pkg/plugin/query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ const (
tailQueryPath = "/select/logsql/tail"
statsQueryPath = "/select/logsql/stats_query"
statsQueryRangePath = "/select/logsql/stats_query_range"
hitsQueryPath = "/select/logsql/hits"
defaultMaxLines = 1000
legendFormatAuto = "__auto"
metricsName = "__name__"
Expand All @@ -37,6 +38,8 @@ const (
QueryTypeStats QueryType = "stats"
// QueryTypeStatsRange represents stats range query type
QueryTypeStatsRange QueryType = "statsRange"
// QueryTypeHits represents hits query type
QueryTypeHits QueryType = "hits"
)

// Query represents backend query object
Expand All @@ -50,6 +53,7 @@ type Query struct {
IntervalMs int64 `json:"intervalMs"`
MaxLines int `json:"maxLines"`
Step string `json:"step"`
Field string `json:"field"`
QueryType QueryType `json:"queryType"`
url *url.URL
}
Expand Down Expand Up @@ -80,6 +84,12 @@ func (q *Query) getQueryURL(rawURL string, queryParams string) (string, error) {
return "", fmt.Errorf("failed to calculate minimal interval: %w", err)
}
return q.statsQueryRangeURL(params, minInterval), nil
case QueryTypeHits:
minInterval, err := q.calculateMinInterval()
if err != nil {
return "", fmt.Errorf("failed to calculate minimal interval: %w", err)
}
return q.histQueryURL(params, minInterval), nil
default:
return q.queryInstantURL(params), nil
}
Expand Down Expand Up @@ -215,6 +225,42 @@ func (q *Query) statsQueryRangeURL(queryParams url.Values, minInterval time.Dura
return q.url.String()
}

// histQueryURL prepare query url for querying log hits
func (q *Query) histQueryURL(queryParams url.Values, minInterval time.Duration) string {
q.url.Path = path.Join(q.url.Path, hitsQueryPath)
values := q.url.Query()

for k, vl := range queryParams {
for _, v := range vl {
values.Add(k, v)
}
}

now := time.Now()
if q.TimeRange.From.IsZero() {
q.TimeRange.From = now.Add(-time.Minute * 5)
}
if q.TimeRange.To.IsZero() {
q.TimeRange.To = now
}

q.Expr = utils.ReplaceTemplateVariable(q.Expr, q.IntervalMs, q.TimeRange)

step := q.Step
if step == "" {
step = utils.CalculateStep(minInterval, q.TimeRange, q.MaxDataPoints).String()
}

values.Set("query", q.Expr)
values.Set("start", strconv.FormatInt(q.TimeRange.From.Unix(), 10))
values.Set("end", strconv.FormatInt(q.TimeRange.To.Unix(), 10))
values.Set("step", step)
values.Set("field", q.Field)

q.url.RawQuery = values.Encode()
return q.url.String()
}

func (q *Query) addMetadataToMultiFrame(frame *data.Frame) {
if len(frame.Fields) < 2 {
return
Expand Down
206 changes: 206 additions & 0 deletions pkg/plugin/query_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,212 @@ func TestQuery_getQueryURL(t *testing.T) {
want: "http://127.0.0.1:9428/select/logsql/stats_query?a=1&b=2&query=_time%3A%5B1609459200%2C+1609462800%5D+%2A+and+syslog+%7C+stats+by%28type%29+count%28%29&time=1609462800",
wantErr: false,
},
{
name: "empty values hits",
fields: fields{
RefID: "1",
Expr: "",
MaxLines: 0,
TimeRange: backend.TimeRange{},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "",
queryParams: "",
},
want: "",
wantErr: true,
},
{
name: "has rawURL without params for hits",
fields: fields{
RefID: "1",
Expr: "",
MaxLines: 0,
TimeRange: backend.TimeRange{
From: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
To: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
wantErr: false,
},
{
name: "has rawURL without params for hits",
fields: fields{
RefID: "1",
Expr: "",
MaxLines: 0,
TimeRange: backend.TimeRange{
From: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
To: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
wantErr: false,
},
{
name: "has rawURL without params for hist",
fields: fields{
RefID: "1",
Expr: "",
MaxLines: 0,
TimeRange: backend.TimeRange{
From: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
To: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=&start=1609459200&step=15s",
wantErr: false,
},
{
name: "has expression and max lines",
fields: fields{
RefID: "1",
Expr: "_time:1s",
MaxLines: 10,
TimeRange: backend.TimeRange{
From: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
To: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s&start=1609459200&step=15s",
wantErr: false,
},
{
name: "has expression and max lines stats",
fields: fields{
RefID: "1",
Expr: "_time:1s | stats by(type) count()",
MaxLines: 10,
TimeRange: backend.TimeRange{
From: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
To: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
wantErr: false,
},
{
name: "has expression and max lines stats",
fields: fields{
RefID: "1",
Expr: "_time:1s | stats by(type) count()",
MaxLines: 10,
TimeRange: backend.TimeRange{
From: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
To: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
wantErr: false,
},
{
name: "has expression and max lines, with queryParams for hits",
fields: fields{
RefID: "1",
Expr: "_time:1s and syslog",
MaxLines: 10,
TimeRange: backend.TimeRange{
From: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
To: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog&start=1609459200&step=15s",
wantErr: false,
},
{
name: "has expression and max lines, with queryParams for hits",
fields: fields{
RefID: "1",
Expr: "_time:1s and syslog | stats by(type) count()",
MaxLines: 10,
TimeRange: backend.TimeRange{
From: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
To: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
wantErr: false,
},
{
name: "has expression and max lines, with queryParams for hits",
fields: fields{
RefID: "1",
Expr: "_time:1s and syslog | stats by(type) count()",
MaxLines: 10,
TimeRange: backend.TimeRange{
From: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
To: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=_time%3A1s+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
wantErr: false,
},
{
name: "stats query without time field for hits",
fields: fields{
RefID: "1",
Expr: "* and syslog | stats by(type) count()",
MaxLines: 10,
TimeRange: backend.TimeRange{
From: time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC),
To: time.Date(2021, 1, 1, 1, 0, 0, 0, time.UTC),
},
QueryType: QueryTypeHits,
},
args: args{
rawURL: "http://127.0.0.1:9429",
queryParams: "",
},
want: "http://127.0.0.1:9429/select/logsql/hits?end=1609462800&field=&query=%2A+and+syslog+%7C+stats+by%28type%29+count%28%29&start=1609459200&step=15s",
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
71 changes: 71 additions & 0 deletions pkg/plugin/response.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ const (
gLabelsField = "labels"
gTimeField = "Time"
gLineField = "Line"
gValueField = "Value"

logsVisualisation = "logs"
)
Expand Down Expand Up @@ -288,6 +289,22 @@ func parseStatsResponse(reader io.Reader, q *Query) backend.DataResponse {
return backend.DataResponse{Frames: frames}
}

func parseHitsResponse(reader io.Reader) backend.DataResponse {
var hr HitsResponse
if err := json.NewDecoder(reader).Decode(&hr); err != nil {
err = fmt.Errorf("failed to decode body response: %w", err)
return newResponseError(err, backend.StatusInternal)
}

frames, err := hr.getDataFrames()
if err != nil {
err = fmt.Errorf("failed to prepare data from response: %w", err)
return newResponseError(err, backend.StatusInternal)
}

return backend.DataResponse{Frames: frames}
}

// parseErrorResponse reads data from the reader and returns error
func parseErrorResponse(reader io.Reader) error {
var rs Response
Expand Down Expand Up @@ -419,3 +436,57 @@ func (r *Response) getDataFrames() (data.Frames, error) {
return nil, fmt.Errorf("unknown result type %q", r.Data.ResultType)
}
}

// Hit represents a single hit from the query
type Hit struct {
Fields map[string]string `json:"fields"`
Timestamps []string `json:"timestamps"`
Values []float64 `json:"values"`
Total int `json:"total"`
}

// HitsResponse represents response from the hits query
type HitsResponse struct {
Hits []Hit `json:"hits"`
}

func (hr *HitsResponse) getDataFrames() (data.Frames, error) {
frames := make(data.Frames, len(hr.Hits))
for i, hit := range hr.Hits {
if len(hit.Timestamps) != len(hit.Values) {
return nil, fmt.Errorf("timestamps and values length mismatch: %d != %d", len(hit.Timestamps), len(hit.Values))
}

timeFd := data.NewFieldFromFieldType(data.FieldTypeTime, len(hit.Timestamps))
timeFd.Name = gTimeField

valueFd := data.NewFieldFromFieldType(data.FieldTypeFloat64, len(hit.Values))
valueFd.Name = gValueField
valueFd.Labels = make(data.Labels)

for j, ts := range hit.Timestamps {
getTime, err := utils.GetTime(ts)
if err != nil {
return nil, fmt.Errorf("error parse time from _time field: %s", err)
}
timeFd.Set(j, getTime)
}

for k, v := range hit.Values {
valueFd.Set(k, v)
}

for key, value := range hit.Fields {
valueFd.Labels[key] = value
d, err := labelsToJSON(valueFd.Labels)
if err != nil {
return nil, fmt.Errorf("error convert labels to json: %s", err)
}
valueFd.Config = &data.FieldConfig{DisplayNameFromDS: string(d)}
}

frames[i] = data.NewFrame("", timeFd, valueFd)
}

return frames, nil
}
Loading

0 comments on commit 49f21ee

Please sign in to comment.