219 lines
8.0 KiB
Go
219 lines
8.0 KiB
Go
|
package delta
|
||
|
|
|
||
|
|
import (
|
||
|
|
"fmt"
|
||
|
|
|
||
|
|
"go.signoz.io/signoz/pkg/query-service/app/metrics/v4/helpers"
|
||
|
|
"go.signoz.io/signoz/pkg/query-service/constants"
|
||
|
|
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
|
||
|
|
"go.signoz.io/signoz/pkg/query-service/utils"
|
||
|
|
)
|
||
|
|
|
||
|
|
// TODO(srikanthccv): support multiple quantiles; see https://github.com/SigNoz/signoz/issues/4016#issuecomment-1838583305
|
||
|
|
var (
|
||
|
|
sketchFmt = "quantilesDDMerge(0.01, %f)(sketch)[1]"
|
||
|
|
)
|
||
|
|
|
||
|
|
// prepareTimeAggregationSubQuery builds the sub-query to be used for temporal aggregation
|
||
|
|
func prepareTimeAggregationSubQuery(start, end, step int64, mq *v3.BuilderQuery) (string, error) {
|
||
|
|
|
||
|
|
var subQuery string
|
||
|
|
|
||
|
|
timeSeriesSubQuery, err := helpers.PrepareTimeseriesFilterQuery(start, end, mq)
|
||
|
|
if err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
|
||
|
|
samplesTableFilter := fmt.Sprintf("metric_name = %s AND unix_milli >= %d AND unix_milli < %d", utils.ClickHouseFormattedValue(mq.AggregateAttribute.Key), start, end)
|
||
|
|
|
||
|
|
// Select the aggregate value for interval
|
||
|
|
queryTmpl :=
|
||
|
|
"SELECT fingerprint, %s" +
|
||
|
|
" toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL %d SECOND) as ts," +
|
||
|
|
" %s as per_series_value" +
|
||
|
|
" FROM " + constants.SIGNOZ_METRIC_DBNAME + "." + constants.SIGNOZ_SAMPLES_V4_TABLENAME +
|
||
|
|
" INNER JOIN" +
|
||
|
|
" (%s) as filtered_time_series" +
|
||
|
|
" USING fingerprint" +
|
||
|
|
" WHERE " + samplesTableFilter +
|
||
|
|
" GROUP BY fingerprint, ts" +
|
||
|
|
" ORDER BY fingerprint, ts"
|
||
|
|
|
||
|
|
selectLabelsAny := helpers.SelectLabelsAny(mq.GroupBy)
|
||
|
|
|
||
|
|
switch mq.TimeAggregation {
|
||
|
|
case v3.TimeAggregationAvg:
|
||
|
|
op := "avg(value)"
|
||
|
|
subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery)
|
||
|
|
case v3.TimeAggregationSum:
|
||
|
|
op := "sum(value)"
|
||
|
|
subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery)
|
||
|
|
case v3.TimeAggregationMin:
|
||
|
|
op := "min(value)"
|
||
|
|
subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery)
|
||
|
|
case v3.TimeAggregationMax:
|
||
|
|
op := "max(value)"
|
||
|
|
subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery)
|
||
|
|
case v3.TimeAggregationCount:
|
||
|
|
op := "count(value)"
|
||
|
|
subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery)
|
||
|
|
case v3.TimeAggregationCountDistinct:
|
||
|
|
op := "count(distinct(value))"
|
||
|
|
subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery)
|
||
|
|
case v3.TimeAggregationAnyLast:
|
||
|
|
op := "anyLast(value)"
|
||
|
|
subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery)
|
||
|
|
case v3.TimeAggregationRate:
|
||
|
|
op := fmt.Sprintf("sum(value)/%d", step)
|
||
|
|
subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery)
|
||
|
|
case v3.TimeAggregationIncrease:
|
||
|
|
op := "sum(value)"
|
||
|
|
subQuery = fmt.Sprintf(queryTmpl, selectLabelsAny, step, op, timeSeriesSubQuery)
|
||
|
|
}
|
||
|
|
return subQuery, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// See `canShortCircuit` below for details
|
||
|
|
func prepareQueryOptimized(start, end, step int64, mq *v3.BuilderQuery) (string, error) {
|
||
|
|
|
||
|
|
groupBy := helpers.GroupingSetsByAttributeKeyTags(mq.GroupBy...)
|
||
|
|
orderBy := helpers.OrderByAttributeKeyTags(mq.OrderBy, mq.GroupBy)
|
||
|
|
selectLabels := helpers.SelectLabels(mq.GroupBy)
|
||
|
|
|
||
|
|
var query string
|
||
|
|
|
||
|
|
timeSeriesSubQuery, err := helpers.PrepareTimeseriesFilterQuery(start, end, mq)
|
||
|
|
if err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
|
||
|
|
samplesTableFilter := fmt.Sprintf("metric_name = %s AND unix_milli >= %d AND unix_milli < %d", utils.ClickHouseFormattedValue(mq.AggregateAttribute.Key), start, end)
|
||
|
|
|
||
|
|
var tableName string = constants.SIGNOZ_SAMPLES_V4_TABLENAME
|
||
|
|
if mq.AggregateAttribute.Type == v3.AttributeKeyType(v3.MetricTypeExponentialHistogram) {
|
||
|
|
tableName = "distributed_exp_hist"
|
||
|
|
}
|
||
|
|
// Select the aggregate value for interval
|
||
|
|
queryTmpl :=
|
||
|
|
"SELECT %s" +
|
||
|
|
" toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), INTERVAL %d SECOND) as ts," +
|
||
|
|
" %s as value" +
|
||
|
|
" FROM " + constants.SIGNOZ_METRIC_DBNAME + "." + tableName +
|
||
|
|
" INNER JOIN" +
|
||
|
|
" (%s) as filtered_time_series" +
|
||
|
|
" USING fingerprint" +
|
||
|
|
" WHERE " + samplesTableFilter +
|
||
|
|
" GROUP BY %s" +
|
||
|
|
" ORDER BY %s"
|
||
|
|
|
||
|
|
switch mq.SpaceAggregation {
|
||
|
|
case v3.SpaceAggregationSum:
|
||
|
|
op := "sum(value)"
|
||
|
|
if mq.TimeAggregation == v3.TimeAggregationRate {
|
||
|
|
op = "sum(value)/" + fmt.Sprintf("%d", step)
|
||
|
|
}
|
||
|
|
query = fmt.Sprintf(queryTmpl, selectLabels, step, op, timeSeriesSubQuery, groupBy, orderBy)
|
||
|
|
case v3.SpaceAggregationMin:
|
||
|
|
op := "min(value)"
|
||
|
|
query = fmt.Sprintf(queryTmpl, selectLabels, step, op, timeSeriesSubQuery, groupBy, orderBy)
|
||
|
|
case v3.SpaceAggregationMax:
|
||
|
|
op := "max(value)"
|
||
|
|
query = fmt.Sprintf(queryTmpl, selectLabels, step, op, timeSeriesSubQuery, groupBy, orderBy)
|
||
|
|
case v3.SpaceAggregationPercentile50,
|
||
|
|
v3.SpaceAggregationPercentile75,
|
||
|
|
v3.SpaceAggregationPercentile90,
|
||
|
|
v3.SpaceAggregationPercentile95,
|
||
|
|
v3.SpaceAggregationPercentile99:
|
||
|
|
op := fmt.Sprintf(sketchFmt, v3.GetPercentileFromOperator(mq.SpaceAggregation))
|
||
|
|
query = fmt.Sprintf(queryTmpl, selectLabels, step, op, timeSeriesSubQuery, groupBy, orderBy)
|
||
|
|
}
|
||
|
|
return query, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// PrepareMetricQueryDeltaTimeSeries builds the query to be used for fetching metrics
|
||
|
|
func PrepareMetricQueryDeltaTimeSeries(start, end, step int64, mq *v3.BuilderQuery) (string, error) {
|
||
|
|
|
||
|
|
if canShortCircuit(mq) {
|
||
|
|
return prepareQueryOptimized(start, end, step, mq)
|
||
|
|
}
|
||
|
|
|
||
|
|
var query string
|
||
|
|
|
||
|
|
temporalAggSubQuery, err := prepareTimeAggregationSubQuery(start, end, step, mq)
|
||
|
|
if err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
|
||
|
|
groupBy := helpers.GroupingSetsByAttributeKeyTags(mq.GroupBy...)
|
||
|
|
orderBy := helpers.OrderByAttributeKeyTags(mq.OrderBy, mq.GroupBy)
|
||
|
|
selectLabels := helpers.GroupByAttributeKeyTags(mq.GroupBy...)
|
||
|
|
|
||
|
|
queryTmpl :=
|
||
|
|
"SELECT %s," +
|
||
|
|
" %s as value" +
|
||
|
|
" FROM (%s)" +
|
||
|
|
" WHERE isNaN(per_series_value) = 0" +
|
||
|
|
" GROUP BY %s" +
|
||
|
|
" ORDER BY %s"
|
||
|
|
|
||
|
|
switch mq.SpaceAggregation {
|
||
|
|
case v3.SpaceAggregationAvg:
|
||
|
|
op := "avg(per_series_value)"
|
||
|
|
query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy)
|
||
|
|
case v3.SpaceAggregationSum:
|
||
|
|
op := "sum(per_series_value)"
|
||
|
|
query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy)
|
||
|
|
case v3.SpaceAggregationMin:
|
||
|
|
op := "min(per_series_value)"
|
||
|
|
query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy)
|
||
|
|
case v3.SpaceAggregationMax:
|
||
|
|
op := "max(per_series_value)"
|
||
|
|
query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy)
|
||
|
|
case v3.SpaceAggregationCount:
|
||
|
|
op := "count(per_series_value)"
|
||
|
|
query = fmt.Sprintf(queryTmpl, selectLabels, op, temporalAggSubQuery, groupBy, orderBy)
|
||
|
|
}
|
||
|
|
|
||
|
|
return query, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// canShortCircuit returns true if we can use the optimized query
|
||
|
|
// for the given query
|
||
|
|
// This is used to avoid the group by fingerprint thus improving the performance
|
||
|
|
// for certain queries
|
||
|
|
// cases where we can short circuit:
|
||
|
|
// 1. time aggregation = (rate|increase) and space aggregation = sum
|
||
|
|
// - rate = sum(value)/step, increase = sum(value) - sum of sums is same as sum of all values
|
||
|
|
//
|
||
|
|
// 2. time aggregation = sum and space aggregation = sum
|
||
|
|
// - sum of sums is same as sum of all values
|
||
|
|
//
|
||
|
|
// 3. time aggregation = min and space aggregation = min
|
||
|
|
// - min of mins is same as min of all values
|
||
|
|
//
|
||
|
|
// 4. time aggregation = max and space aggregation = max
|
||
|
|
// - max of maxs is same as max of all values
|
||
|
|
//
|
||
|
|
// 5. special case exphist, there is no need for per series/fingerprint aggregation
|
||
|
|
// we can directly use the quantilesDDMerge function
|
||
|
|
//
|
||
|
|
// all of this is true only for delta metrics
|
||
|
|
func canShortCircuit(mq *v3.BuilderQuery) bool {
|
||
|
|
if (mq.TimeAggregation == v3.TimeAggregationRate || mq.TimeAggregation == v3.TimeAggregationIncrease) && mq.SpaceAggregation == v3.SpaceAggregationSum {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
if mq.TimeAggregation == v3.TimeAggregationSum && mq.SpaceAggregation == v3.SpaceAggregationSum {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
if mq.TimeAggregation == v3.TimeAggregationMin && mq.SpaceAggregation == v3.SpaceAggregationMin {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
if mq.TimeAggregation == v3.TimeAggregationMax && mq.SpaceAggregation == v3.SpaceAggregationMax {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
if mq.AggregateAttribute.Type == v3.AttributeKeyType(v3.MetricTypeExponentialHistogram) && v3.IsPercentileOperator(mq.SpaceAggregation) {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|