logs-analyzer/signoz/pkg/query-service/postprocess/process.go

package postprocess

import (
	"github.com/SigNoz/govaluate"
	"go.signoz.io/signoz/pkg/query-service/app/queryBuilder"
	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
	"go.uber.org/zap"
)

// postProcessResult applies having clause, metric limit, reduce function to the result
// This function is effective for metrics data source for now, but it can be extended to other data sources
// if needed
// Much of this work can be done in the ClickHouse query, but we decided to do it here because:
// 1. Effective use of caching
// 2. Easier to add new functions
func PostProcessResult(result []*v3.Result, queryRangeParams *v3.QueryRangeParamsV3) ([]*v3.Result, error) {
	// Having clause is not part of the clickhouse query, so we need to apply it here
	// It's not included in the query because it doesn't work nicely with caching
	// With this change, if you have a query with a having clause, and then you change the having clause
	// to something else, the query will still be cached.
	ApplyHavingClause(result, queryRangeParams)
	// We apply the metric limit here because it's not part of the clickhouse query
	// The limit in the context of the time series query is the number of time series
	// So for the limit to work, we need to know what series to keep and what to discard
	// For us to know that, we need to execute the query first, and then apply the limit
	// which we found expensive, because we are executing the query twice on the same data
	// So we decided to apply the limit here, after the query is executed
	// The function is named applyMetricLimit because it only applies to metrics data source
	// In traces and logs, the limit is achieved using subqueries
	ApplyMetricLimit(result, queryRangeParams)
	// Each series in the result produces N number of points, where N is (end - start) / step
	// For the panel type table, we need to show one point for each series in the row
	// We do that by applying a reduce function to each series
	applyReduceTo(result, queryRangeParams)
	// We apply the functions here it's easier to add new functions
	ApplyFunctions(result, queryRangeParams)

	// expressions are executed at query serivce so the value of time.now in the invdividual
	// queries will be different so for table panel we are making it same.
	if queryRangeParams.CompositeQuery.PanelType == v3.PanelTypeTable {
		tablePanelResultProcessor(result)
	}

	canDefaultZero := make(map[string]bool)

	for _, query := range queryRangeParams.CompositeQuery.BuilderQueries {
		canDefaultZero[query.QueryName] = query.CanDefaultZero()
	}

	for _, query := range queryRangeParams.CompositeQuery.BuilderQueries {
		// The way we distinguish between a formula and a query is by checking if the expression
		// is the same as the query name
		// TODO(srikanthccv): Update the UI to send a flag to distinguish between a formula and a query
		if query.Expression != query.QueryName {
			expression, err := govaluate.NewEvaluableExpressionWithFunctions(query.Expression, EvalFuncs())
			// This shouldn't happen here, because it should have been caught earlier in validation
			if err != nil {
				zap.L().Error("error in expression", zap.Error(err))
				return nil, err
			}
			formulaResult, err := processResults(result, expression, canDefaultZero)
			if err != nil {
				zap.L().Error("error in expression", zap.Error(err))
				return nil, err
			}
			formulaResult.QueryName = query.QueryName
			ApplyHavingClause([]*v3.Result{formulaResult}, queryRangeParams)
			ApplyMetricLimit([]*v3.Result{formulaResult}, queryRangeParams)
			result = append(result, formulaResult)
		}
	}
	// we are done with the formula calculations, only send the results for enabled queries
	removeDisabledQueries := func(result []*v3.Result) []*v3.Result {
		var newResult []*v3.Result
		for _, res := range result {
			if queryRangeParams.CompositeQuery.BuilderQueries[res.QueryName].Disabled {
				continue
			}
			newResult = append(newResult, res)
		}
		return newResult
	}
	if queryRangeParams.CompositeQuery.QueryType == v3.QueryTypeBuilder {
		result = removeDisabledQueries(result)
	}
	if queryRangeParams.CompositeQuery.FillGaps {
		FillGaps(result, queryRangeParams)
	}

	if queryRangeParams.FormatForWeb &&
		queryRangeParams.CompositeQuery.QueryType == v3.QueryTypeBuilder &&
		queryRangeParams.CompositeQuery.PanelType == v3.PanelTypeTable {
		result = TransformToTableForBuilderQueries(result, queryRangeParams)
	}

	return result, nil
}

// ApplyFunctions applies functions for each query in the composite query
// The functions can be more than one, and they are applied in the order they are defined
func ApplyFunctions(results []*v3.Result, queryRangeParams *v3.QueryRangeParamsV3) {
	for idx, result := range results {
		builderQueries := queryRangeParams.CompositeQuery.BuilderQueries

		if builderQueries != nil {
			functions := builderQueries[result.QueryName].Functions

			for _, function := range functions {
				results[idx] = queryBuilder.ApplyFunction(function, result)
			}
		}
	}
}

func tablePanelResultProcessor(results []*v3.Result) {
	var ts int64
	for ridx := range results {
		for sidx := range results[ridx].Series {
			for pidx := range results[ridx].Series[sidx].Points {
				if ts == 0 {
					ts = results[ridx].Series[sidx].Points[pidx].Timestamp
				} else {
					results[ridx].Series[sidx].Points[pidx].Timestamp = ts
				}
			}
		}
	}
}