logs-analyzer/signoz/pkg/query-service/app/logs/v3/enrich_query.go

235 lines
6.1 KiB
Go
Raw Normal View History

2024-09-02 22:47:30 +03:00
package v3
import (
"fmt"
"strconv"
"strings"
"go.signoz.io/signoz/pkg/query-service/constants"
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
)
func EnrichmentRequired(params *v3.QueryRangeParamsV3) bool {
compositeQuery := params.CompositeQuery
if compositeQuery == nil {
return false
}
// Build queries for each builder query
for queryName, query := range compositeQuery.BuilderQueries {
if query.Expression != queryName && query.DataSource != v3.DataSourceLogs {
continue
}
// check aggregation attribute
if query.AggregateAttribute.Key != "" {
if !isEnriched(query.AggregateAttribute) {
return true
}
}
// check filter attribute
if query.Filters != nil && len(query.Filters.Items) != 0 {
for _, item := range query.Filters.Items {
if !isEnriched(item.Key) {
return true
}
}
}
groupByLookup := map[string]struct{}{}
// check groupby
for _, groupBy := range query.GroupBy {
if !isEnriched(groupBy) {
return true
}
groupByLookup[groupBy.Key] = struct{}{}
}
// check orderby
for _, orderBy := range query.OrderBy {
if _, ok := groupByLookup[orderBy.ColumnName]; !ok {
key := v3.AttributeKey{Key: orderBy.ColumnName}
if !isEnriched(key) {
return true
}
}
}
}
return false
}
func isEnriched(field v3.AttributeKey) bool {
// if it is timestamp/id dont check
if field.Key == "timestamp" || field.Key == "id" || field.Key == constants.SigNozOrderByValue {
return true
}
if field.IsColumn {
return true
}
if field.Type == v3.AttributeKeyTypeUnspecified || field.DataType == v3.AttributeKeyDataTypeUnspecified {
return false
}
// try to enrich all attributes which doesn't have isColumn = true
if !field.IsColumn {
return false
}
return true
}
func Enrich(params *v3.QueryRangeParamsV3, fields map[string]v3.AttributeKey) {
compositeQuery := params.CompositeQuery
if compositeQuery == nil {
return
}
// Build queries for each builder query
for queryName, query := range compositeQuery.BuilderQueries {
if query.Expression != queryName && query.DataSource != v3.DataSourceLogs {
continue
}
enrichLogsQuery(query, fields)
}
}
func enrichLogsQuery(query *v3.BuilderQuery, fields map[string]v3.AttributeKey) error {
// enrich aggregation attribute
if query.AggregateAttribute.Key != "" {
query.AggregateAttribute = enrichFieldWithMetadata(query.AggregateAttribute, fields)
}
// enrich filter attribute
if query.Filters != nil && len(query.Filters.Items) != 0 {
for i := 0; i < len(query.Filters.Items); i++ {
query.Filters.Items[i] = jsonFilterEnrich(query.Filters.Items[i])
if query.Filters.Items[i].Key.IsJSON {
query.Filters.Items[i] = jsonReplaceField(query.Filters.Items[i], fields)
continue
}
query.Filters.Items[i].Key = enrichFieldWithMetadata(query.Filters.Items[i].Key, fields)
}
}
// enrich groupby
for i := 0; i < len(query.GroupBy); i++ {
query.GroupBy[i] = enrichFieldWithMetadata(query.GroupBy[i], fields)
}
// enrich orderby
for i := 0; i < len(query.OrderBy); i++ {
key := v3.AttributeKey{Key: query.OrderBy[i].ColumnName}
key = enrichFieldWithMetadata(key, fields)
query.OrderBy[i].Key = key.Key
query.OrderBy[i].Type = key.Type
query.OrderBy[i].DataType = key.DataType
query.OrderBy[i].IsColumn = key.IsColumn
}
return nil
}
func enrichFieldWithMetadata(field v3.AttributeKey, fields map[string]v3.AttributeKey) v3.AttributeKey {
if isEnriched(field) {
return field
}
// if type is unknown check if it is a top level key
if v, ok := constants.StaticFieldsLogsV3[field.Key]; ok {
return v
}
// check if the field is present in the fields map
if existingField, ok := fields[field.Key]; ok {
// don't update if type is not the same
if (field.Type == "" && field.DataType == "") ||
(field.Type == existingField.Type && field.DataType == existingField.DataType) ||
(field.Type == "" && field.DataType == existingField.DataType) ||
(field.DataType == "" && field.Type == existingField.Type) {
return existingField
}
}
// enrich with default values if metadata is not found
if field.Type == "" {
field.Type = v3.AttributeKeyTypeTag
}
if field.DataType == "" {
field.DataType = v3.AttributeKeyDataTypeString
}
return field
}
func jsonFilterEnrich(filter v3.FilterItem) v3.FilterItem {
// check if it is a json request
if !strings.HasPrefix(filter.Key.Key, "body.") {
return filter
}
// check if the value is a int, float, string, bool
valueType := ""
switch filter.Value.(type) {
case uint8, uint16, uint32, uint64, int, int8, int16, int32, int64:
valueType = "int64"
case float32, float64:
valueType = "float64"
case string:
valueType, filter.Value = parseStrValue(filter.Value.(string), filter.Operator)
case bool:
valueType = "bool"
}
// check if it is array
if strings.HasSuffix(filter.Key.Key, "[*]") {
valueType = fmt.Sprintf("array(%s)", valueType)
}
filter.Key.DataType = v3.AttributeKeyDataType(valueType)
filter.Key.IsJSON = true
return filter
}
func jsonReplaceField(filter v3.FilterItem, fields map[string]v3.AttributeKey) v3.FilterItem {
key, found := strings.CutPrefix(filter.Key.Key, "body.")
if !found {
return filter
}
if field, ok := fields[key]; ok && field.DataType == filter.Key.DataType {
filter.Key = field
}
return filter
}
func parseStrValue(valueStr string, operator v3.FilterOperator) (string, interface{}) {
valueType := "string"
// for the following operators it will always be string
if operator == v3.FilterOperatorContains || operator == v3.FilterOperatorNotContains ||
operator == v3.FilterOperatorRegex || operator == v3.FilterOperatorNotRegex ||
operator == v3.FilterOperatorLike || operator == v3.FilterOperatorNotLike {
return valueType, valueStr
}
var err error
var parsedValue interface{}
if parsedValue, err = strconv.ParseBool(valueStr); err == nil {
valueType = "bool"
} else if parsedValue, err = strconv.ParseInt(valueStr, 10, 64); err == nil {
valueType = "int64"
} else if parsedValue, err = strconv.ParseFloat(valueStr, 64); err == nil {
valueType = "float64"
} else {
parsedValue = valueStr
valueType = "string"
}
return valueType, parsedValue
}