105 lines
3.7 KiB
Go
105 lines
3.7 KiB
Go
|
package logparsingpipeline
|
||
|
|
|
||
|
|
import (
|
||
|
|
"encoding/json"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/pkg/errors"
|
||
|
|
v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
|
||
|
|
)
|
||
|
|
|
||
|
|
// Pipeline is stored and also deployed finally to collector config
|
||
|
|
type Pipeline struct {
|
||
|
|
Id string `json:"id,omitempty" db:"id"`
|
||
|
|
OrderId int `json:"orderId" db:"order_id"`
|
||
|
|
Name string `json:"name,omitempty" db:"name"`
|
||
|
|
Alias string `json:"alias" db:"alias"`
|
||
|
|
Description *string `json:"description" db:"description"`
|
||
|
|
Enabled bool `json:"enabled" db:"enabled"`
|
||
|
|
Filter *v3.FilterSet `json:"filter" db:"filter"`
|
||
|
|
|
||
|
|
// configuration for pipeline
|
||
|
|
RawConfig string `db:"config_json" json:"-"`
|
||
|
|
|
||
|
|
Config []PipelineOperator `json:"config"`
|
||
|
|
|
||
|
|
// Updater not required as any change will result in new version
|
||
|
|
Creator
|
||
|
|
}
|
||
|
|
|
||
|
|
type Creator struct {
|
||
|
|
CreatedBy string `json:"createdBy" db:"created_by"`
|
||
|
|
CreatedAt time.Time `json:"createdAt" db:"created_at"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Processor struct {
|
||
|
|
Operators []PipelineOperator `json:"operators" yaml:"operators"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type PipelineOperator struct {
|
||
|
|
Type string `json:"type" yaml:"type"`
|
||
|
|
ID string `json:"id,omitempty" yaml:"id,omitempty"`
|
||
|
|
Output string `json:"output,omitempty" yaml:"output,omitempty"`
|
||
|
|
OnError string `json:"on_error,omitempty" yaml:"on_error,omitempty"`
|
||
|
|
If string `json:"if,omitempty" yaml:"if,omitempty"`
|
||
|
|
|
||
|
|
// don't need the following in the final config
|
||
|
|
OrderId int `json:"orderId" yaml:"-"`
|
||
|
|
Enabled bool `json:"enabled" yaml:"-"`
|
||
|
|
Name string `json:"name,omitempty" yaml:"-"`
|
||
|
|
|
||
|
|
// optional keys depending on the type
|
||
|
|
ParseTo string `json:"parse_to,omitempty" yaml:"parse_to,omitempty"`
|
||
|
|
Pattern string `json:"pattern,omitempty" yaml:"pattern,omitempty"`
|
||
|
|
Regex string `json:"regex,omitempty" yaml:"regex,omitempty"`
|
||
|
|
ParseFrom string `json:"parse_from,omitempty" yaml:"parse_from,omitempty"`
|
||
|
|
*TraceParser `yaml:",inline,omitempty"`
|
||
|
|
Field string `json:"field,omitempty" yaml:"field,omitempty"`
|
||
|
|
Value string `json:"value,omitempty" yaml:"value,omitempty"`
|
||
|
|
From string `json:"from,omitempty" yaml:"from,omitempty"`
|
||
|
|
To string `json:"to,omitempty" yaml:"to,omitempty"`
|
||
|
|
Expr string `json:"expr,omitempty" yaml:"expr,omitempty"`
|
||
|
|
Routes *[]Route `json:"routes,omitempty" yaml:"routes,omitempty"`
|
||
|
|
Fields []string `json:"fields,omitempty" yaml:"fields,omitempty"`
|
||
|
|
Default string `json:"default,omitempty" yaml:"default,omitempty"`
|
||
|
|
|
||
|
|
// time_parser fields.
|
||
|
|
Layout string `json:"layout,omitempty" yaml:"layout,omitempty"`
|
||
|
|
LayoutType string `json:"layout_type,omitempty" yaml:"layout_type,omitempty"`
|
||
|
|
|
||
|
|
// severity parser fields
|
||
|
|
SeverityMapping map[string][]string `json:"mapping,omitempty" yaml:"mapping,omitempty"`
|
||
|
|
OverwriteSeverityText bool `json:"overwrite_text,omitempty" yaml:"overwrite_text,omitempty"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type TimestampParser struct {
|
||
|
|
Layout string `json:"layout" yaml:"layout"`
|
||
|
|
LayoutType string `json:"layout_type" yaml:"layout_type"`
|
||
|
|
ParseFrom string `json:"parse_from" yaml:"parse_from"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type TraceParser struct {
|
||
|
|
TraceId *ParseFrom `json:"trace_id,omitempty" yaml:"trace_id,omitempty"`
|
||
|
|
SpanId *ParseFrom `json:"span_id,omitempty" yaml:"span_id,omitempty"`
|
||
|
|
TraceFlags *ParseFrom `json:"trace_flags,omitempty" yaml:"trace_flags,omitempty"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type ParseFrom struct {
|
||
|
|
ParseFrom string `json:"parse_from" yaml:"parse_from"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type Route struct {
|
||
|
|
Output string `json:"output" yaml:"output"`
|
||
|
|
Expr string `json:"expr" yaml:"expr"`
|
||
|
|
}
|
||
|
|
|
||
|
|
func (i *Pipeline) ParseRawConfig() error {
|
||
|
|
c := []PipelineOperator{}
|
||
|
|
err := json.Unmarshal([]byte(i.RawConfig), &c)
|
||
|
|
if err != nil {
|
||
|
|
return errors.Wrap(err, "failed to parse ingestion rule config")
|
||
|
|
}
|
||
|
|
i.Config = c
|
||
|
|
return nil
|
||
|
|
}
|