package logparsingpipeline import ( "context" "encoding/json" "fmt" "strings" "testing" "time" "github.com/antonmedv/expr" "github.com/stretchr/testify/require" "go.signoz.io/signoz/pkg/query-service/model" v3 "go.signoz.io/signoz/pkg/query-service/model/v3" ) func TestRegexForStrptimeLayout(t *testing.T) { require := require.New(t) var testCases = []struct { strptimeLayout string str string shouldMatch bool }{ { strptimeLayout: "%Y-%m-%dT%H:%M:%S.%f%z", str: "2023-11-26T12:03:28.239907+0530", shouldMatch: true, }, { strptimeLayout: "%d-%m-%Y", str: "26-11-2023", shouldMatch: true, }, { strptimeLayout: "%d-%m-%Y", str: "26-11-2023", shouldMatch: true, }, { strptimeLayout: "%d/%m/%y", str: "11/03/02", shouldMatch: true, }, { strptimeLayout: "%A, %d. %B %Y %I:%M%p", str: "Tuesday, 21. November 2006 04:30PM11/03/02", shouldMatch: true, }, { strptimeLayout: "%A, %d. %B %Y %I:%M%p", str: "some random text", shouldMatch: false, }, } for _, test := range testCases { regex, err := RegexForStrptimeLayout(test.strptimeLayout) require.Nil(err, test.strptimeLayout) code := fmt.Sprintf(`"%s" matches "%s"`, test.str, regex) program, err := expr.Compile(code) require.Nil(err, test.strptimeLayout) output, err := expr.Run(program, map[string]string{}) require.Nil(err, test.strptimeLayout) require.Equal(output, test.shouldMatch, test.strptimeLayout) } } func TestTimestampParsingProcessor(t *testing.T) { require := require.New(t) testPipelines := []Pipeline{ { OrderId: 1, Name: "pipeline1", Alias: "pipeline1", Enabled: true, Filter: &v3.FilterSet{ Operator: "AND", Items: []v3.FilterItem{ { Key: v3.AttributeKey{ Key: "method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, }, Operator: "=", Value: "GET", }, }, }, Config: []PipelineOperator{}, }, } var timestampParserOp PipelineOperator err := json.Unmarshal([]byte(` { "orderId": 1, "enabled": true, "type": "time_parser", "name": "Test timestamp parser", "id": "test-timestamp-parser", "parse_from": "attributes.test_timestamp", "layout_type": "strptime", "layout": "%Y-%m-%dT%H:%M:%S.%f%z" } `), ×tampParserOp) require.Nil(err) testPipelines[0].Config = append(testPipelines[0].Config, timestampParserOp) testTimestampStr := "2023-11-27T12:03:28.239907+0530" testLog := makeTestSignozLog( "test log", map[string]interface{}{ "method": "GET", "test_timestamp": testTimestampStr, }, ) result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing( context.Background(), testPipelines, []model.SignozLog{ testLog, }, ) require.Nil(err) require.Equal(1, len(result)) require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n")) processed := result[0] expectedTimestamp, err := time.Parse("2006-01-02T15:04:05.999999-0700", testTimestampStr) require.Nil(err) require.Equal(uint64(expectedTimestamp.UnixNano()), processed.Timestamp) }