package logs import ( "net/http" "net/http/httptest" "testing" . "github.com/smartystreets/goconvey/convey" "go.signoz.io/signoz/pkg/query-service/model" ) var correctQueriesTest = []struct { Name string InputQuery string WantSqlTokens []string }{ { `filter with fulltext`, `OPERATION in ('bcd') AND FULLTEXT contains 'searchstring'`, []string{`OPERATION IN ('bcd') `, `AND body ILIKE '%searchstring%' `}, }, { `fulltext`, `searchstring`, []string{`body ILIKE '%searchstring%' `}, }, { `fulltext with quotes and space`, `FULLTEXT contains 'Hello, "World"'`, []string{`body ILIKE '%Hello, "World"%' `}, }, { `contains search with a different attributes`, `resource contains 'Hello, "World" and user\'s'`, []string{`resource ILIKE '%Hello, "World" and user\'s%' `}, }, { `more than one continas`, `resource contains 'Hello, "World"' and myresource contains 'abcd'`, []string{`resource ILIKE '%Hello, "World"%' `, `AND myresource ILIKE '%abcd%' `}, }, { "contains with or", `id in ('2CkBCauK8m3nkyKR19YhCw6WbdY') or fulltext contains 'OPTIONS /api/v1/logs'`, []string{`id IN ('2CkBCauK8m3nkyKR19YhCw6WbdY') `, `OR body ILIKE '%OPTIONS /api/v1/logs%' `}, }, { "mixing and or", `id in ('2CkBCauK8m3nkyKR19YhCw6WbdY') and id in ('2CkBCauK8m3nkyKR19YhCw6WbdY','2CkBCauK8m3nkyKR19YhCw6WbdY') or fulltext contains 'OPTIONS /api/v1/logs'`, []string{`id IN ('2CkBCauK8m3nkyKR19YhCw6WbdY') `, `and id IN ('2CkBCauK8m3nkyKR19YhCw6WbdY','2CkBCauK8m3nkyKR19YhCw6WbdY') `, `OR body ILIKE '%OPTIONS /api/v1/logs%' `}, }, { `filters with lt,gt,lte,gte operators`, `id lt 100 and id gt 50 and code lte 500 and code gte 400`, []string{`id < 100 `, `and id > 50 `, `and code <= 500 `, `and code >= 400 `}, }, { `filters with lt,gt,lte,gte operators seprated by OR`, `id lt 100 or id gt 50 or code lte 500 or code gte 400`, []string{`id < 100 `, `or id > 50 `, `or code <= 500 `, `or code >= 400 `}, }, { `filter with number`, `status gte 200 AND FULLTEXT ncontains '"key"'`, []string{`status >= 200 `, `AND body NOT ILIKE '%"key"%' `}, }, { `characters inside string`, `service NIN ('name > 100') AND length gt 100`, []string{`service NOT IN ('name > 100') `, `AND length > 100 `}, }, { `fulltext with in`, `key in 2`, []string{`body ILIKE '%key in 2%' `}, }, { `not valid fulltext but a filter`, `key in (2,3)`, []string{`key IN (2,3) `}, }, { `filters with extra spaces`, `service IN ('name > 100') AND length gt 100`, []string{`service IN ('name > 100') `, `AND length > 100 `}, }, { `Extra space within a filter expression`, `service IN ('name > 100')`, []string{`service IN ('name > 100') `}, }, { `Extra space between a query filter`, `data contains 'hello world .'`, []string{`data ILIKE '%hello world .%' `}, }, { `filters with special characters in key name`, `id.userid in (100) and id_userid gt 50`, []string{`id.userid IN (100) `, `and id_userid > 50 `}, }, { `filters with case sensitive key name`, `userIdentifier in ('user') and userIdentifier contains 'user'`, []string{`userIdentifier IN ('user') `, `AND userIdentifier ILIKE '%user%' `}, }, { `filters with for exists`, `userIdentifier exists and user nexists`, []string{`userIdentifier exists `, `and user nexists`}, }, } func TestParseLogQueryCorrect(t *testing.T) { for _, test := range correctQueriesTest { Convey(test.Name, t, func() { query, err := parseLogQuery(test.InputQuery) So(err, ShouldBeNil) So(query, ShouldResemble, test.WantSqlTokens) }) } } var incorrectQueries = []struct { Name string Query string }{ { "filter without a key", "OPERATION in ('bcd') AND 'abcd' FULLTEXT contains 'helloxyz'", }, { "fulltext without fulltext keyword", "OPERATION in ('bcd') AND 'searchstring'", }, { "fulltext in the beginning without keyword", "'searchstring and OPERATION in ('bcd')", }, } func TestParseLogQueryInCorrect(t *testing.T) { for _, test := range incorrectQueries { Convey(test.Name, t, func() { _, err := parseLogQuery(test.Query) So(err, ShouldBeError) }) } } var parseCorrectColumns = []struct { Name string Filter string Column string }{ { "column with IN operator", "id.userid IN (100) ", "id.userid", }, { "column with NOT IN operator", "service NOT IN ('name > 100') ", "service", }, { "column with > operator", "and id_userid > 50 ", "id_userid", }, { "column with < operator", "and id_userid < 50 ", "id_userid", }, { "column with <= operator", "and id_userid <= 50 ", "id_userid", }, { "column with >= operator", "and id_userid >= 50 ", "id_userid", }, { "column starting with and", "andor = 1", "andor", }, { "column starting with and after an 'and'", "and andor = 1", "andor", }, { "column starting with And", "Andor = 1", "Andor", }, { "column starting with and after an 'and'", "and Andor = 1", "Andor", }, { "column with ilike", `AND body ILIKE '%searchstring%' `, "body", }, { "column with not ilike", `AND body ILIKE '%searchstring%' `, "body", }, { "column with exists", `AND user exists`, "user", }, { "column with nexists", `AND user nexists `, "user", }, } func TestParseColumn(t *testing.T) { for _, test := range parseCorrectColumns { Convey(test.Name, t, func() { column, err := parseColumn(test.Filter) So(err, ShouldBeNil) So(*column, ShouldEqual, test.Column) }) } } func TestReplaceInterestingFields(t *testing.T) { queryTokens := []string{"id.userid IN (100) ", "and id_key >= 50 ", `AND body ILIKE '%searchstring%'`} allFields := model.GetFieldsResponse{ Selected: []model.LogField{ { Name: "id_key", DataType: "int64", Type: "attributes", }, }, Interesting: []model.LogField{ { Name: "id.userid", DataType: "int64", Type: "attributes", }, }, } expectedTokens := []string{"attributes_int64_value[indexOf(attributes_int64_key, 'id.userid')] IN (100) ", "and `attribute_int64_id_key` >= 50 ", `AND body ILIKE '%searchstring%'`} Convey("testInterestingFields", t, func() { tokens, err := replaceInterestingFields(&allFields, queryTokens) So(err, ShouldBeNil) So(tokens, ShouldResemble, expectedTokens) }) } var previousPaginateTestCases = []struct { Name string Filter model.LogsFilterParams IsPaginatePrev bool Order string }{ { Name: "empty", Filter: model.LogsFilterParams{}, IsPaginatePrev: false, }, { Name: "next ordery by asc", Filter: model.LogsFilterParams{ OrderBy: TIMESTAMP, Order: ASC, IdGt: "myid", }, IsPaginatePrev: false, Order: ASC, }, { Name: "next ordery by desc", Filter: model.LogsFilterParams{ OrderBy: TIMESTAMP, Order: DESC, IdLT: "myid", }, IsPaginatePrev: false, Order: DESC, }, { Name: "prev ordery by desc", Filter: model.LogsFilterParams{ OrderBy: TIMESTAMP, Order: DESC, IdGt: "myid", }, IsPaginatePrev: true, Order: ASC, }, { Name: "prev ordery by asc", Filter: model.LogsFilterParams{ OrderBy: TIMESTAMP, Order: ASC, IdLT: "myid", }, IsPaginatePrev: true, Order: DESC, }, } func TestCheckIfPrevousPaginateAndModifyOrder(t *testing.T) { for _, test := range previousPaginateTestCases { Convey(test.Name, t, func() { isPrevPaginate := CheckIfPrevousPaginateAndModifyOrder(&test.Filter) So(isPrevPaginate, ShouldEqual, test.IsPaginatePrev) So(test.Order, ShouldEqual, test.Filter.Order) }) } } var generateSQLQueryFields = model.GetFieldsResponse{ Selected: []model.LogField{ { Name: "field1", DataType: "int64", Type: "attributes", }, { Name: "Field2", DataType: "double64", Type: "attributes", }, { Name: "field2", DataType: "string", Type: "attributes", }, { Name: "severity_number", DataType: "string", Type: "static", }, }, Interesting: []model.LogField{ { Name: "FielD1", DataType: "int64", Type: "attributes", }, { Name: "code", DataType: "int64", Type: "attributes", }, }, } var generateSQLQueryTestCases = []struct { Name string Filter model.LogsFilterParams SqlFilter string }{ { Name: "first query with more than 1 compulsory filters", Filter: model.LogsFilterParams{ Query: "field1 lt 100 and field1 gt 50 and code lte 500 and code gte 400", TimestampStart: uint64(1657689292000), TimestampEnd: uint64(1657689294000), IdGt: "2BsKLKv8cZrLCn6rkOcRGkdjBdM", IdLT: "2BsKG6tRpFWjYMcWsAGKfSxoQdU", }, SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' and id > '2BsKLKv8cZrLCn6rkOcRGkdjBdM' and id < '2BsKG6tRpFWjYMcWsAGKfSxoQdU' ) and ( `attribute_int64_field1` < 100 and `attribute_int64_field1` > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 ) ", }, { Name: "second query with only timestamp range", Filter: model.LogsFilterParams{ Query: "field1 lt 100 and field1 gt 50 and code lte 500 and code gte 400", TimestampStart: uint64(1657689292000), TimestampEnd: uint64(1657689294000), }, SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( `attribute_int64_field1` < 100 and `attribute_int64_field1` > 50 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 ) ", }, { Name: "generate case sensitive query", Filter: model.LogsFilterParams{ Query: "field1 lt 100 and FielD1 gt 50 and Field2 gt 10 and code lte 500 and code gte 400", TimestampStart: uint64(1657689292000), TimestampEnd: uint64(1657689294000), }, SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( `attribute_int64_field1` < 100 and attributes_int64_value[indexOf(attributes_int64_key, 'FielD1')] > 50 and `attribute_double64_Field2` > 10 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] <= 500 and attributes_int64_value[indexOf(attributes_int64_key, 'code')] >= 400 ) ", }, { Name: "Check exists and not exists", Filter: model.LogsFilterParams{ Query: "field1 exists and Field2 nexists and Field2 gt 10", TimestampStart: uint64(1657689292000), TimestampEnd: uint64(1657689294000), }, SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( has(attributes_int64_key, 'field1') and NOT has(attributes_double64_key, 'Field2') and `attribute_double64_Field2` > 10 ) ", }, { Name: "Check top level key filter", Filter: model.LogsFilterParams{ Query: "severity_number in (1)", TimestampStart: uint64(1657689292000), TimestampEnd: uint64(1657689294000), }, SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( severity_number IN (1) ) ", }, { Name: "Check exists and not exists on top level keys", Filter: model.LogsFilterParams{ Query: "trace_id exists and span_id nexists and trace_flags exists and severity_number nexists", TimestampStart: uint64(1657689292000), TimestampEnd: uint64(1657689294000), }, SqlFilter: "( timestamp >= '1657689292000' and timestamp <= '1657689294000' ) and ( trace_id != '' and span_id = '' and trace_flags != 0 and severity_number = 0) ", }, } func TestGenerateSQLQuery(t *testing.T) { for _, test := range generateSQLQueryTestCases { Convey("testGenerateSQL", t, func() { res, _, err := GenerateSQLWhere(&generateSQLQueryFields, &test.Filter) So(err, ShouldBeNil) So(res, ShouldEqual, test.SqlFilter) }) } } var parseLogFilterParams = []struct { Name string ReqParams string ExpectedLogFilterParams *model.LogsFilterParams }{ { Name: "test with proper order by", ReqParams: "order=desc&q=service.name='myservice'&limit=10", ExpectedLogFilterParams: &model.LogsFilterParams{ Limit: 10, OrderBy: "timestamp", Order: DESC, Query: "service.name='myservice'", }, }, { Name: "test with proper order by asc", ReqParams: "order=asc&q=service.name='myservice'&limit=10", ExpectedLogFilterParams: &model.LogsFilterParams{ Limit: 10, OrderBy: "timestamp", Order: ASC, Query: "service.name='myservice'", }, }, { Name: "test with incorrect order by", ReqParams: "order=undefined&q=service.name='myservice'&limit=10", ExpectedLogFilterParams: &model.LogsFilterParams{ Limit: 10, OrderBy: "timestamp", Order: DESC, Query: "service.name='myservice'", }, }, } func TestParseLogFilterParams(t *testing.T) { for _, test := range parseLogFilterParams { Convey(test.Name, t, func() { req := httptest.NewRequest(http.MethodGet, "/logs?"+test.ReqParams, nil) params, err := ParseLogFilterParams(req) So(err, ShouldBeNil) So(params, ShouldEqual, test.ExpectedLogFilterParams) }) } }