logs-analyzer/signoz/pkg/query-service/app/integrations/builtin.go

package integrations

import (
	"bytes"
	"context"
	"embed"
	"strings"
	"unicode"

	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/fs"
	"path"

	koanfJson "github.com/knadh/koanf/parsers/json"
	"go.signoz.io/signoz/pkg/query-service/model"
	"golang.org/x/exp/maps"
	"golang.org/x/exp/slices"
)

type BuiltInIntegrations struct{}

var builtInIntegrations map[string]IntegrationDetails

func (bi *BuiltInIntegrations) list(ctx context.Context) (
	[]IntegrationDetails, *model.ApiError,
) {
	integrations := maps.Values(builtInIntegrations)
	slices.SortFunc(integrations, func(i1, i2 IntegrationDetails) int {
		return strings.Compare(i1.Id, i2.Id)
	})
	return integrations, nil
}

func (bi *BuiltInIntegrations) get(
	ctx context.Context, integrationIds []string,
) (
	map[string]IntegrationDetails, *model.ApiError,
) {
	result := map[string]IntegrationDetails{}
	for _, iid := range integrationIds {
		i, exists := builtInIntegrations[iid]
		if exists {
			result[iid] = i
		}
	}
	return result, nil
}

//go:embed builtin_integrations/*
var integrationFiles embed.FS

func init() {
	err := readBuiltIns()
	if err != nil {
		panic(fmt.Errorf("couldn't read builtin integrations: %w", err))
	}
}

func readBuiltIns() error {
	rootDirName := "builtin_integrations"
	builtinDirs, err := fs.ReadDir(integrationFiles, rootDirName)
	if err != nil {
		return fmt.Errorf("couldn't list integrations dirs: %w", err)
	}

	builtInIntegrations = map[string]IntegrationDetails{}
	for _, d := range builtinDirs {
		if !d.IsDir() {
			continue
		}

		integrationDir := path.Join(rootDirName, d.Name())
		i, err := readBuiltInIntegration(integrationDir)
		if err != nil {
			return fmt.Errorf("couldn't parse integration %s from files: %w", d.Name(), err)
		}

		_, exists := builtInIntegrations[i.Id]
		if exists {
			return fmt.Errorf(
				"duplicate integration for id %s at %s", i.Id, d.Name(),
			)
		}
		builtInIntegrations[i.Id] = *i
	}
	return nil
}

func readBuiltInIntegration(dirpath string) (
	*IntegrationDetails, error,
) {
	integrationJsonPath := path.Join(dirpath, "integration.json")

	serializedSpec, err := integrationFiles.ReadFile(integrationJsonPath)
	if err != nil {
		return nil, fmt.Errorf("couldn't find integration.json in %s: %w", dirpath, err)
	}

	integrationSpec, err := koanfJson.Parser().Unmarshal(serializedSpec)
	if err != nil {
		return nil, fmt.Errorf(
			"couldn't parse integration json from %s: %w", integrationJsonPath, err,
		)
	}

	hydrated, err := hydrateFileUris(integrationSpec, dirpath)
	if err != nil {
		return nil, fmt.Errorf(
			"couldn't hydrate files referenced in integration %s: %w", integrationJsonPath, err,
		)
	}

	hydratedSpec := hydrated.(map[string]interface{})
	hydratedSpecJson, err := koanfJson.Parser().Marshal(hydratedSpec)
	if err != nil {
		return nil, fmt.Errorf(
			"couldn't serialize hydrated integration spec back to JSON %s: %w", integrationJsonPath, err,
		)
	}

	var integration IntegrationDetails
	decoder := json.NewDecoder(bytes.NewReader(hydratedSpecJson))
	decoder.DisallowUnknownFields()
	err = decoder.Decode(&integration)
	if err != nil {
		return nil, fmt.Errorf(
			"couldn't parse hydrated JSON spec read from %s: %w",
			integrationJsonPath, err,
		)
	}

	err = validateIntegration(integration)
	if err != nil {
		return nil, fmt.Errorf("invalid integration spec %s: %w", integration.Id, err)
	}

	integration.Id = "builtin-" + integration.Id
	if len(integration.DataCollected.Metrics) > 0 {
		metricsForConnTest := []string{}
		for _, collectedMetric := range integration.DataCollected.Metrics {
			promName := toPromMetricName(collectedMetric.Name)
			metricsForConnTest = append(metricsForConnTest, promName)
		}
		integration.ConnectionTests.Metrics = metricsForConnTest
	}

	return &integration, nil
}

func validateIntegration(i IntegrationDetails) error {
	// Validate dashboard data
	seenDashboardIds := map[string]interface{}{}
	for _, dd := range i.Assets.Dashboards {
		did, exists := dd["id"]
		if !exists {
			return fmt.Errorf("id is required. not specified in dashboard titled %v", dd["title"])
		}
		dashboardId, ok := did.(string)
		if !ok {
			return fmt.Errorf("id must be string in dashboard titled %v", dd["title"])
		}
		if _, seen := seenDashboardIds[dashboardId]; seen {
			return fmt.Errorf("multiple dashboards found with id %s", dashboardId)
		}
		seenDashboardIds[dashboardId] = nil
	}

	// TODO(Raj): Validate all parts of plugged in integrations

	return nil
}

func hydrateFileUris(spec interface{}, basedir string) (interface{}, error) {
	if specMap, ok := spec.(map[string]interface{}); ok {
		result := map[string]interface{}{}
		for k, v := range specMap {
			hydrated, err := hydrateFileUris(v, basedir)
			if err != nil {
				return nil, err
			}
			result[k] = hydrated
		}
		return result, nil

	} else if specSlice, ok := spec.([]interface{}); ok {
		result := []interface{}{}
		for _, v := range specSlice {
			hydrated, err := hydrateFileUris(v, basedir)
			if err != nil {
				return nil, err
			}
			result = append(result, hydrated)
		}
		return result, nil

	} else if maybeFileUri, ok := spec.(string); ok {
		return readFileIfUri(maybeFileUri, basedir)
	}

	return spec, nil

}

func readFileIfUri(maybeFileUri string, basedir string) (interface{}, error) {
	fileUriPrefix := "file://"
	if !strings.HasPrefix(maybeFileUri, fileUriPrefix) {
		return maybeFileUri, nil
	}

	relativePath := maybeFileUri[len(fileUriPrefix):]
	fullPath := path.Join(basedir, relativePath)

	fileContents, err := integrationFiles.ReadFile(fullPath)
	if err != nil {
		return nil, fmt.Errorf("couldn't read referenced file: %w", err)
	}
	if strings.HasSuffix(maybeFileUri, ".md") {
		return string(fileContents), nil

	} else if strings.HasSuffix(maybeFileUri, ".json") {
		parsed, err := koanfJson.Parser().Unmarshal(fileContents)
		if err != nil {
			return nil, fmt.Errorf("couldn't parse referenced JSON file: %w", err)
		}
		return parsed, nil

	} else if strings.HasSuffix(maybeFileUri, ".svg") {
		base64Svg := base64.StdEncoding.EncodeToString(fileContents)
		dataUri := fmt.Sprintf("data:image/svg+xml;base64,%s", base64Svg)
		return dataUri, nil

	}

	return nil, fmt.Errorf("unsupported file type %s", maybeFileUri)
}

// copied from signoz clickhouse exporter's `sanitize` which
// in turn is copied from prometheus-go-metric-exporter
//
// replaces non-alphanumeric characters with underscores in s.
func toPromMetricName(s string) string {
	if len(s) == 0 {
		return s
	}

	// Note: No length limit for label keys because Prometheus doesn't
	// define a length limit, thus we should NOT be truncating label keys.
	// See https://github.com/orijtech/prometheus-go-metrics-exporter/issues/4.

	s = strings.Map(func(r rune) rune {
		// sanitizeRune converts anything that is not a letter or digit to an underscore
		if unicode.IsLetter(r) || unicode.IsDigit(r) {
			return r
		}
		// Everything else turns into an underscore
		return '_'
	}, s)

	if unicode.IsDigit(rune(s[0])) {
		s = "key" + "_" + s
	}
	if s[0] == '_' {
		s = "key" + s
	}
	return s
}