refactor: restructure backend code organization

Makefile

@@ -49,7 +49,7 @@ generate: ## Run go generate
 	@go generate ./...
 
 config: ## Generate sample config file
-	@cp config/config.yaml config.yaml
+	@cp internal/config/config.example.yaml config.yaml
 
 proto: ## Generate protobuf files
 	@rm -rf proto/

buf.gen.yaml

@@ -1,17 +1,22 @@
 version: v2
+# Proto source lives in the proton repo (../proton).
+# After running `buf generate`, remove unwanted generated files:
+#   find gen/raystack -mindepth 1 -maxdepth 1 ! -name compass -exec rm -rf {} +
+inputs:
+  - directory: ../proton
+  - module: buf.build/bufbuild/protovalidate
 managed:
   enabled: true
   override:
     - file_option: go_package
       path: raystack/compass/v1beta1/service.proto
-      value: github.com/raystack/compass/proto/compassv1beta1
+      value: github.com/raystack/compass/gen/raystack/compass/v1beta1;compassv1beta1
 plugins:
   - remote: buf.build/protocolbuffers/go:v1.36.11
-    out: proto
-    opt: module=github.com/raystack/compass/proto
-  - remote: buf.build/bufbuild/validate-go:v1.0.2
-    out: proto
-    opt: module=github.com/raystack/compass/proto
+    out: gen
+    opt:
+      - paths=source_relative
   - remote: buf.build/connectrpc/go:v1.18.1
-    out: proto
-    opt: module=github.com/raystack/compass/proto
+    out: gen
+    opt:
+      - paths=source_relative

cli/assets.go

@@ -7,7 +7,7 @@ import (
 	"github.com/MakeNowJust/heredoc"
 	"github.com/raystack/compass/core/namespace"
 	"github.com/raystack/compass/internal/client"
-	compassv1beta1 "github.com/raystack/compass/proto/compassv1beta1"
+	compassv1beta1 "github.com/raystack/compass/gen/raystack/compass/v1beta1"
 	"github.com/raystack/salt/cli/printer"
 	"github.com/spf13/cobra"
 )
@@ -187,11 +187,6 @@ func editAssetCommand(cfg *Config) *cobra.Command {
 				return err
 			}
 
-			err := reqBody.ValidateAll()
-			if err != nil {
-				return err
-			}
-
 			clnt, err := client.Create(cmd.Context(), cfg.Client)
 			if err != nil {
 				return err

cli/config.go

@@ -6,11 +6,10 @@ import (
 
 	"github.com/MakeNowJust/heredoc"
 	"github.com/raystack/compass/internal/client"
-	"github.com/raystack/compass/internal/server"
-	esStore "github.com/raystack/compass/internal/store/elasticsearch"
-	"github.com/raystack/compass/internal/store/postgres"
-	"github.com/raystack/compass/pkg/metrics"
-	"github.com/raystack/compass/pkg/telemetry"
+	compassconfig "github.com/raystack/compass/internal/config"
+	esStore "github.com/raystack/compass/store/elasticsearch"
+	"github.com/raystack/compass/store/postgres"
+	"github.com/raystack/compass/internal/telemetry"
 	"github.com/raystack/salt/config"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
@@ -77,9 +76,6 @@ type Config struct {
 	// Log
 	LogLevel string `yaml:"log_level" mapstructure:"log_level" default:"info"`
 
-	// NewRelic
-	NewRelic metrics.NewRelicConfig `mapstructure:"newrelic"`
-
 	// Telemetry
 	Telemetry telemetry.Config `mapstructure:"telemetry"`
 
@@ -90,7 +86,7 @@ type Config struct {
 	DB postgres.Config `mapstructure:"db"`
 
 	// Service
-	Service server.Config `mapstructure:"service"`
+	Service compassconfig.ServerConfig `mapstructure:"service"`
 
 	// Client
 	Client client.Config `mapstructure:"client"`

cli/discussions.go

@@ -7,7 +7,7 @@ import (
 	"github.com/MakeNowJust/heredoc"
 	"github.com/raystack/compass/core/namespace"
 	"github.com/raystack/compass/internal/client"
-	compassv1beta1 "github.com/raystack/compass/proto/compassv1beta1"
+	compassv1beta1 "github.com/raystack/compass/gen/raystack/compass/v1beta1"
 	"github.com/raystack/salt/cli/printer"
 	"github.com/spf13/cobra"
 )
@@ -143,11 +143,6 @@ func postDiscussionCommand(cfg *Config) *cobra.Command {
 			if err := parseFile(filePath, &reqBody); err != nil {
 				return err
 			}
-			err := reqBody.ValidateAll()
-			if err != nil {
-				return err
-			}
-
 			clnt, err := client.Create(cmd.Context(), cfg.Client)
 			if err != nil {
 				return err

cli/lineage.go

@@ -6,7 +6,7 @@ import (
 	"github.com/MakeNowJust/heredoc"
 	"github.com/raystack/compass/core/namespace"
 	"github.com/raystack/compass/internal/client"
-	compassv1beta1 "github.com/raystack/compass/proto/compassv1beta1"
+	compassv1beta1 "github.com/raystack/compass/gen/raystack/compass/v1beta1"
 	"github.com/raystack/salt/cli/printer"
 	"github.com/spf13/cobra"
 )

cli/namespace.go

@@ -9,7 +9,7 @@ import (
 	"github.com/MakeNowJust/heredoc"
 	"github.com/raystack/compass/core/namespace"
 	"github.com/raystack/compass/internal/client"
-	compassv1beta1 "github.com/raystack/compass/proto/compassv1beta1"
+	compassv1beta1 "github.com/raystack/compass/gen/raystack/compass/v1beta1"
 	"github.com/raystack/salt/cli/printer"
 	"github.com/spf13/cobra"
 )

cli/search.go

@@ -6,7 +6,7 @@ import (
 	"github.com/MakeNowJust/heredoc"
 	"github.com/raystack/compass/core/namespace"
 	"github.com/raystack/compass/internal/client"
-	compassv1beta1 "github.com/raystack/compass/proto/compassv1beta1"
+	compassv1beta1 "github.com/raystack/compass/gen/raystack/compass/v1beta1"
 	"github.com/raystack/salt/cli/printer"
 	"github.com/spf13/cobra"
 )

cli/server.go

@@ -10,16 +10,15 @@ import (
 	"syscall"
 
 	"github.com/MakeNowJust/heredoc"
-	"github.com/newrelic/go-agent/v3/newrelic"
 	"github.com/raystack/compass/core/asset"
 	"github.com/raystack/compass/core/discussion"
 	"github.com/raystack/compass/core/star"
 	"github.com/raystack/compass/core/tag"
 	"github.com/raystack/compass/core/user"
 	compassserver "github.com/raystack/compass/internal/server"
-	esStore "github.com/raystack/compass/internal/store/elasticsearch"
-	"github.com/raystack/compass/internal/store/postgres"
-	"github.com/raystack/compass/pkg/telemetry"
+	esStore "github.com/raystack/compass/store/elasticsearch"
+	"github.com/raystack/compass/store/postgres"
+	"github.com/raystack/compass/internal/telemetry"
 	log "github.com/raystack/salt/observability/logger"
 	"github.com/spf13/cobra"
 )
@@ -81,7 +80,7 @@ func serverMigrateCommand(cfg *Config) *cobra.Command {
 			ctx, cancel := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
 			defer cancel()
 			if down {
-				return migrateDown(cfg)
+				return migrateDown(ctx, cfg)
 			}
 			return runMigrations(ctx, cfg)
 		},
@@ -97,11 +96,6 @@ func runServer(config *Config) error {
 	logger := initLogger(config.LogLevel)
 	logger.Info("compass starting", "version", Version)
 
-	nrApp, err := initNewRelicMonitor(config, logger)
-	if err != nil {
-		return err
-	}
-
 	otelCleanup, err := telemetry.Init(ctx, config.Telemetry, logger)
 	if err != nil {
 		return fmt.Errorf("failed to initialize telemetry: %w", err)
@@ -117,6 +111,13 @@ func runServer(config *Config) error {
 	if err != nil {
 		return err
 	}
+	defer func() {
+		logger.Warn("closing db...")
+		if err := pgClient.Close(); err != nil {
+			logger.Error("error when closing db", "err", err)
+		}
+		logger.Warn("db closed...")
+	}()
 
 	// init tag
 	tagRepository, err := postgres.NewTagRepository(pgClient)
@@ -169,8 +170,6 @@ func runServer(config *Config) error {
 		ctx,
 		config.Service,
 		logger,
-		pgClient,
-		nrApp,
 		namespaceService,
 		assetService,
 		starService,
@@ -212,41 +211,20 @@ func initPostgres(logger log.Logger, config *Config) (*postgres.Client, error) {
 	return pgClient, nil
 }
 
-func initNewRelicMonitor(config *Config, logger log.Logger) (*newrelic.Application, error) {
-	if !config.NewRelic.Enabled {
-		logger.Info("New Relic monitoring is disabled.")
-		return nil, nil
-	}
-	app, err := newrelic.NewApplication(
-		newrelic.ConfigAppName(config.NewRelic.AppName),
-		newrelic.ConfigLicense(config.NewRelic.LicenseKey),
-	)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create New Relic Application: %w", err)
-	}
-	logger.Info("New Relic monitoring is enabled for", "config", config.NewRelic.AppName)
-
-	return app, nil
-}
-
 func runMigrations(ctx context.Context, config *Config) error {
-	fmt.Println("Preparing migration...")
-
 	logger := initLogger(config.LogLevel)
 	logger.Info("compass is migrating", "version", Version)
 
-	logger.Info("Migrating Postgres & ElasticSearch...")
 	esClient, err := initElasticsearch(logger, config.Elasticsearch)
 	if err != nil {
 		return err
 	}
 
-	logger.Info("Initiating Postgres client...")
-	pgClient, err := postgres.NewClient(config.DB)
+	pgClient, err := initPostgres(logger, config)
 	if err != nil {
-		logger.Error("failed to prepare migration", "error", err)
 		return err
 	}
+	defer pgClient.Close()
 
 	ver, err := pgClient.Migrate(config.DB)
 	if err != nil {
@@ -265,27 +243,24 @@ func runMigrations(ctx context.Context, config *Config) error {
 	} else if err != nil {
 		return fmt.Errorf("problem with migration %w", err)
 	}
-	logger.Info(fmt.Sprintf("Migration finished. Version: %d", ver))
+	logger.Info("migration finished", "version", ver)
 	return nil
 }
 
-func migrateDown(config *Config) error {
-	fmt.Println("Preparing rolling back one step of migration...")
-
+func migrateDown(ctx context.Context, config *Config) error {
 	logger := initLogger(config.LogLevel)
-	logger.Info("compass is migrating", "version", Version)
+	logger.Info("compass is rolling back migration", "version", Version)
 
-	logger.Info("Initiating Postgres client...")
-	pgClient, err := postgres.NewClient(config.DB)
+	pgClient, err := initPostgres(logger, config)
 	if err != nil {
-		logger.Error("failed to prepare migration", "error", err)
 		return err
 	}
+	defer pgClient.Close()
 
 	ver, err := pgClient.MigrateDown(config.DB)
 	if err != nil {
 		return fmt.Errorf("problem with migration %w", err)
 	}
-	logger.Info(fmt.Sprintf("Migration finished. Version: %d", ver))
+	logger.Info("migration finished", "version", ver)
 	return nil
 }

docs/docs/configuration.md

@@ -237,17 +237,11 @@ app:
   config:
     COMPASS_SERVICE_PORT: 8080
     # COMPASS_SERVICE_HOST: 0.0.0.0
-    # COMPASS_STATSD_ENABLED: false
-    # COMPASS_STATSD_PREFIX: compass
-    # COMPASS_NEWRELIC_ENABLED: false
-    # COMPASS_NEWRELIC_APPNAME: compass
     # COMPASS_LOG_LEVEL: info
 
   secretConfig:
     {}
     # COMPASS_ELASTICSEARCH_BROKERS: ~
-    # COMPASS_STATSD_ADDRESS: ~
-    # COMPASS_NEWRELIC_LICENSEKEY: ~
     # COMPASS_DB_HOST: ~
     # COMPASS_DB_PORT: 5432
     # COMPASS_DB_NAME: ~
@@ -284,8 +278,6 @@ If everything goes ok, you should see something like this:
 ```bash
 time="2022-04-27T09:18:08Z" level=info msg="compass starting" version=v0.2.0
 time="2022-04-27T09:18:08Z" level=info msg="connected to elasticsearch cluster" config="\"docker-cluster\" (server version 7.6.1)"
-time="2022-04-27T09:18:08Z" level=info msg="New Relic monitoring is disabled."
-time="2022-04-27T09:18:08Z" level=info msg="statsd metrics monitoring is disabled."
 time="2022-04-27T09:18:08Z" level=info msg="connected to postgres server" host=postgres port=5432
 time="2022-04-27T09:18:08Z" level=info msg="server started"
 ```

docs/docs/guides/telemetry.md

@@ -1,23 +1,19 @@
 # Telemetry
 
-Compass collects basic HTTP metrics (response time, duration, etc) and sends it to [statsd](https://github.com/statsd/statsd) and [New Relic](https://newrelic.com/) when enabled.
+Compass uses [OpenTelemetry](https://opentelemetry.io/) to collect traces and metrics and export them via OTLP gRPC to a collector.
 
-## Statsd
-By default statsd is not enabled. To enable statsd, we just need to set these configurations below
+## OpenTelemetry
+
+By default OpenTelemetry is not enabled. To enable it, set these configurations:
 
 ```
-STATSD_ENABLED=true
-STATSD_ADDRESS=127.0.0.1:8125
-STATSD_PREFIX=compass
+TELEMETRY_OPENTELEMETRY_ENABLED=true
+TELEMETRY_OPENTELEMETRY_COLLECTORADDR=localhost:4317
+TELEMETRY_SERVICENAME=compass
 ```
 
-
-## New Relic
-Similar with statsd, New Relic is not enabled by default. To enable New Relic, you can set these configurations
+You can optionally configure trace sampling probability:
 
 ```
-NEW_RELIC_LICENSE_KEY=mf9d13c838u252252c43ji47q1u4ynzpDDDDTSPQ
-NEW_RELIC_APP_NAME=compass
+TELEMETRY_OPENTELEMETRY_TRACESAMPLEPROBABILITY=0.1
 ```
-
-Empty `NEW_RELIC_LICENSE_KEY` will disable New Relic integration.