chore: add sql migration for dashboards, alerts, and saved views (#8642)

## 📄 Summary

To reliably migrate the alerts and dashboards, we need access to the telemetrystore to fetch some metadata and while doing migration, I need to log some stuff to fix stuff later.

Key changes:
- Modified the migration to include telemetrystore and a logging provider (open to using a standard logger instead)
- To avoid the previous issues with imported dashboards failing during migration, I've ensured that imported JSON files are automatically transformed when migration is active
- Implemented detailed logic to handle dashboard migration cleanly and prevent unnecessary errors
- Separated the core migration logic from SQL migration code, as users from the dot metrics migration requested shareable code snippets for local migrations. This modular approach allows others to easily reuse the migration functionality.

Known: I didn't register the migration yet in this PR, and will not merge this yet, so please review with that in mid.

Author: srikanthccv

go.mod

@@ -70,6 +70,7 @@ require (
 	go.uber.org/zap v1.27.0
 	golang.org/x/crypto v0.39.0
 	golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac
+	golang.org/x/net v0.41.0
 	golang.org/x/oauth2 v0.30.0
 	golang.org/x/sync v0.15.0
 	golang.org/x/text v0.26.0
@@ -283,7 +284,6 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/mod v0.25.0 // indirect
-	golang.org/x/net v0.41.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/time v0.11.0 // indirect
 	golang.org/x/tools v0.33.0 // indirect

pkg/apis/fields/api.go

@@ -63,15 +63,15 @@ func (api *API) GetFieldsKeys(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	keys, err := api.telemetryMetadataStore.GetKeys(ctx, fieldKeySelector)
+	keys, complete, err := api.telemetryMetadataStore.GetKeys(ctx, fieldKeySelector)
 	if err != nil {
 		render.Error(w, err)
 		return
 	}
 
 	response := fieldKeysResponse{
 		Keys:     keys,
-		Complete: len(keys) < fieldKeySelector.Limit,
+		Complete: complete,
 	}
 
 	render.Success(w, http.StatusOK, response)
@@ -94,13 +94,13 @@ func (api *API) GetFieldsValues(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	allValues, err := api.telemetryMetadataStore.GetAllValues(ctx, fieldValueSelector)
+	allValues, allComplete, err := api.telemetryMetadataStore.GetAllValues(ctx, fieldValueSelector)
 	if err != nil {
 		render.Error(w, err)
 		return
 	}
 
-	relatedValues, err := api.telemetryMetadataStore.GetRelatedValues(ctx, fieldValueSelector)
+	relatedValues, relatedComplete, err := api.telemetryMetadataStore.GetRelatedValues(ctx, fieldValueSelector)
 	if err != nil {
 		// we don't want to return error if we fail to get related values for some reason
 		relatedValues = []string{}
@@ -113,11 +113,8 @@ func (api *API) GetFieldsValues(w http.ResponseWriter, r *http.Request) {
 	}
 
 	response := fieldValuesResponse{
-		Values: values,
-		Complete: len(values.StringValues) < fieldValueSelector.Limit &&
-			len(values.BoolValues) < fieldValueSelector.Limit &&
-			len(values.NumberValues) < fieldValueSelector.Limit &&
-			len(values.RelatedValues) < fieldValueSelector.Limit,
+		Values:   values,
+		Complete: allComplete && relatedComplete,
 	}
 
 	render.Success(w, http.StatusOK, response)

pkg/modules/dashboard/impldashboard/handler.go

@@ -7,20 +7,24 @@ import (
 	"time"
 
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/http/render"
 	"github.com/SigNoz/signoz/pkg/modules/dashboard"
+	"github.com/SigNoz/signoz/pkg/querybuilder"
+	"github.com/SigNoz/signoz/pkg/transition"
 	"github.com/SigNoz/signoz/pkg/types/authtypes"
 	"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
 	"github.com/SigNoz/signoz/pkg/valuer"
 	"github.com/gorilla/mux"
 )
 
 type handler struct {
-	module dashboard.Module
+	module           dashboard.Module
+	providerSettings factory.ProviderSettings
 }
 
-func NewHandler(module dashboard.Module) dashboard.Handler {
-	return &handler{module: module}
+func NewHandler(module dashboard.Module, providerSettings factory.ProviderSettings) dashboard.Handler {
+	return &handler{module: module, providerSettings: providerSettings}
 }
 
 func (handler *handler) Create(rw http.ResponseWriter, r *http.Request) {
@@ -46,6 +50,13 @@ func (handler *handler) Create(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if querybuilder.QBV5Enabled {
+		dashboardMigrator := transition.NewDashboardMigrateV5(handler.providerSettings.Logger, nil, nil)
+		if req["version"] != "v5" {
+			dashboardMigrator.Migrate(ctx, req)
+		}
+	}
+
 	dashboard, err := handler.module.Create(ctx, orgID, claims.Email, valuer.MustNewUUID(claims.UserID), req)
 	if err != nil {
 		render.Error(rw, err)

pkg/querier/querier.go

@@ -387,7 +387,7 @@ func (q *querier) run(
 			}
 		}
 
-		resp.Warning = qbtypes.QueryWarnData{
+		resp.Warning = &qbtypes.QueryWarnData{
 			Message:  "Encountered warnings",
 			Url:      warningsDocURL,
 			Warnings: warns,
@@ -501,23 +501,36 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
 
 // createRangedQuery creates a copy of the query with a different time range
 func (q *querier) createRangedQuery(originalQuery qbtypes.Query, timeRange qbtypes.TimeRange) qbtypes.Query {
+	// this is called in a goroutine, so we create a copy of the query to avoid race conditions
 	switch qt := originalQuery.(type) {
 	case *promqlQuery:
-		return newPromqlQuery(q.logger, q.promEngine, qt.query, timeRange, qt.requestType, qt.vars)
+		queryCopy := qt.query.Copy()
+		return newPromqlQuery(q.logger, q.promEngine, queryCopy, timeRange, qt.requestType, qt.vars)
+
 	case *chSQLQuery:
-		return newchSQLQuery(q.logger, q.telemetryStore, qt.query, qt.args, timeRange, qt.kind, qt.vars)
+		queryCopy := qt.query.Copy()
+		argsCopy := make([]any, len(qt.args))
+		copy(argsCopy, qt.args)
+		return newchSQLQuery(q.logger, q.telemetryStore, queryCopy, argsCopy, timeRange, qt.kind, qt.vars)
+
 	case *builderQuery[qbtypes.TraceAggregation]:
-		qt.spec.ShiftBy = extractShiftFromBuilderQuery(qt.spec)
-		adjustedTimeRange := adjustTimeRangeForShift(qt.spec, timeRange, qt.kind)
-		return newBuilderQuery(q.telemetryStore, q.traceStmtBuilder, qt.spec, adjustedTimeRange, qt.kind, qt.variables)
+		specCopy := qt.spec.Copy()
+		specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
+		adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
+		return newBuilderQuery(q.telemetryStore, q.traceStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
+
 	case *builderQuery[qbtypes.LogAggregation]:
-		qt.spec.ShiftBy = extractShiftFromBuilderQuery(qt.spec)
-		adjustedTimeRange := adjustTimeRangeForShift(qt.spec, timeRange, qt.kind)
-		return newBuilderQuery(q.telemetryStore, q.logStmtBuilder, qt.spec, adjustedTimeRange, qt.kind, qt.variables)
+		specCopy := qt.spec.Copy()
+		specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
+		adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
+		return newBuilderQuery(q.telemetryStore, q.logStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
+
 	case *builderQuery[qbtypes.MetricAggregation]:
-		qt.spec.ShiftBy = extractShiftFromBuilderQuery(qt.spec)
-		adjustedTimeRange := adjustTimeRangeForShift(qt.spec, timeRange, qt.kind)
-		return newBuilderQuery(q.telemetryStore, q.metricStmtBuilder, qt.spec, adjustedTimeRange, qt.kind, qt.variables)
+		specCopy := qt.spec.Copy()
+		specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
+		adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
+		return newBuilderQuery(q.telemetryStore, q.metricStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
+
 	default:
 		return nil
 	}

pkg/query-service/rules/threshold_rule.go

@@ -543,6 +543,11 @@ func (r *ThresholdRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUI
 		return resultVector, nil
 	}
 
+	if queryResult == nil {
+		r.logger.WarnContext(ctx, "query result is nil", "rule_name", r.Name(), "query_name", selectedQuery)
+		return resultVector, nil
+	}
+
 	for _, series := range queryResult.Series {
 		smpl, shouldAlert := r.ShouldAlert(*series)
 		if shouldAlert {

pkg/query-service/tests/integration/filter_suggestions_test.go

@@ -318,7 +318,7 @@ func NewFilterSuggestionsTestBed(t *testing.T) *FilterSuggestionsTestBed {
 	emailing := emailingtest.New()
 	analytics := analyticstest.New()
 	modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics)
-	handlers := signoz.NewHandlers(modules)
+	handlers := signoz.NewHandlers(modules, providerSettings)
 
 	apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{
 		Reader: reader,

pkg/query-service/tests/integration/logparsingpipeline_test.go

@@ -498,7 +498,7 @@ func NewTestbedWithoutOpamp(t *testing.T, sqlStore sqlstore.SQLStore) *LogPipeli
 	emailing := emailingtest.New()
 	analytics := analyticstest.New()
 	modules := signoz.NewModules(sqlStore, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics)
-	handlers := signoz.NewHandlers(modules)
+	handlers := signoz.NewHandlers(modules, providerSettings)
 
 	apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{
 		LogsParsingPipelineController: controller,

pkg/query-service/tests/integration/signoz_cloud_integrations_test.go

@@ -379,7 +379,7 @@ func NewCloudIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *CloudI
 	emailing := emailingtest.New()
 	analytics := analyticstest.New()
 	modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics)
-	handlers := signoz.NewHandlers(modules)
+	handlers := signoz.NewHandlers(modules, providerSettings)
 
 	apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{
 		Reader:                      reader,

pkg/query-service/tests/integration/signoz_integrations_test.go

@@ -594,7 +594,7 @@ func NewIntegrationsTestBed(t *testing.T, testDB sqlstore.SQLStore) *Integration
 	emailing := emailingtest.New()
 	analytics := analyticstest.New()
 	modules := signoz.NewModules(testDB, jwt, emailing, providerSettings, orgGetter, alertmanager, analytics)
-	handlers := signoz.NewHandlers(modules)
+	handlers := signoz.NewHandlers(modules, providerSettings)
 
 	apiHandler, err := app.NewAPIHandler(app.APIHandlerOpts{
 		Reader:                      reader,

pkg/querybuilder/init.go

@@ -0,0 +1,15 @@
+package querybuilder
+
+import (
+	"os"
+	"strings"
+)
+
+var QBV5Enabled = false
+
+func init() {
+	v := os.Getenv("ENABLE_QB_V5")
+	if strings.ToLower(v) == "true" || strings.ToLower(v) == "1" {
+		QBV5Enabled = true
+	}
+}