Skip to content

Commit fd44f2e

Browse files
leonorfmartinsleonorfmartins
authored
Init dualwriter metrics (grafana#89003)
* Pass prometheus registerer to the dual writer * Fix tests * Remove unused var * Fix tests * Uncomment test * Remove leading line * Fix tests. Reuse registerer if there's already one * Lint * Improve double registering logic * Rebase main
1 parent e3da5ed commit fd44f2e

File tree

28 files changed

+114
-60
lines changed

28 files changed

+114
-60
lines changed

‎pkg/apiserver/builder/common.go

+2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"net/http"
55

66
grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
7+
"github.com/prometheus/client_golang/prometheus"
78
"k8s.io/apimachinery/pkg/runtime"
89
"k8s.io/apimachinery/pkg/runtime/schema"
910
"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -29,6 +30,7 @@ type APIGroupBuilder interface {
2930
codecs serializer.CodecFactory,
3031
optsGetter generic.RESTOptionsGetter,
3132
desiredMode grafanarest.DualWriterMode,
33+
reg prometheus.Registerer,
3234
) (*genericapiserver.APIGroupInfo, error)
3335

3436
// Get OpenAPI definitions

‎pkg/apiserver/builder/helper.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525

2626
"github.com/grafana/grafana/pkg/apiserver/endpoints/filters"
2727
"github.com/grafana/grafana/pkg/services/apiserver/options"
28+
"github.com/prometheus/client_golang/prometheus"
2829
)
2930

3031
// TODO: this is a temporary hack to make rest.Connecter work with resource level routes
@@ -128,6 +129,7 @@ func InstallAPIs(
128129
optsGetter generic.RESTOptionsGetter,
129130
builders []APIGroupBuilder,
130131
storageOpts *options.StorageOptions,
132+
reg prometheus.Registerer,
131133
) error {
132134
// dual writing is only enabled when the storage type is not legacy.
133135
// this is needed to support setting a default RESTOptionsGetter for new APIs that don't
@@ -136,7 +138,7 @@ func InstallAPIs(
136138

137139
for _, b := range builders {
138140
mode := b.GetDesiredDualWriterMode(dualWriteEnabled, storageOpts.DualWriterDesiredModes)
139-
g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, mode)
141+
g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, mode, reg)
140142
if err != nil {
141143
return err
142144
}

‎pkg/apiserver/rest/dualwriter.go

+12-7
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"fmt"
77

88
"github.com/grafana/grafana/pkg/infra/kvstore"
9+
"github.com/prometheus/client_golang/prometheus"
910
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1011
"k8s.io/apimachinery/pkg/runtime"
1112
"k8s.io/apiserver/pkg/registry/rest"
@@ -95,25 +96,28 @@ const (
9596
Mode4
9697
)
9798

99+
// TODO: make this function private as there should only be one public way of setting the dual writing mode
98100
// NewDualWriter returns a new DualWriter.
99-
func NewDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage) DualWriter {
101+
func NewDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage, reg prometheus.Registerer) DualWriter {
102+
metrics := &dualWriterMetrics{}
103+
metrics.init(reg)
100104
switch mode {
101105
// It is not possible to initialize a mode 0 dual writer. Mode 0 represents
102106
// writing to legacy storage without `unifiedStorage` enabled.
103107
case Mode1:
104108
// read and write only from legacy storage
105-
return newDualWriterMode1(legacy, storage)
109+
return newDualWriterMode1(legacy, storage, metrics)
106110
case Mode2:
107111
// write to both, read from storage but use legacy as backup
108-
return newDualWriterMode2(legacy, storage)
112+
return newDualWriterMode2(legacy, storage, metrics)
109113
case Mode3:
110114
// write to both, read from storage only
111-
return newDualWriterMode3(legacy, storage)
115+
return newDualWriterMode3(legacy, storage, metrics)
112116
case Mode4:
113117
// read and write only from storage
114-
return newDualWriterMode4(legacy, storage)
118+
return newDualWriterMode4(legacy, storage, metrics)
115119
default:
116-
return newDualWriterMode1(legacy, storage)
120+
return newDualWriterMode1(legacy, storage, metrics)
117121
}
118122
}
119123

@@ -142,6 +146,7 @@ func SetDualWritingMode(
142146
storage Storage,
143147
entity string,
144148
desiredMode DualWriterMode,
149+
reg prometheus.Registerer,
145150
) (DualWriter, error) {
146151
toMode := map[string]DualWriterMode{
147152
// It is not possible to initialize a mode 0 dual writer. Mode 0 represents
@@ -200,5 +205,5 @@ func SetDualWritingMode(
200205

201206
// #TODO add support for other combinations of desired and current modes
202207

203-
return NewDualWriter(currentMode, legacy, storage), nil
208+
return NewDualWriter(currentMode, legacy, storage, reg), nil
204209
}

‎pkg/apiserver/rest/dualwriter_mode1.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,8 @@ const mode1Str = "1"
2424

2525
// NewDualWriterMode1 returns a new DualWriter in mode 1.
2626
// Mode 1 represents writing to and reading from LegacyStorage.
27-
func newDualWriterMode1(legacy LegacyStorage, storage Storage) *DualWriterMode1 {
28-
metrics := &dualWriterMetrics{}
29-
metrics.init()
30-
return &DualWriterMode1{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode1"), dualWriterMetrics: metrics}
27+
func newDualWriterMode1(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode1 {
28+
return &DualWriterMode1{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode1"), dualWriterMetrics: dwm}
3129
}
3230

3331
// Mode returns the mode of the dual writer.

‎pkg/apiserver/rest/dualwriter_mode1_test.go

+10-6
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"errors"
66
"testing"
77

8+
"github.com/prometheus/client_golang/prometheus"
89
"github.com/stretchr/testify/assert"
910
"github.com/stretchr/testify/mock"
1011
"k8s.io/apimachinery/pkg/api/meta"
@@ -22,6 +23,8 @@ var failingObj = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta
2223
var exampleList = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, Items: []example.Pod{*exampleObj}}
2324
var anotherList = &example.PodList{Items: []example.Pod{*anotherObj}}
2425

26+
var p = prometheus.NewRegistry()
27+
2528
func TestMode1_Create(t *testing.T) {
2629
type testCase struct {
2730
input runtime.Object
@@ -68,7 +71,7 @@ func TestMode1_Create(t *testing.T) {
6871
tt.setupStorageFn(m)
6972
}
7073

71-
dw := NewDualWriter(Mode1, ls, us)
74+
dw := NewDualWriter(Mode1, ls, us, p)
7275

7376
obj, err := dw.Create(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.CreateOptions{})
7477

@@ -131,7 +134,8 @@ func TestMode1_Get(t *testing.T) {
131134
tt.setupStorageFn(m, tt.input)
132135
}
133136

134-
dw := NewDualWriter(Mode1, ls, us)
137+
p := prometheus.NewRegistry()
138+
dw := NewDualWriter(Mode1, ls, us, p)
135139

136140
obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{})
137141

@@ -182,7 +186,7 @@ func TestMode1_List(t *testing.T) {
182186
tt.setupStorageFn(m)
183187
}
184188

185-
dw := NewDualWriter(Mode1, ls, us)
189+
dw := NewDualWriter(Mode1, ls, us, p)
186190

187191
_, err := dw.List(context.Background(), &metainternalversion.ListOptions{})
188192

@@ -237,7 +241,7 @@ func TestMode1_Delete(t *testing.T) {
237241
tt.setupStorageFn(m, tt.input)
238242
}
239243

240-
dw := NewDualWriter(Mode1, ls, us)
244+
dw := NewDualWriter(Mode1, ls, us, p)
241245

242246
obj, _, err := dw.Delete(context.Background(), tt.input, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{})
243247

@@ -296,7 +300,7 @@ func TestMode1_DeleteCollection(t *testing.T) {
296300
tt.setupStorageFn(m, tt.input)
297301
}
298302

299-
dw := NewDualWriter(Mode1, ls, us)
303+
dw := NewDualWriter(Mode1, ls, us, p)
300304

301305
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, tt.input, &metainternalversion.ListOptions{})
302306

@@ -372,7 +376,7 @@ func TestMode1_Update(t *testing.T) {
372376
tt.setupGetFn(m, tt.input)
373377
}
374378

375-
dw := NewDualWriter(Mode1, ls, us)
379+
dw := NewDualWriter(Mode1, ls, us, p)
376380

377381
obj, _, err := dw.Update(context.Background(), tt.input, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
378382

‎pkg/apiserver/rest/dualwriter_mode2.go

+3-6
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"time"
66

7+
"github.com/grafana/grafana/pkg/apimachinery/utils"
78
apierrors "k8s.io/apimachinery/pkg/api/errors"
89
"k8s.io/apimachinery/pkg/api/meta"
910
metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
@@ -13,8 +14,6 @@ import (
1314
"k8s.io/apimachinery/pkg/selection"
1415
"k8s.io/apiserver/pkg/registry/rest"
1516
"k8s.io/klog/v2"
16-
17-
"github.com/grafana/grafana/pkg/apimachinery/utils"
1817
)
1918

2019
type DualWriterMode2 struct {
@@ -28,10 +27,8 @@ const mode2Str = "2"
2827

2928
// NewDualWriterMode2 returns a new DualWriter in mode 2.
3029
// Mode 2 represents writing to LegacyStorage and Storage and reading from LegacyStorage.
31-
func newDualWriterMode2(legacy LegacyStorage, storage Storage) *DualWriterMode2 {
32-
metrics := &dualWriterMetrics{}
33-
metrics.init()
34-
return &DualWriterMode2{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode2"), dualWriterMetrics: metrics}
30+
func newDualWriterMode2(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode2 {
31+
return &DualWriterMode2{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode2"), dualWriterMetrics: dwm}
3532
}
3633

3734
// Mode returns the mode of the dual writer.

‎pkg/apiserver/rest/dualwriter_mode2_test.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func TestMode2_Create(t *testing.T) {
6767
tt.setupStorageFn(m, tt.input)
6868
}
6969

70-
dw := NewDualWriter(Mode2, ls, us)
70+
dw := NewDualWriter(Mode2, ls, us, p)
7171

7272
obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{})
7373

@@ -143,7 +143,7 @@ func TestMode2_Get(t *testing.T) {
143143
tt.setupStorageFn(m, tt.input)
144144
}
145145

146-
dw := NewDualWriter(Mode2, ls, us)
146+
dw := NewDualWriter(Mode2, ls, us, p)
147147

148148
obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{})
149149

@@ -196,7 +196,7 @@ func TestMode2_List(t *testing.T) {
196196
tt.setupStorageFn(m)
197197
}
198198

199-
dw := NewDualWriter(Mode2, ls, us)
199+
dw := NewDualWriter(Mode2, ls, us, p)
200200

201201
obj, err := dw.List(context.Background(), &metainternalversion.ListOptions{})
202202

@@ -289,7 +289,7 @@ func TestMode2_Delete(t *testing.T) {
289289
tt.setupStorageFn(m, tt.input)
290290
}
291291

292-
dw := NewDualWriter(Mode2, ls, us)
292+
dw := NewDualWriter(Mode2, ls, us, p)
293293

294294
obj, _, err := dw.Delete(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{})
295295

@@ -361,7 +361,7 @@ func TestMode2_DeleteCollection(t *testing.T) {
361361
tt.setupStorageFn(m)
362362
}
363363

364-
dw := NewDualWriter(Mode2, ls, us)
364+
dw := NewDualWriter(Mode2, ls, us, p)
365365

366366
obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: tt.input}}, &metainternalversion.ListOptions{})
367367

@@ -469,7 +469,7 @@ func TestMode2_Update(t *testing.T) {
469469
tt.setupStorageFn(m, tt.input)
470470
}
471471

472-
dw := NewDualWriter(Mode2, ls, us)
472+
dw := NewDualWriter(Mode2, ls, us, p)
473473

474474
obj, _, err := dw.Update(context.Background(), tt.input, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
475475

‎pkg/apiserver/rest/dualwriter_mode3.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,8 @@ type DualWriterMode3 struct {
2020

2121
// newDualWriterMode3 returns a new DualWriter in mode 3.
2222
// Mode 3 represents writing to LegacyStorage and Storage and reading from Storage.
23-
func newDualWriterMode3(legacy LegacyStorage, storage Storage) *DualWriterMode3 {
24-
metrics := &dualWriterMetrics{}
25-
metrics.init()
26-
return &DualWriterMode3{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode3"), dualWriterMetrics: metrics}
23+
func newDualWriterMode3(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode3 {
24+
return &DualWriterMode3{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode3"), dualWriterMetrics: dwm}
2725
}
2826

2927
// Mode returns the mode of the dual writer.

‎pkg/apiserver/rest/dualwriter_mode4.go

+2-4
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,8 @@ type DualWriterMode4 struct {
1919

2020
// newDualWriterMode4 returns a new DualWriter in mode 4.
2121
// Mode 4 represents writing and reading from Storage.
22-
func newDualWriterMode4(legacy LegacyStorage, storage Storage) *DualWriterMode4 {
23-
metrics := &dualWriterMetrics{}
24-
metrics.init()
25-
return &DualWriterMode4{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode4"), dualWriterMetrics: metrics}
22+
func newDualWriterMode4(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode4 {
23+
return &DualWriterMode4{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode4"), dualWriterMetrics: dwm}
2624
}
2725

2826
// Mode returns the mode of the dual writer.

‎pkg/apiserver/rest/dualwriter_test.go

+3-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77

88
playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1"
99
"github.com/grafana/grafana/pkg/infra/kvstore"
10+
"github.com/prometheus/client_golang/prometheus"
1011
"github.com/stretchr/testify/assert"
1112
"github.com/stretchr/testify/mock"
1213
)
@@ -45,7 +46,8 @@ func TestSetDualWritingMode(t *testing.T) {
4546

4647
kvStore := kvstore.WithNamespace(kvstore.NewFakeKVStore(), 0, "storage.dualwriting."+tt.stackID)
4748

48-
dw, err := SetDualWritingMode(context.Background(), kvStore, ls, us, playlist.GROUPRESOURCE, tt.desiredMode)
49+
p := prometheus.NewRegistry()
50+
dw, err := SetDualWritingMode(context.Background(), kvStore, ls, us, playlist.GROUPRESOURCE, tt.desiredMode, p)
4951
assert.NoError(t, err)
5052
assert.Equal(t, tt.expectedMode, dw.Mode())
5153

‎pkg/apiserver/rest/metrics.go

+9-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"time"
66

77
"github.com/prometheus/client_golang/prometheus"
8+
"k8s.io/klog/v2"
89
)
910

1011
type dualWriterMetrics struct {
@@ -37,10 +38,17 @@ var DualWriterOutcome = prometheus.NewHistogramVec(prometheus.HistogramOpts{
3738
NativeHistogramBucketFactor: 1.1,
3839
}, []string{"mode", "name", "method"})
3940

40-
func (m *dualWriterMetrics) init() {
41+
func (m *dualWriterMetrics) init(reg prometheus.Registerer) {
42+
log := klog.NewKlogr()
4143
m.legacy = DualWriterLegacyDuration
4244
m.storage = DualWriterStorageDuration
4345
m.outcome = DualWriterOutcome
46+
errLegacy := reg.Register(m.legacy)
47+
errStorage := reg.Register(m.storage)
48+
errOutcome := reg.Register(m.outcome)
49+
if errLegacy != nil || errStorage != nil || errOutcome != nil {
50+
log.Info("cloud migration metrics already registered")
51+
}
4452
}
4553

4654
func (m *dualWriterMetrics) recordLegacyDuration(isError bool, mode string, name string, method string, startFrom time.Time) {

‎pkg/cmd/grafana/apiserver/server.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf
165165

166166
// Install the API Group+version
167167
// #TODO figure out how to configure storage type in o.Options.StorageOptions
168-
err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, o.Options.StorageOptions)
168+
err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, o.Options.StorageOptions, o.Options.MetricsOptions.MetricsRegisterer)
169169
if err != nil {
170170
return err
171171
}

‎pkg/registry/apis/dashboard/register.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/grafana/grafana/pkg/services/featuremgmt"
2626
"github.com/grafana/grafana/pkg/services/provisioning"
2727
"github.com/grafana/grafana/pkg/setting"
28+
"github.com/prometheus/client_golang/prometheus"
2829
)
2930

3031
var _ builder.APIGroupBuilder = (*DashboardsAPIBuilder)(nil)
@@ -49,6 +50,7 @@ func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles,
4950
accessControl accesscontrol.AccessControl,
5051
provisioning provisioning.ProvisioningService,
5152
dashStore dashboards.Store,
53+
reg prometheus.Registerer,
5254
sql db.DB,
5355
) *DashboardsAPIBuilder {
5456
if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) {
@@ -115,6 +117,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo(
115117
codecs serializer.CodecFactory, // pointer?
116118
optsGetter generic.RESTOptionsGetter,
117119
desiredMode grafanarest.DualWriterMode,
120+
reg prometheus.Registerer,
118121
) (*genericapiserver.APIGroupInfo, error) {
119122
apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs)
120123

@@ -145,7 +148,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo(
145148
if err := store.CompleteWithOptions(options); err != nil {
146149
return nil, err
147150
}
148-
storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store)
151+
storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store, reg)
149152
}
150153

151154
// Summary

0 commit comments

Comments
 (0)