Provisioning: export worker pre-check if export would be out of quota (#118383)
* provisioning: export worker pre-check if export would be out of quota * provisioning: update utils import * provisioning: add integration tests for export job quota check
This commit is contained in:
committed by
GitHub
parent
3390095789
commit
2b37a47c99
@@ -249,6 +249,7 @@ func setupWorkers(
|
||||
exportWorker := export.NewExportWorker(
|
||||
clients,
|
||||
repositoryResources,
|
||||
resourceLister,
|
||||
export.ExportAll,
|
||||
stageIfPossible,
|
||||
metrics,
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/grafana/grafana-app-sdk/logging"
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/quotas"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
@@ -23,6 +24,7 @@ type WrapWithStageFn func(ctx context.Context, repo repository.Repository, stage
|
||||
type ExportWorker struct {
|
||||
clientFactory resources.ClientFactory
|
||||
repositoryResources resources.RepositoryResourcesFactory
|
||||
resourceLister resources.ResourceLister
|
||||
exportFn ExportFn
|
||||
wrapWithStageFn WrapWithStageFn
|
||||
metrics jobs.JobMetrics
|
||||
@@ -31,6 +33,7 @@ type ExportWorker struct {
|
||||
func NewExportWorker(
|
||||
clientFactory resources.ClientFactory,
|
||||
repositoryResources resources.RepositoryResourcesFactory,
|
||||
resourceLister resources.ResourceLister,
|
||||
exportFn ExportFn,
|
||||
wrapWithStageFn WrapWithStageFn,
|
||||
metrics jobs.JobMetrics,
|
||||
@@ -38,6 +41,7 @@ func NewExportWorker(
|
||||
return &ExportWorker{
|
||||
clientFactory: clientFactory,
|
||||
repositoryResources: repositoryResources,
|
||||
resourceLister: resourceLister,
|
||||
exportFn: exportFn,
|
||||
wrapWithStageFn: wrapWithStageFn,
|
||||
metrics: metrics,
|
||||
@@ -68,6 +72,11 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository,
|
||||
return err
|
||||
}
|
||||
|
||||
if err := checkExportQuota(ctx, cfg, r.resourceLister); err != nil {
|
||||
progress.Complete(ctx, err)
|
||||
return err
|
||||
}
|
||||
|
||||
msg := options.Message
|
||||
if msg == "" {
|
||||
msg = fmt.Sprintf("Export from Grafana %s", job.Name)
|
||||
@@ -127,3 +136,41 @@ func (r *ExportWorker) Process(ctx context.Context, repo repository.Repository,
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkExportQuota(ctx context.Context, cfg *provisioning.Repository, lister resources.ResourceLister) error {
|
||||
quota := cfg.Status.Quota
|
||||
if quota.MaxResourcesPerRepository == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
usage := quotas.NewQuotaUsageFromStats(cfg.Status.Stats)
|
||||
|
||||
stats, err := lister.Stats(ctx, cfg.Namespace, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("get resource stats for quota check: %w", err)
|
||||
}
|
||||
|
||||
netChange := countSupportedResources(stats.Unmanaged)
|
||||
|
||||
if !quotas.WouldStayWithinQuota(quota, usage, netChange) {
|
||||
total := usage.TotalResources + netChange
|
||||
return quotas.NewQuotaExceededError(
|
||||
fmt.Errorf("export would exceed quota: %d/%d resources", total, quota.MaxResourcesPerRepository),
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// countSupportedResources sums counts for resource types that support provisioning.
|
||||
func countSupportedResources(stats []provisioning.ResourceCount) int64 {
|
||||
var total int64
|
||||
for _, stat := range stats {
|
||||
for _, kind := range resources.SupportedProvisioningResources {
|
||||
if stat.Group == kind.Group && stat.Resource == kind.Resource {
|
||||
total += stat.Count
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
mock "github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
v0alpha1 "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/quotas"
|
||||
"github.com/grafana/grafana/apps/provisioning/pkg/repository"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/jobs"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/provisioning/resources"
|
||||
@@ -56,7 +58,7 @@ func TestExportWorker_IsSupported(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := NewExportWorker(nil, nil, nil, nil, metrics)
|
||||
r := NewExportWorker(nil, nil, nil, nil, nil, metrics)
|
||||
got := r.IsSupported(context.Background(), tt.job)
|
||||
require.Equal(t, tt.want, got)
|
||||
})
|
||||
@@ -70,7 +72,7 @@ func TestExportWorker_ProcessNoExportSettings(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
r := NewExportWorker(nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(nil, nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), nil, job, nil)
|
||||
require.EqualError(t, err, "missing export settings")
|
||||
}
|
||||
@@ -93,7 +95,7 @@ func TestExportWorker_ProcessWriteNotAllowed(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
r := NewExportWorker(nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(nil, nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, nil)
|
||||
require.EqualError(t, err, "this repository is read only")
|
||||
}
|
||||
@@ -117,7 +119,7 @@ func TestExportWorker_ProcessBranchNotAllowedForLocal(t *testing.T) {
|
||||
},
|
||||
})
|
||||
|
||||
r := NewExportWorker(nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(nil, nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, nil)
|
||||
require.EqualError(t, err, "this repository does not support the branch workflow")
|
||||
}
|
||||
@@ -149,7 +151,7 @@ func TestExportWorker_ProcessFailedToCreateClients(t *testing.T) {
|
||||
return fn(repo, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, nil, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, nil, nil, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
mockProgress := jobs.NewMockJobProgressRecorder(t)
|
||||
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
@@ -185,7 +187,7 @@ func TestExportWorker_ProcessNotReaderWriter(t *testing.T) {
|
||||
return fn(repo, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, nil, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, nil, nil, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.EqualError(t, err, "export job submitted targeting repository that is not a ReaderWriter")
|
||||
}
|
||||
@@ -221,7 +223,7 @@ func TestExportWorker_ProcessRepositoryResourcesError(t *testing.T) {
|
||||
mockStageFn.On("Execute", context.Background(), mockRepo, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, stageOpts repository.StageOptions, fn func(repository.Repository, bool) error) error {
|
||||
return fn(repo, true)
|
||||
})
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.EqualError(t, err, "create repository resource client: failed to create repository resources client")
|
||||
}
|
||||
@@ -273,7 +275,7 @@ func TestExportWorker_ProcessStageOptions(t *testing.T) {
|
||||
return fn(repo, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -355,7 +357,7 @@ func TestExportWorker_ProcessStageOptionsWithBranch(t *testing.T) {
|
||||
return fn(repo, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.NoError(t, err)
|
||||
})
|
||||
@@ -398,7 +400,7 @@ func TestExportWorker_ProcessExportFnError(t *testing.T) {
|
||||
return fn(repo, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.EqualError(t, err, "export failed")
|
||||
}
|
||||
@@ -426,7 +428,7 @@ func TestExportWorker_ProcessWrapWithStageFnError(t *testing.T) {
|
||||
mockStageFn := NewMockWrapWithStageFn(t)
|
||||
mockStageFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything).Return(errors.New("stage failed"))
|
||||
|
||||
r := NewExportWorker(nil, nil, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(nil, nil, nil, nil, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.EqualError(t, err, "stage failed")
|
||||
}
|
||||
@@ -452,7 +454,7 @@ func TestExportWorker_ProcessBranchNotAllowedForStageableRepositories(t *testing
|
||||
mockProgress := jobs.NewMockJobProgressRecorder(t)
|
||||
// No progress messages expected in current implementation
|
||||
|
||||
r := NewExportWorker(nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(nil, nil, nil, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.EqualError(t, err, "this repository does not support the branch workflow")
|
||||
}
|
||||
@@ -504,7 +506,7 @@ func TestExportWorker_ProcessGitRepository(t *testing.T) {
|
||||
return fn(repo, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -550,7 +552,7 @@ func TestExportWorker_ProcessGitRepositoryExportFnError(t *testing.T) {
|
||||
return fn(repo, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.EqualError(t, err, "export failed")
|
||||
}
|
||||
@@ -613,7 +615,7 @@ func TestExportWorker_RefURLsSetWithBranch(t *testing.T) {
|
||||
return fn(mockReaderWriter, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepoWithURLs, job, mockProgress)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -670,7 +672,7 @@ func TestExportWorker_RefURLsNotSetWithoutBranch(t *testing.T) {
|
||||
return fn(mockReaderWriter, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepoWithURLs, job, mockProgress)
|
||||
require.NoError(t, err)
|
||||
|
||||
@@ -727,10 +729,376 @@ func TestExportWorker_RefURLsNotSetForNonURLRepository(t *testing.T) {
|
||||
return fn(mockReaderWriter, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, mockRepoResources, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify that SetRefURLs was NOT called since repo doesn't support URLs
|
||||
mockProgress.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestCountSupportedResources(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
stats []v0alpha1.ResourceCount
|
||||
expected int64
|
||||
}{
|
||||
{
|
||||
name: "nil stats",
|
||||
stats: nil,
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "empty stats",
|
||||
stats: []v0alpha1.ResourceCount{},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "only dashboards",
|
||||
stats: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 10},
|
||||
},
|
||||
expected: 10,
|
||||
},
|
||||
{
|
||||
name: "only folders",
|
||||
stats: []v0alpha1.ResourceCount{
|
||||
{Group: "folder.grafana.app", Resource: "folders", Count: 5},
|
||||
},
|
||||
expected: 5,
|
||||
},
|
||||
{
|
||||
name: "dashboards and folders",
|
||||
stats: []v0alpha1.ResourceCount{
|
||||
{Group: "folder.grafana.app", Resource: "folders", Count: 3},
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 7},
|
||||
},
|
||||
expected: 10,
|
||||
},
|
||||
{
|
||||
name: "includes unsupported resources",
|
||||
stats: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 4},
|
||||
{Group: "alerting.grafana.app", Resource: "alertrules", Count: 100},
|
||||
{Group: "folder.grafana.app", Resource: "folders", Count: 2},
|
||||
},
|
||||
expected: 6,
|
||||
},
|
||||
{
|
||||
name: "only unsupported resources",
|
||||
stats: []v0alpha1.ResourceCount{
|
||||
{Group: "alerting.grafana.app", Resource: "alertrules", Count: 50},
|
||||
{Group: "datasource.grafana.app", Resource: "datasources", Count: 10},
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := countSupportedResources(tt.stats)
|
||||
assert.Equal(t, tt.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckExportQuota(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
repoName string
|
||||
quota v0alpha1.QuotaStatus
|
||||
repoStats []v0alpha1.ResourceCount
|
||||
listerStats *v0alpha1.ResourceStats
|
||||
expectError bool
|
||||
}{
|
||||
{
|
||||
name: "unlimited quota allows export",
|
||||
repoName: "test-repo",
|
||||
quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 0,
|
||||
},
|
||||
listerStats: nil,
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "within quota allows export",
|
||||
repoName: "test-repo",
|
||||
quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 100,
|
||||
},
|
||||
listerStats: &v0alpha1.ResourceStats{
|
||||
Unmanaged: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 50},
|
||||
{Group: "folder.grafana.app", Resource: "folders", Count: 10},
|
||||
},
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "at quota allows export",
|
||||
repoName: "test-repo",
|
||||
quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 100,
|
||||
},
|
||||
listerStats: &v0alpha1.ResourceStats{
|
||||
Unmanaged: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 90},
|
||||
{Group: "folder.grafana.app", Resource: "folders", Count: 10},
|
||||
},
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "exceeds quota blocks export",
|
||||
repoName: "test-repo",
|
||||
quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 100,
|
||||
},
|
||||
listerStats: &v0alpha1.ResourceStats{
|
||||
Unmanaged: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 80},
|
||||
{Group: "folder.grafana.app", Resource: "folders", Count: 25},
|
||||
},
|
||||
},
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "unsupported unmanaged resources not counted as net change",
|
||||
repoName: "test-repo",
|
||||
quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 10,
|
||||
},
|
||||
listerStats: &v0alpha1.ResourceStats{
|
||||
Unmanaged: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 5},
|
||||
{Group: "alerting.grafana.app", Resource: "alertrules", Count: 500},
|
||||
},
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "empty stats within quota",
|
||||
repoName: "test-repo",
|
||||
quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 10,
|
||||
},
|
||||
listerStats: &v0alpha1.ResourceStats{},
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "repo usage plus unmanaged exceeds quota",
|
||||
repoName: "test-repo",
|
||||
quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 10,
|
||||
},
|
||||
repoStats: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 6},
|
||||
},
|
||||
listerStats: &v0alpha1.ResourceStats{
|
||||
Unmanaged: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 5},
|
||||
},
|
||||
},
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "repo usage plus unmanaged within quota",
|
||||
repoName: "test-repo",
|
||||
quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 20,
|
||||
},
|
||||
repoStats: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 6},
|
||||
},
|
||||
listerStats: &v0alpha1.ResourceStats{
|
||||
Unmanaged: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 5},
|
||||
},
|
||||
},
|
||||
expectError: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
cfg := &v0alpha1.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: tt.repoName,
|
||||
Namespace: "test-namespace",
|
||||
},
|
||||
Status: v0alpha1.RepositoryStatus{
|
||||
Quota: tt.quota,
|
||||
Stats: tt.repoStats,
|
||||
},
|
||||
}
|
||||
|
||||
var lister resources.ResourceLister
|
||||
if tt.listerStats != nil {
|
||||
mockLister := resources.NewMockResourceLister(t)
|
||||
mockLister.On("Stats", mock.Anything, "test-namespace", "").Return(tt.listerStats, nil)
|
||||
lister = mockLister
|
||||
}
|
||||
|
||||
err := checkExportQuota(context.Background(), cfg, lister)
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
var quotaErr *quotas.QuotaExceededError
|
||||
require.ErrorAs(t, err, "aErr)
|
||||
assert.Contains(t, err.Error(), "export would exceed quota")
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportWorker_ProcessQuotaExceeded(t *testing.T) {
|
||||
job := v0alpha1.Job{
|
||||
Spec: v0alpha1.JobSpec{
|
||||
Action: v0alpha1.JobActionPush,
|
||||
Push: &v0alpha1.ExportJobOptions{},
|
||||
},
|
||||
}
|
||||
|
||||
mockRepo := repository.NewMockRepository(t)
|
||||
mockRepo.On("Config").Return(&v0alpha1.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-repo",
|
||||
Namespace: "test-namespace",
|
||||
},
|
||||
Spec: v0alpha1.RepositorySpec{
|
||||
Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow},
|
||||
},
|
||||
Status: v0alpha1.RepositoryStatus{
|
||||
Quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 10,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
mockLister := resources.NewMockResourceLister(t)
|
||||
mockLister.On("Stats", mock.Anything, "test-namespace", "").Return(&v0alpha1.ResourceStats{
|
||||
Unmanaged: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 15},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
mockProgress := jobs.NewMockJobProgressRecorder(t)
|
||||
mockProgress.On("Complete", mock.Anything, mock.MatchedBy(func(err error) bool {
|
||||
var quotaErr *quotas.QuotaExceededError
|
||||
return errors.As(err, "aErr)
|
||||
})).Return(v0alpha1.JobStatus{})
|
||||
|
||||
r := NewExportWorker(nil, nil, mockLister, nil, nil, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
|
||||
require.Error(t, err)
|
||||
var quotaErr *quotas.QuotaExceededError
|
||||
require.ErrorAs(t, err, "aErr)
|
||||
assert.Contains(t, err.Error(), "export would exceed quota: 15/10 resources")
|
||||
mockProgress.AssertExpectations(t)
|
||||
}
|
||||
|
||||
func TestExportWorker_ProcessQuotaNotExceeded(t *testing.T) {
|
||||
job := v0alpha1.Job{
|
||||
Spec: v0alpha1.JobSpec{
|
||||
Action: v0alpha1.JobActionPush,
|
||||
Push: &v0alpha1.ExportJobOptions{},
|
||||
},
|
||||
}
|
||||
|
||||
mockRepo := repository.NewMockRepository(t)
|
||||
mockRepo.On("Config").Return(&v0alpha1.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-repo",
|
||||
Namespace: "test-namespace",
|
||||
},
|
||||
Spec: v0alpha1.RepositorySpec{
|
||||
Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow},
|
||||
},
|
||||
Status: v0alpha1.RepositoryStatus{
|
||||
Quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 100,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
mockLister := resources.NewMockResourceLister(t)
|
||||
mockLister.On("Stats", mock.Anything, "test-namespace", "").Return(&v0alpha1.ResourceStats{
|
||||
Unmanaged: []v0alpha1.ResourceCount{
|
||||
{Group: "dashboard.grafana.app", Resource: "dashboards", Count: 10},
|
||||
{Group: "folder.grafana.app", Resource: "folders", Count: 5},
|
||||
},
|
||||
}, nil)
|
||||
|
||||
mockProgress := jobs.NewMockJobProgressRecorder(t)
|
||||
mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{})
|
||||
|
||||
mockClients := resources.NewMockClientFactory(t)
|
||||
mockResourceClients := resources.NewMockResourceClients(t)
|
||||
mockClients.On("Clients", mock.Anything, "test-namespace").Return(mockResourceClients, nil)
|
||||
|
||||
mockRepoResources := resources.NewMockRepositoryResourcesFactory(t)
|
||||
mockRepoResourcesClient := resources.NewMockRepositoryResources(t)
|
||||
mockRepoResources.On("Client", mock.Anything, mock.Anything).Return(mockRepoResourcesClient, nil)
|
||||
|
||||
mockExportFn := NewMockExportFn(t)
|
||||
mockExportFn.On("Execute", mock.Anything, "test-repo", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
|
||||
mockStageFn := NewMockWrapWithStageFn(t)
|
||||
mockStageFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, stageOpts repository.StageOptions, fn func(repository.Repository, bool) error) error {
|
||||
return fn(repo, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, mockRepoResources, mockLister, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestExportWorker_ProcessQuotaUnlimited(t *testing.T) {
|
||||
job := v0alpha1.Job{
|
||||
Spec: v0alpha1.JobSpec{
|
||||
Action: v0alpha1.JobActionPush,
|
||||
Push: &v0alpha1.ExportJobOptions{},
|
||||
},
|
||||
}
|
||||
|
||||
mockRepo := repository.NewMockRepository(t)
|
||||
mockRepo.On("Config").Return(&v0alpha1.Repository{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-repo",
|
||||
Namespace: "test-namespace",
|
||||
},
|
||||
Spec: v0alpha1.RepositorySpec{
|
||||
Workflows: []v0alpha1.Workflow{v0alpha1.WriteWorkflow},
|
||||
},
|
||||
Status: v0alpha1.RepositoryStatus{
|
||||
Quota: v0alpha1.QuotaStatus{
|
||||
MaxResourcesPerRepository: 0,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
mockProgress := jobs.NewMockJobProgressRecorder(t)
|
||||
mockProgress.On("Complete", mock.Anything, mock.Anything).Return(v0alpha1.JobStatus{})
|
||||
|
||||
mockClients := resources.NewMockClientFactory(t)
|
||||
mockResourceClients := resources.NewMockResourceClients(t)
|
||||
mockClients.On("Clients", mock.Anything, "test-namespace").Return(mockResourceClients, nil)
|
||||
|
||||
mockRepoResources := resources.NewMockRepositoryResourcesFactory(t)
|
||||
mockRepoResourcesClient := resources.NewMockRepositoryResources(t)
|
||||
mockRepoResources.On("Client", mock.Anything, mock.Anything).Return(mockRepoResourcesClient, nil)
|
||||
|
||||
mockExportFn := NewMockExportFn(t)
|
||||
mockExportFn.On("Execute", mock.Anything, "test-repo", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
|
||||
|
||||
mockStageFn := NewMockWrapWithStageFn(t)
|
||||
mockStageFn.On("Execute", mock.Anything, mockRepo, mock.Anything, mock.Anything).Return(func(ctx context.Context, repo repository.Repository, stageOpts repository.StageOptions, fn func(repository.Repository, bool) error) error {
|
||||
return fn(repo, true)
|
||||
})
|
||||
|
||||
r := NewExportWorker(mockClients, mockRepoResources, nil, mockExportFn.Execute, mockStageFn.Execute, jobs.RegisterJobMetrics(prometheus.NewPedanticRegistry()))
|
||||
err := r.Process(context.Background(), mockRepo, job, mockProgress)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
@@ -802,6 +802,7 @@ func (b *APIBuilder) GetPostStartHooks() (map[string]genericapiserver.PostStartH
|
||||
exportWorker := export.NewExportWorker(
|
||||
b.clients,
|
||||
b.repositoryResources,
|
||||
b.resourceLister,
|
||||
export.ExportAll,
|
||||
stageIfPossible,
|
||||
metrics,
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
package provisioning
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
|
||||
foldersV1 "github.com/grafana/grafana/apps/folder/pkg/apis/folder/v1beta1"
|
||||
provisioning "github.com/grafana/grafana/apps/provisioning/pkg/apis/provisioning/v0alpha1"
|
||||
"github.com/grafana/grafana/pkg/tests/testinfra"
|
||||
"github.com/grafana/grafana/pkg/util/testutil"
|
||||
)
|
||||
|
||||
func TestIntegrationProvisioning_ExportQuota(t *testing.T) {
|
||||
testutil.SkipIntegrationTestInShortMode(t)
|
||||
|
||||
t.Run("export succeeds when resources are within quota", func(t *testing.T) {
|
||||
helper := runGrafana(t, func(opts *testinfra.GrafanaOpts) {
|
||||
opts.ProvisioningMaxResourcesPerRepository = 10
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
// Create 2 unmanaged dashboards directly in Grafana
|
||||
dashboard1 := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v1.yaml")
|
||||
_, err := helper.DashboardsV1.Resource.Create(ctx, dashboard1, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create first dashboard")
|
||||
|
||||
dashboard2 := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v0.yaml")
|
||||
_, err = helper.DashboardsV0.Resource.Create(ctx, dashboard2, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create second dashboard")
|
||||
|
||||
// Create an empty repository with instance target for export
|
||||
const repo = "export-quota-success"
|
||||
testRepo := TestRepo{
|
||||
Name: repo,
|
||||
Target: "instance",
|
||||
Copies: map[string]string{},
|
||||
ExpectedDashboards: 2,
|
||||
ExpectedFolders: 0,
|
||||
}
|
||||
helper.CreateRepo(t, testRepo)
|
||||
|
||||
// Wait for quota reconciliation to confirm limits are set on the repository
|
||||
helper.WaitForQuotaReconciliation(t, repo, provisioning.ReasonWithinQuota)
|
||||
|
||||
// Export should succeed: 2 dashboards + 0 folders = 2 resources <= quota of 10
|
||||
spec := provisioning.JobSpec{
|
||||
Action: provisioning.JobActionPush,
|
||||
Push: &provisioning.ExportJobOptions{
|
||||
Folder: "",
|
||||
Path: "",
|
||||
},
|
||||
}
|
||||
helper.TriggerJobAndWaitForSuccess(t, repo, spec)
|
||||
|
||||
// Verify files were actually exported to the provisioning path
|
||||
files, err := countFilesInDir(helper.ProvisioningPath)
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, files, 0, "should have exported dashboard files")
|
||||
})
|
||||
|
||||
t.Run("export fails when existing resources already exceed the quota", func(t *testing.T) {
|
||||
helper := runGrafana(t, func(opts *testinfra.GrafanaOpts) {
|
||||
opts.ProvisioningMaxResourcesPerRepository = 1
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
// Create 2 unmanaged dashboards — these alone will exceed the quota of 1
|
||||
dashboard1 := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v1.yaml")
|
||||
_, err := helper.DashboardsV1.Resource.Create(ctx, dashboard1, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create first dashboard")
|
||||
|
||||
dashboard2 := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v0.yaml")
|
||||
_, err = helper.DashboardsV0.Resource.Create(ctx, dashboard2, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create second dashboard")
|
||||
|
||||
const repo = "export-quota-resources-exceeded"
|
||||
testRepo := TestRepo{
|
||||
Name: repo,
|
||||
Target: "instance",
|
||||
Copies: map[string]string{},
|
||||
ExpectedDashboards: 2,
|
||||
ExpectedFolders: 0,
|
||||
}
|
||||
helper.CreateRepo(t, testRepo)
|
||||
|
||||
helper.WaitForQuotaReconciliation(t, repo, provisioning.ReasonWithinQuota)
|
||||
|
||||
// Export should fail: 2 dashboards + 0 folders = 2 resources > quota of 1
|
||||
spec := provisioning.JobSpec{
|
||||
Action: provisioning.JobActionPush,
|
||||
Push: &provisioning.ExportJobOptions{
|
||||
Folder: "",
|
||||
Path: "",
|
||||
},
|
||||
}
|
||||
job := helper.TriggerJobAndWaitForComplete(t, repo, spec)
|
||||
|
||||
jobObj := &provisioning.Job{}
|
||||
err = runtime.DefaultUnstructuredConverter.FromUnstructured(job.Object, jobObj)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, provisioning.JobStateError, jobObj.Status.State,
|
||||
"export job should fail when resources exceed quota")
|
||||
require.Equal(t, "export would exceed quota: 2/1 resources", jobObj.Status.Message,
|
||||
"error message should report the exact resource count vs quota limit")
|
||||
|
||||
// Verify no files were exported since quota check happens before any writes
|
||||
files, err := countFilesInDir(helper.ProvisioningPath)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, files, "should not have exported any files when quota is exceeded")
|
||||
})
|
||||
|
||||
t.Run("export fails when exceeding folders and resources already exceed the quota", func(t *testing.T) {
|
||||
helper := runGrafana(t, func(opts *testinfra.GrafanaOpts) {
|
||||
opts.ProvisioningMaxResourcesPerRepository = 2
|
||||
})
|
||||
ctx := context.Background()
|
||||
|
||||
// Create 1 unmanaged dashboard (alone would fit in quota of 2)
|
||||
dashboard := helper.LoadYAMLOrJSONFile("exportunifiedtorepository/dashboard-test-v1.yaml")
|
||||
_, err := helper.DashboardsV1.Resource.Create(ctx, dashboard, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create dashboard")
|
||||
|
||||
// Create 2 unmanaged folders — together with the dashboard, the total
|
||||
// becomes 1 dashboard + 2 folders = 3 resources which exceeds the quota of 2.
|
||||
for i, name := range []string{"export-test-folder-1", "export-test-folder-2"} {
|
||||
folderObj := &unstructured.Unstructured{
|
||||
Object: map[string]interface{}{
|
||||
"apiVersion": foldersV1.FolderResourceInfo.GroupVersion().String(),
|
||||
"kind": foldersV1.FolderResourceInfo.GroupVersionKind().Kind,
|
||||
"metadata": map[string]interface{}{
|
||||
"name": name,
|
||||
},
|
||||
"spec": map[string]interface{}{
|
||||
"title": fmt.Sprintf("Export Test Folder %d", i+1),
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = helper.Folders.Resource.Create(ctx, folderObj, metav1.CreateOptions{})
|
||||
require.NoError(t, err, "should be able to create folder %s", name)
|
||||
}
|
||||
|
||||
const repo = "export-quota-folders-exceeded"
|
||||
testRepo := TestRepo{
|
||||
Name: repo,
|
||||
Target: "instance",
|
||||
Copies: map[string]string{},
|
||||
ExpectedDashboards: 1,
|
||||
ExpectedFolders: 2,
|
||||
}
|
||||
helper.CreateRepo(t, testRepo)
|
||||
|
||||
helper.WaitForQuotaReconciliation(t, repo, provisioning.ReasonWithinQuota)
|
||||
|
||||
// Export should fail: 1 dashboard + 2 folders = 3 resources > quota of 2
|
||||
spec := provisioning.JobSpec{
|
||||
Action: provisioning.JobActionPush,
|
||||
Push: &provisioning.ExportJobOptions{
|
||||
Folder: "",
|
||||
Path: "",
|
||||
},
|
||||
}
|
||||
job := helper.TriggerJobAndWaitForComplete(t, repo, spec)
|
||||
|
||||
jobObj := &provisioning.Job{}
|
||||
err = runtime.DefaultUnstructuredConverter.FromUnstructured(job.Object, jobObj)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.Equal(t, provisioning.JobStateError, jobObj.Status.State,
|
||||
"export job should fail when folders push total over quota")
|
||||
require.Equal(t, "export would exceed quota: 3/2 resources", jobObj.Status.Message,
|
||||
"error message should report the exact resource count vs quota limit")
|
||||
|
||||
// Verify no files were exported since quota check happens before any writes
|
||||
files, err := countFilesInDir(helper.ProvisioningPath)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, files, "should not have exported any files when quota is exceeded due to folders")
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user