From 027e999722d6ace8f53bedaf9e3f7045658c9624 Mon Sep 17 00:00:00 2001 From: Uriah Rokach Date: Sun, 9 Aug 2026 16:27:01 +0300 Subject: [PATCH 01/14] feat(monorepo): add plugin for per-sub-project DORA metrics in a monorepo Repos containing multiple logically separate projects (each deployed by its own CI job, PRs tagged by label) previously collapsed into a single set of DORA numbers, since DevLake's scope model is one-scope-per-repo. Adds a new metric plugin, monorepo, that runs after dora and attributes deployments (by CI job name) and merged pull requests (by label) to a configured sub-project, writing per-sub-project deployment and change lead time metrics to two new tables. Nothing existing is modified: dora and the core scope model are untouched, and PR coding/pickup/review time are reused from dora's own project_pr_metrics rather than recomputed. Includes unit tests, an e2e test with fixtures, and Grafana dashboards (mysql + postgresql) to view the output. --- .../plugins/monorepo/e2e/attribution_test.go | 101 +++ .../cicd_deployment_commits.csv | 7 + .../e2e/monorepo_attribution/cicd_tasks.csv | 9 + .../monorepo_attribution/project_mapping.csv | 5 + .../project_pr_metrics.csv | 7 + .../pull_request_labels.csv | 9 + .../monorepo_attribution/pull_requests.csv | 8 + .../monorepo_subproject_deployments.csv | 7 + .../monorepo_subproject_pr_metrics.csv | 5 + backend/plugins/monorepo/impl/impl.go | 170 +++++ .../20260809_add_init_tables.go | 44 ++ .../models/migrationscripts/register.go | 29 + .../monorepo/models/subproject_deployment.go | 53 ++ .../monorepo/models/subproject_pr_metric.go | 57 ++ backend/plugins/monorepo/monorepo.go | 43 ++ .../monorepo/tasks/deployment_attributor.go | 119 ++++ .../plugins/monorepo/tasks/pr_attributor.go | 275 ++++++++ .../monorepo/tasks/pr_attributor_test.go | 125 ++++ backend/plugins/monorepo/tasks/task_data.go | 154 +++++ .../plugins/monorepo/tasks/task_data_test.go | 220 ++++++ .../mysql/monorepo-subprojects.json | 340 ++++++++++ .../postgresql/monorepo-subprojects.json | 628 ++++++++++++++++++ 22 files changed, 2415 insertions(+) create mode 100644 backend/plugins/monorepo/e2e/attribution_test.go create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv create mode 100644 backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv create mode 100644 backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv create mode 100644 backend/plugins/monorepo/impl/impl.go create mode 100644 backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go create mode 100644 backend/plugins/monorepo/models/migrationscripts/register.go create mode 100644 backend/plugins/monorepo/models/subproject_deployment.go create mode 100644 backend/plugins/monorepo/models/subproject_pr_metric.go create mode 100644 backend/plugins/monorepo/monorepo.go create mode 100644 backend/plugins/monorepo/tasks/deployment_attributor.go create mode 100644 backend/plugins/monorepo/tasks/pr_attributor.go create mode 100644 backend/plugins/monorepo/tasks/pr_attributor_test.go create mode 100644 backend/plugins/monorepo/tasks/task_data.go create mode 100644 backend/plugins/monorepo/tasks/task_data_test.go create mode 100644 grafana/dashboards/mysql/monorepo-subprojects.json create mode 100644 grafana/dashboards/postgresql/monorepo-subprojects.json diff --git a/backend/plugins/monorepo/e2e/attribution_test.go b/backend/plugins/monorepo/e2e/attribution_test.go new file mode 100644 index 00000000000..b56d1ce4a07 --- /dev/null +++ b/backend/plugins/monorepo/e2e/attribution_test.go @@ -0,0 +1,101 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/monorepo/impl" + "github.com/apache/incubator-devlake/plugins/monorepo/models" + "github.com/apache/incubator-devlake/plugins/monorepo/tasks" + "github.com/stretchr/testify/assert" +) + +// TestMonorepoAttributionDataFlow exercises both subtasks against a monorepo containing +// serviceA and serviceB, each with its own deploy job. +// +// The fixtures deliberately include the cases that motivated this plugin: +// - pr2 (serviceB) merges at 09:00 while serviceA deploys at 10:00 and serviceB only at +// 12:00. DORA would link pr2 to the 10:00 deployment because it searches the whole +// repository; pr2 must instead link to 12:00. +// - pipeline3 runs both deploy jobs, so it must yield one row per sub-project. +// - a failed deployment and a staging deployment sit between pr1's merge and the +// deployment that actually shipped it, so neither may be linked. +func TestMonorepoAttributionDataFlow(t *testing.T) { + var plugin impl.Monorepo + dataflowTester := e2ehelper.NewDataFlowTester(t, "monorepo", plugin) + + subProjects := []tasks.SubProjectConfig{ + { + Name: "serviceA", + PrLabels: []string{"serviceA"}, + DeployJobPattern: "^deploy-serviceA$", + }, + { + Name: "serviceB", + PrLabels: []string{"serviceB"}, + DeployJobPattern: "^deploy-serviceB$", + }, + } + matcher, err := tasks.NewSubProjectMatcher(subProjects) + assert.Nil(t, err) + + taskData := &tasks.MonorepoTaskData{ + Options: &tasks.MonorepoOptions{ + ProjectName: "monorepo", + SubProjects: subProjects, + }, + Matcher: matcher, + } + + // seed the domain layer + dataflowTester.FlushTabler(&crossdomain.ProjectMapping{}) + dataflowTester.FlushTabler(&devops.CICDTask{}) + dataflowTester.FlushTabler(&devops.CicdDeploymentCommit{}) + dataflowTester.FlushTabler(&code.PullRequest{}) + dataflowTester.FlushTabler(&code.PullRequestLabel{}) + dataflowTester.FlushTabler(&crossdomain.ProjectPrMetric{}) + + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/project_mapping.csv", &crossdomain.ProjectMapping{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/cicd_tasks.csv", &devops.CICDTask{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/cicd_deployment_commits.csv", &devops.CicdDeploymentCommit{}) + dataflowTester.ImportNullableCsvIntoTabler("./monorepo_attribution/pull_requests.csv", &code.PullRequest{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/pull_request_labels.csv", &code.PullRequestLabel{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/project_pr_metrics.csv", &crossdomain.ProjectPrMetric{}) + + // deployments must be attributed first: the pull request subtask reads them back to + // work out which deployment shipped each merged pull request. + dataflowTester.FlushTabler(&models.SubProjectDeployment{}) + dataflowTester.Subtask(tasks.AttributeDeploymentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(&models.SubProjectDeployment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/monorepo_subproject_deployments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + dataflowTester.FlushTabler(&models.SubProjectPrMetric{}) + dataflowTester.Subtask(tasks.AttributePullRequestsMeta, taskData) + dataflowTester.VerifyTableWithOptions(&models.SubProjectPrMetric{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/monorepo_subproject_pr_metrics.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv new file mode 100644 index 00000000000..0bf5e350c27 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv @@ -0,0 +1,7 @@ +id,cicd_deployment_id,cicd_scope_id,name,result,status,environment,repo_url,commit_sha,created_date,finished_date +dc1,pipeline1,cicd1,deploy-serviceA,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitA1,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +dc2,pipeline2,cicd1,deploy-serviceB,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitB1,2026-08-01T11:50:00.000+00:00,2026-08-01T12:00:00.000+00:00 +dc3,pipeline3,cicd1,deploy-both,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitAB,2026-08-02T09:50:00.000+00:00,2026-08-02T10:00:00.000+00:00 +dc5,pipeline5,cicd2,deploy-serviceA,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/other,commitOther,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +dc6,pipeline6,cicd1,deploy-serviceA,FAILURE,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitFail,2026-08-01T09:20:00.000+00:00,2026-08-01T09:30:00.000+00:00 +dc7,pipeline7,cicd1,deploy-serviceA,SUCCESS,DONE,STAGING,https://gitlab.example.com/acme/monorepo,commitStg,2026-08-01T09:35:00.000+00:00,2026-08-01T09:45:00.000+00:00 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv new file mode 100644 index 00000000000..9324d96ffb1 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv @@ -0,0 +1,9 @@ +id,name,pipeline_id,type,result,status,environment,cicd_scope_id,created_date,finished_date +task1,deploy-serviceA,pipeline1,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +task1b,build,pipeline1,,SUCCESS,DONE,,cicd1,2026-08-01T09:40:00.000+00:00,2026-08-01T09:50:00.000+00:00 +task2,deploy-serviceB,pipeline2,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-01T11:50:00.000+00:00,2026-08-01T12:00:00.000+00:00 +task3a,deploy-serviceA,pipeline3,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-02T09:50:00.000+00:00,2026-08-02T10:00:00.000+00:00 +task3b,deploy-serviceB,pipeline3,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-02T09:50:00.000+00:00,2026-08-02T10:00:00.000+00:00 +task5,deploy-serviceA,pipeline5,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd2,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +task6,deploy-serviceA,pipeline6,DEPLOYMENT,FAILURE,DONE,PRODUCTION,cicd1,2026-08-01T09:20:00.000+00:00,2026-08-01T09:30:00.000+00:00 +task7,deploy-serviceA,pipeline7,DEPLOYMENT,SUCCESS,DONE,STAGING,cicd1,2026-08-01T09:35:00.000+00:00,2026-08-01T09:45:00.000+00:00 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv new file mode 100644 index 00000000000..c871e7cb114 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv @@ -0,0 +1,5 @@ +project_name,table,row_id +monorepo,cicd_scopes,cicd1 +monorepo,repos,repo1 +other,cicd_scopes,cicd2 +other,repos,repo2 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv new file mode 100644 index 00000000000..c60538d6507 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv @@ -0,0 +1,7 @@ +id,project_name,pr_coding_time,pr_pickup_time,pr_review_time +pr1,monorepo,100,20,30 +pr2,monorepo,200,40,60 +pr3,monorepo,300,60,90 +pr4,monorepo,400,80,120 +pr5,monorepo,500,100,150 +pr7,monorepo,700,140,210 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv new file mode 100644 index 00000000000..95b45e7f473 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv @@ -0,0 +1,9 @@ +pull_request_id,label_name +pr1,serviceA +pr2,serviceB +pr3,serviceB +pr3,serviceA +pr4,bug +pr5,serviceA +pr6,serviceA +pr7,serviceA diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv new file mode 100644 index 00000000000..c1809224f2b --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv @@ -0,0 +1,8 @@ +id,base_repo_id,created_date,merged_date,merge_commit_sha +pr1,repo1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitA1 +pr2,repo1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitB1 +pr3,repo1,2026-08-01T08:30:00.000+00:00,2026-08-01T09:30:00.000+00:00,commitAB +pr4,repo1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitBug +pr5,repo1,2026-08-03T08:00:00.000+00:00,2026-08-03T09:00:00.000+00:00,commitLate +pr6,repo2,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitOther +pr7,repo1,2026-08-01T08:00:00.000+00:00,NULL,commitOpen diff --git a/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv new file mode 100644 index 00000000000..1504f88eff7 --- /dev/null +++ b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv @@ -0,0 +1,7 @@ +project_name,sub_project,cicd_deployment_id,commit_sha,job_name,result,environment,finished_date +monorepo,serviceA,pipeline1,commitA1,deploy-serviceA,SUCCESS,PRODUCTION,2026-08-01T10:00:00.000+00:00 +monorepo,serviceB,pipeline2,commitB1,deploy-serviceB,SUCCESS,PRODUCTION,2026-08-01T12:00:00.000+00:00 +monorepo,serviceA,pipeline3,commitAB,deploy-serviceA,SUCCESS,PRODUCTION,2026-08-02T10:00:00.000+00:00 +monorepo,serviceB,pipeline3,commitAB,deploy-serviceB,SUCCESS,PRODUCTION,2026-08-02T10:00:00.000+00:00 +monorepo,serviceA,pipeline6,commitFail,deploy-serviceA,FAILURE,PRODUCTION,2026-08-01T09:30:00.000+00:00 +monorepo,serviceA,pipeline7,commitStg,deploy-serviceA,SUCCESS,STAGING,2026-08-01T09:45:00.000+00:00 diff --git a/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv new file mode 100644 index 00000000000..f40d9459bbf --- /dev/null +++ b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv @@ -0,0 +1,5 @@ +project_name,pull_request_id,sub_project,coding_time,pickup_time,review_time,deploy_time,cycle_time,deployment_id,pr_created_date,pr_merged_date,deployed_date +monorepo,pr1,serviceA,100,20,30,60,220,pipeline1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,2026-08-01T10:00:00.000+00:00 +monorepo,pr2,serviceB,200,40,60,180,440,pipeline2,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,2026-08-01T12:00:00.000+00:00 +monorepo,pr3,serviceA,300,60,90,30,390,pipeline1,2026-08-01T08:30:00.000+00:00,2026-08-01T09:30:00.000+00:00,2026-08-01T10:00:00.000+00:00 +monorepo,pr5,serviceA,500,100,150,,560,,2026-08-03T08:00:00.000+00:00,2026-08-03T09:00:00.000+00:00, diff --git a/backend/plugins/monorepo/impl/impl.go b/backend/plugins/monorepo/impl/impl.go new file mode 100644 index 00000000000..32d5b52d901 --- /dev/null +++ b/backend/plugins/monorepo/impl/impl.go @@ -0,0 +1,170 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/monorepo/models" + "github.com/apache/incubator-devlake/plugins/monorepo/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/monorepo/tasks" +) + +// make sure interface is implemented +var _ interface { + plugin.PluginMeta + plugin.PluginTask + plugin.PluginModel + plugin.PluginMetric + plugin.PluginMigration + plugin.MetricPluginBlueprintV200 +} = (*Monorepo)(nil) + +type Monorepo struct{} + +func (p Monorepo) Description() string { + return "Split a monorepo into sub-projects and compute per-sub-project DORA metrics" +} + +func (p Monorepo) Name() string { + return "monorepo" +} + +func (p Monorepo) Dashboards() []plugin.GrafanaDashboard { + return nil +} + +func (p Monorepo) SvgIcon() string { + return ` + +` +} + +// RequiredDataEntities declares that deployments must be recognisable as CI/CD tasks of +// type Deployment, which is what sub-project attribution matches job names against. +func (p Monorepo) RequiredDataEntities() (data []map[string]interface{}, err errors.Error) { + return []map[string]interface{}{ + { + "model": "cicd_tasks", + "requiredFields": map[string]string{ + "column": "type", + "execptedValue": "Deployment", + }, + }, + }, nil +} + +func (p Monorepo) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.SubProjectDeployment{}, + &models.SubProjectPrMetric{}, + } +} + +func (p Monorepo) IsProjectMetric() bool { + return true +} + +// RunAfter ensures DORA has produced project_pr_metrics (coding/pickup/review times) and +// its deployment records before sub-project attribution reads them. +func (p Monorepo) RunAfter() ([]string, errors.Error) { + return []string{"dora"}, nil +} + +func (p Monorepo) Settings() interface{} { + return nil +} + +func (p Monorepo) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.AttributeDeploymentsMeta, + tasks.AttributePullRequestsMeta, + } +} + +func (p Monorepo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + op, err := tasks.DecodeAndValidateTaskOptions(options) + if err != nil { + return nil, err + } + matcher, err := tasks.NewSubProjectMatcher(op.SubProjects) + if err != nil { + return nil, err + } + return &tasks.MonorepoTaskData{ + Options: op, + Matcher: matcher, + }, nil +} + +// RootPkgPath information lost when compiled as plugin(.so) +func (p Monorepo) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/monorepo" +} + +func (p Monorepo) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Monorepo) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) { + op := &tasks.MonorepoOptions{} + if options != nil && string(options) != "\"\"" { + if err := json.Unmarshal(options, op); err != nil { + return nil, errors.Default.WrapRaw(err) + } + } + if len(op.SubProjects) == 0 { + return nil, errors.BadInput.New( + "the monorepo plugin requires a subProjects list in its metric plugin options") + } + // Validate eagerly so a bad regex is reported when the blueprint is saved rather + // than midway through a pipeline run. + if _, err := tasks.NewSubProjectMatcher(op.SubProjects); err != nil { + return nil, err + } + + subProjects := make([]map[string]interface{}, 0, len(op.SubProjects)) + for _, sp := range op.SubProjects { + subProjects = append(subProjects, map[string]interface{}{ + "name": sp.Name, + "prLabels": sp.PrLabels, + "deployJobPattern": sp.DeployJobPattern, + }) + } + + plan := coreModels.PipelinePlan{ + { + { + Plugin: "monorepo", + Options: map[string]interface{}{ + "projectName": projectName, + "subProjects": subProjects, + }, + Subtasks: []string{ + tasks.AttributeDeploymentsMeta.Name, + tasks.AttributePullRequestsMeta.Name, + }, + }, + }, + } + return plan, nil +} diff --git a/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go b/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go new file mode 100644 index 00000000000..ce485504013 --- /dev/null +++ b/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/monorepo/models" +) + +var _ plugin.MigrationScript = (*addInitTables)(nil) + +type addInitTables struct{} + +func (script *addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &models.SubProjectDeployment{}, + &models.SubProjectPrMetric{}, + ) +} + +func (*addInitTables) Version() uint64 { return 20260809100000 } + +func (*addInitTables) Name() string { + return "create init tables for the monorepo plugin" +} diff --git a/backend/plugins/monorepo/models/migrationscripts/register.go b/backend/plugins/monorepo/models/migrationscripts/register.go new file mode 100644 index 00000000000..ec054748c27 --- /dev/null +++ b/backend/plugins/monorepo/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/monorepo/models/subproject_deployment.go b/backend/plugins/monorepo/models/subproject_deployment.go new file mode 100644 index 00000000000..04f55279a06 --- /dev/null +++ b/backend/plugins/monorepo/models/subproject_deployment.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// SubProjectDeployment attributes a deployment to a single sub-project of a monorepo, +// based on the name of the CI job that performed the deployment. +// +// One deployment may produce several rows when a single pipeline runs the deploy jobs +// of several sub-projects. That is not double counting: each sub-project really was +// deployed by that pipeline. +type SubProjectDeployment struct { + common.NoPKModel + // The four primary key columns are deliberately kept narrow: MySQL caps a composite + // index at 3072 bytes, which is 768 characters under utf8mb4. + ProjectName string `gorm:"primaryKey;type:varchar(100)"` + SubProject string `gorm:"primaryKey;type:varchar(100)"` + // CicdDeploymentId is the id of the deployment (a cicd_pipelines.id when the + // deployment was generated from a pipeline), taken from cicd_deployment_commits. + CicdDeploymentId string `gorm:"primaryKey;type:varchar(255)"` + // CommitSha is wide enough for a SHA-256 hash; the source column is varchar(255) but + // only ever holds a git object id. + CommitSha string `gorm:"primaryKey;type:varchar(64)"` + // JobName is the cicd_tasks.name that matched this sub-project's DeployJobPattern. + JobName string `gorm:"type:varchar(255)"` + Result string `gorm:"type:varchar(100)"` + Environment string `gorm:"type:varchar(255)"` + FinishedDate *time.Time +} + +func (SubProjectDeployment) TableName() string { + return "monorepo_subproject_deployments" +} diff --git a/backend/plugins/monorepo/models/subproject_pr_metric.go b/backend/plugins/monorepo/models/subproject_pr_metric.go new file mode 100644 index 00000000000..ab138230c8d --- /dev/null +++ b/backend/plugins/monorepo/models/subproject_pr_metric.go @@ -0,0 +1,57 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// SubProjectPrMetric holds the change-lead-time breakdown for a merged pull request, +// attributed to exactly one sub-project of a monorepo. +// +// CodingTime/PickupTime/ReviewTime are carried over from DORA's project_pr_metrics: +// they depend only on the pull request itself, so DORA already computes them correctly +// for a monorepo. Only DeployTime (and therefore CycleTime) is recomputed here, against +// the deployments of this sub-project rather than the whole repository's. +// +// All durations are in minutes, matching DORA's convention. +type SubProjectPrMetric struct { + common.NoPKModel + ProjectName string `gorm:"primaryKey;type:varchar(100)"` + PullRequestId string `gorm:"primaryKey;type:varchar(255)"` + SubProject string `gorm:"index;type:varchar(255)"` + + CodingTime *int64 + PickupTime *int64 + ReviewTime *int64 + DeployTime *int64 + CycleTime *int64 + + // DeploymentId is the sub-project deployment this PR was linked to, if any. + DeploymentId string `gorm:"type:varchar(255)"` + + PrCreatedDate *time.Time + PrMergedDate *time.Time + DeployedDate *time.Time +} + +func (SubProjectPrMetric) TableName() string { + return "monorepo_subproject_pr_metrics" +} diff --git a/backend/plugins/monorepo/monorepo.go b/backend/plugins/monorepo/monorepo.go new file mode 100644 index 00000000000..0b23cd9b74a --- /dev/null +++ b/backend/plugins/monorepo/monorepo.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/monorepo/impl" + "github.com/spf13/cobra" +) + +// PluginEntry exports for Framework to search and load +var PluginEntry impl.Monorepo //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "monorepo"} + + projectName := cmd.Flags().StringP("projectName", "p", "", "project name") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("projectName") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "projectName": *projectName, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/monorepo/tasks/deployment_attributor.go b/backend/plugins/monorepo/tasks/deployment_attributor.go new file mode 100644 index 00000000000..e9ac492b612 --- /dev/null +++ b/backend/plugins/monorepo/tasks/deployment_attributor.go @@ -0,0 +1,119 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/monorepo/models" +) + +var AttributeDeploymentsMeta = plugin.SubTaskMeta{ + Name: "attributeDeployments", + EntryPoint: AttributeDeployments, + EnabledByDefault: true, + Description: "Attribute each deployment to a monorepo sub-project by the name of the CI job that deployed it", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, +} + +// deploymentJobRow is one (deployment, deploy job) pair as returned by the query below. +// +// RawDataOrigin is embedded because DataConverter copies that field from the input row +// onto every result; without it the conversion panics. +type deploymentJobRow struct { + common.RawDataOrigin + CicdDeploymentId string + CommitSha string + Result string + Environment string + FinishedDate *time.Time + JobName string +} + +func AttributeDeployments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*MonorepoTaskData) + + // Rebuild from scratch: attribution depends on configuration that may have changed + // since the last run, so stale rows cannot be reconciled incrementally. + if err := db.Exec( + "DELETE FROM monorepo_subproject_deployments WHERE project_name = ?", + data.Options.ProjectName, + ); err != nil { + return errors.Default.Wrap(err, "error deleting previous monorepo_subproject_deployments") + } + + // Only deployments generated from pipelines can be attributed: cicd_deployment_id is + // the pipeline id, which is what cicd_tasks rows hang off. Deployments imported + // straight from a provider's deployment API carry no job and are skipped. + clauses := []dal.Clause{ + dal.Select(`dc.cicd_deployment_id, dc.commit_sha, dc.result, dc.environment, + dc.finished_date, t.name AS job_name`), + dal.From("cicd_deployment_commits dc"), + dal.Join("JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)"), + dal.Join("JOIN cicd_tasks t ON (t.pipeline_id = dc.cicd_deployment_id)"), + dal.Where("pm.project_name = ? AND t.type = ?", data.Options.ProjectName, devops.DEPLOYMENT), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: MonorepoApiParams{ + ProjectName: data.Options.ProjectName, + }, + Table: "cicd_deployment_commits", + }, + InputRowType: reflect.TypeOf(deploymentJobRow{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + row := inputRow.(*deploymentJobRow) + matched := data.Matcher.MatchDeployJob(row.JobName) + results := make([]interface{}, 0, len(matched)) + for _, subProject := range matched { + results = append(results, &models.SubProjectDeployment{ + ProjectName: data.Options.ProjectName, + SubProject: subProject, + CicdDeploymentId: row.CicdDeploymentId, + CommitSha: row.CommitSha, + JobName: row.JobName, + Result: row.Result, + Environment: row.Environment, + FinishedDate: row.FinishedDate, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/monorepo/tasks/pr_attributor.go b/backend/plugins/monorepo/tasks/pr_attributor.go new file mode 100644 index 00000000000..405c80f7219 --- /dev/null +++ b/backend/plugins/monorepo/tasks/pr_attributor.go @@ -0,0 +1,275 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "math" + "reflect" + "sort" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/monorepo/models" +) + +var AttributePullRequestsMeta = plugin.SubTaskMeta{ + Name: "attributePullRequests", + EntryPoint: AttributePullRequests, + EnabledByDefault: true, + Description: "Attribute merged pull requests to monorepo sub-projects by label and compute their change lead time", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW}, +} + +// RawDataOrigin is embedded because DataConverter copies that field from the input row +// onto every result; without it the conversion panics. +type pullRequestRow struct { + common.RawDataOrigin + Id string + CreatedDate time.Time + MergedDate *time.Time +} + +type prLabelRow struct { + PullRequestId string + LabelName string +} + +// deployedAt is the minimum information needed to link a merged PR to a deployment. +type deployedAt struct { + Id string + FinishedDate time.Time +} + +func AttributePullRequests(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + data := taskCtx.GetData().(*MonorepoTaskData) + projectName := data.Options.ProjectName + + if err := db.Exec( + "DELETE FROM monorepo_subproject_pr_metrics WHERE project_name = ?", + projectName, + ); err != nil { + return errors.Default.Wrap(err, "error deleting previous monorepo_subproject_pr_metrics") + } + + labelsByPr, err := loadPrLabels(db, projectName) + if err != nil { + return err + } + // DORA already computes coding/pickup/review time correctly for a monorepo: they + // depend only on the pull request itself. Only the deploy leg needs recomputing. + doraMetrics, err := loadDoraPrMetrics(db, projectName) + if err != nil { + return err + } + deploymentsBySubProject, err := loadSubProjectDeployments(db, projectName) + if err != nil { + return err + } + logger.Info("monorepo: %d labelled PRs, %d DORA metric rows, %d sub-projects with deployments", + len(labelsByPr), len(doraMetrics), len(deploymentsBySubProject)) + + clauses := []dal.Clause{ + dal.Select("pr.id, pr.created_date, pr.merged_date"), + dal.From("pull_requests pr"), + dal.Join("JOIN project_mapping pm ON (pm.table = 'repos' AND pm.row_id = pr.base_repo_id)"), + dal.Where("pm.project_name = ? AND pr.merged_date IS NOT NULL", projectName), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + unattributed := 0 + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: MonorepoApiParams{ + ProjectName: projectName, + }, + Table: "pull_requests", + }, + InputRowType: reflect.TypeOf(pullRequestRow{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + pr := inputRow.(*pullRequestRow) + subProject := data.Matcher.MatchPrLabels(labelsByPr[pr.Id]) + if subProject == "" { + // No sub-project claims this PR; it is simply out of scope here. + unattributed++ + return nil, nil + } + + metric := &models.SubProjectPrMetric{ + ProjectName: projectName, + PullRequestId: pr.Id, + SubProject: subProject, + PrCreatedDate: &pr.CreatedDate, + PrMergedDate: pr.MergedDate, + } + if dm := doraMetrics[pr.Id]; dm != nil { + metric.CodingTime = dm.PrCodingTime + metric.PickupTime = dm.PrPickupTime + metric.ReviewTime = dm.PrReviewTime + } + + if deployment := firstDeploymentAfter(deploymentsBySubProject[subProject], pr.MergedDate); deployment != nil { + metric.DeploymentId = deployment.Id + metric.DeployedDate = &deployment.FinishedDate + metric.DeployTime = computeTimeSpan(pr.MergedDate, &deployment.FinishedDate) + } + + // Mirrors DORA's definition: coding + (merged - created) + deploy. + var cycleTime int64 + if metric.CodingTime != nil { + cycleTime += *metric.CodingTime + } + if prDuring := computeTimeSpan(&pr.CreatedDate, pr.MergedDate); prDuring != nil { + cycleTime += *prDuring + } + if metric.DeployTime != nil { + cycleTime += *metric.DeployTime + } + metric.CycleTime = &cycleTime + + return []interface{}{metric}, nil + }, + }) + if err != nil { + return err + } + + if err := converter.Execute(); err != nil { + return err + } + if unattributed > 0 { + logger.Info("monorepo: %d merged PRs matched no sub-project label and were skipped", unattributed) + } + return nil +} + +// firstDeploymentAfter returns the earliest deployment that finished after mergedDate. +// +// This is an approximation: it assumes a merged change is shipped by the next successful +// production deployment of its sub-project. Hotfixes, cherry-picks, rollbacks and re-runs +// can break that assumption. Exact attribution would need each deployment's commit range +// from the refdiff plugin's commits_diffs table; this function is the seam where that +// swap would happen. +func firstDeploymentAfter(deployments []deployedAt, mergedDate *time.Time) *deployedAt { + if mergedDate == nil || len(deployments) == 0 { + return nil + } + // deployments is sorted by FinishedDate ascending. + i := sort.Search(len(deployments), func(i int) bool { + return deployments[i].FinishedDate.After(*mergedDate) + }) + if i >= len(deployments) { + return nil + } + return &deployments[i] +} + +func loadPrLabels(db dal.Dal, projectName string) (map[string][]string, errors.Error) { + var rows []prLabelRow + err := db.All(&rows, + dal.Select("prl.pull_request_id, prl.label_name"), + dal.From("pull_request_labels prl"), + dal.Join("JOIN pull_requests pr ON (pr.id = prl.pull_request_id)"), + dal.Join("JOIN project_mapping pm ON (pm.table = 'repos' AND pm.row_id = pr.base_repo_id)"), + dal.Where("pm.project_name = ?", projectName), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "error loading pull request labels") + } + byPr := make(map[string][]string) + for _, r := range rows { + byPr[r.PullRequestId] = append(byPr[r.PullRequestId], r.LabelName) + } + return byPr, nil +} + +func loadDoraPrMetrics(db dal.Dal, projectName string) (map[string]*crossdomain.ProjectPrMetric, errors.Error) { + var rows []*crossdomain.ProjectPrMetric + err := db.All(&rows, + dal.From(&crossdomain.ProjectPrMetric{}), + dal.Where("project_name = ?", projectName), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "error loading project_pr_metrics") + } + byPr := make(map[string]*crossdomain.ProjectPrMetric, len(rows)) + for _, r := range rows { + byPr[r.Id] = r + } + return byPr, nil +} + +// loadSubProjectDeployments returns the successful production deployments of each +// sub-project, sorted by finish time so they can be searched by merge date. +func loadSubProjectDeployments(db dal.Dal, projectName string) (map[string][]deployedAt, errors.Error) { + var rows []models.SubProjectDeployment + err := db.All(&rows, + dal.From(&models.SubProjectDeployment{}), + dal.Where( + "project_name = ? AND result = ? AND environment = ? AND finished_date IS NOT NULL", + projectName, devops.RESULT_SUCCESS, devops.PRODUCTION, + ), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "error loading monorepo_subproject_deployments") + } + bySubProject := make(map[string][]deployedAt) + for _, r := range rows { + bySubProject[r.SubProject] = append(bySubProject[r.SubProject], deployedAt{ + Id: r.CicdDeploymentId, + FinishedDate: *r.FinishedDate, + }) + } + for name := range bySubProject { + list := bySubProject[name] + sort.Slice(list, func(i, j int) bool { + return list[i].FinishedDate.Before(list[j].FinishedDate) + }) + bySubProject[name] = list + } + return bySubProject, nil +} + +// computeTimeSpan returns the whole minutes between start and end, or nil when either is +// missing or the span is negative. Mirrors the identical unexported helper in the DORA +// plugin (plugins/dora/tasks/change_lead_time_calculator.go) so the two produce the same +// numbers; it cannot be imported because it is not exported there. +func computeTimeSpan(start, end *time.Time) *int64 { + if start == nil || end == nil { + return nil + } + span := end.Sub(*start) + minutes := int64(math.Ceil(span.Minutes())) + if minutes < 0 { + return nil + } + return &minutes +} diff --git a/backend/plugins/monorepo/tasks/pr_attributor_test.go b/backend/plugins/monorepo/tasks/pr_attributor_test.go new file mode 100644 index 00000000000..c716d005a56 --- /dev/null +++ b/backend/plugins/monorepo/tasks/pr_attributor_test.go @@ -0,0 +1,125 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func at(hour int) time.Time { + return time.Date(2026, 8, 9, hour, 0, 0, 0, time.UTC) +} + +func TestFirstDeploymentAfter(t *testing.T) { + // Sorted ascending, as loadSubProjectDeployments guarantees. + deployments := []deployedAt{ + {Id: "deploy-08", FinishedDate: at(8)}, + {Id: "deploy-12", FinishedDate: at(12)}, + {Id: "deploy-18", FinishedDate: at(18)}, + } + + cases := []struct { + name string + deployments []deployedAt + mergedDate *time.Time + expectedId string + }{ + { + name: "picks the earliest deployment after the merge", + deployments: deployments, + mergedDate: ptrTime(at(10)), + expectedId: "deploy-12", + }, + { + name: "merge before every deployment picks the first", + deployments: deployments, + mergedDate: ptrTime(at(1)), + expectedId: "deploy-08", + }, + { + name: "merge after every deployment has none to link", + deployments: deployments, + mergedDate: ptrTime(at(20)), + expectedId: "", + }, + { + name: "a deployment finishing exactly at merge time does not count", + deployments: deployments, + mergedDate: ptrTime(at(12)), + expectedId: "deploy-18", + }, + { + name: "no deployments at all", + deployments: nil, + mergedDate: ptrTime(at(10)), + expectedId: "", + }, + { + name: "unmerged pull request", + deployments: deployments, + mergedDate: nil, + expectedId: "", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := firstDeploymentAfter(tc.deployments, tc.mergedDate) + if tc.expectedId == "" { + assert.Nil(t, got) + return + } + assert.NotNil(t, got) + assert.Equal(t, tc.expectedId, got.Id) + }) + } +} + +func TestComputeTimeSpan(t *testing.T) { + start := at(10) + end := at(12) + + t.Run("whole minutes between two times", func(t *testing.T) { + got := computeTimeSpan(&start, &end) + assert.NotNil(t, got) + assert.Equal(t, int64(120), *got) + }) + + t.Run("partial minutes round up", func(t *testing.T) { + later := start.Add(90 * time.Second) + got := computeTimeSpan(&start, &later) + assert.NotNil(t, got) + assert.Equal(t, int64(2), *got) + }) + + t.Run("negative spans are discarded", func(t *testing.T) { + assert.Nil(t, computeTimeSpan(&end, &start)) + }) + + t.Run("missing endpoints yield nil", func(t *testing.T) { + assert.Nil(t, computeTimeSpan(nil, &end)) + assert.Nil(t, computeTimeSpan(&start, nil)) + assert.Nil(t, computeTimeSpan(nil, nil)) + }) +} + +func ptrTime(t time.Time) *time.Time { + return &t +} diff --git a/backend/plugins/monorepo/tasks/task_data.go b/backend/plugins/monorepo/tasks/task_data.go new file mode 100644 index 00000000000..26b57b390f6 --- /dev/null +++ b/backend/plugins/monorepo/tasks/task_data.go @@ -0,0 +1,154 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "regexp" + + "github.com/apache/incubator-devlake/core/errors" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +type MonorepoApiParams struct { + ProjectName string +} + +// SubProjectConfig declares one logical project living inside a monorepo. +type SubProjectConfig struct { + // Name identifies the sub-project in the output tables and dashboards. + Name string `json:"name" mapstructure:"name"` + // PrLabels are the pull request labels that mark a PR as belonging to this + // sub-project. Matching is exact and case-sensitive. + PrLabels []string `json:"prLabels" mapstructure:"prLabels"` + // DeployJobPattern is a regular expression matched against cicd_tasks.name to + // recognise this sub-project's deployment jobs, e.g. "^deploy-serviceA$". + DeployJobPattern string `json:"deployJobPattern" mapstructure:"deployJobPattern"` +} + +type MonorepoOptions struct { + ProjectName string `json:"projectName" mapstructure:"projectName"` + // SubProjects is ordered: when a pull request carries the labels of more than one + // sub-project, the earliest entry in this list wins. + SubProjects []SubProjectConfig `json:"subProjects" mapstructure:"subProjects"` +} + +type MonorepoTaskData struct { + Options *MonorepoOptions + Matcher *SubProjectMatcher +} + +// SubProjectMatcher resolves deployments and pull requests to sub-projects. It holds +// the compiled form of the configuration so the regexes are built once per task rather +// than once per row. +type SubProjectMatcher struct { + names []string + prLabels []map[string]struct{} + deployJobRes []*regexp.Regexp +} + +// NewSubProjectMatcher compiles the sub-project configuration, validating it along the way. +func NewSubProjectMatcher(subProjects []SubProjectConfig) (*SubProjectMatcher, errors.Error) { + m := &SubProjectMatcher{ + names: make([]string, 0, len(subProjects)), + prLabels: make([]map[string]struct{}, 0, len(subProjects)), + deployJobRes: make([]*regexp.Regexp, 0, len(subProjects)), + } + seen := make(map[string]struct{}, len(subProjects)) + for i, sp := range subProjects { + if sp.Name == "" { + return nil, errors.BadInput.New(fmt.Sprintf("subProjects[%d]: name is required", i)) + } + if _, dup := seen[sp.Name]; dup { + return nil, errors.BadInput.New(fmt.Sprintf("subProjects[%d]: duplicate name %q", i, sp.Name)) + } + seen[sp.Name] = struct{}{} + + var jobRe *regexp.Regexp + if sp.DeployJobPattern != "" { + compiled, err := regexp.Compile(sp.DeployJobPattern) + if err != nil { + return nil, errors.BadInput.Wrap(err, fmt.Sprintf( + "subProjects[%d] (%s): invalid deployJobPattern", i, sp.Name)) + } + jobRe = compiled + } + + labels := make(map[string]struct{}, len(sp.PrLabels)) + for _, l := range sp.PrLabels { + if l != "" { + labels[l] = struct{}{} + } + } + + m.names = append(m.names, sp.Name) + m.prLabels = append(m.prLabels, labels) + m.deployJobRes = append(m.deployJobRes, jobRe) + } + return m, nil +} + +// MatchDeployJob returns every sub-project whose DeployJobPattern matches jobName. +// +// More than one match is possible and is reported faithfully: a single pipeline running +// both deploy-serviceA and deploy-serviceB genuinely deploys two sub-projects. If a +// single job name matches two patterns, that indicates overlapping configuration. +func (m *SubProjectMatcher) MatchDeployJob(jobName string) []string { + var matched []string + for i, re := range m.deployJobRes { + if re != nil && re.MatchString(jobName) { + matched = append(matched, m.names[i]) + } + } + return matched +} + +// MatchPrLabels returns the single sub-project a pull request belongs to, or "" when no +// sub-project claims it. When several sub-projects match, the earliest one in the +// configured order wins — labels carry no size signal that could rank them otherwise. +func (m *SubProjectMatcher) MatchPrLabels(labels []string) string { + if len(labels) == 0 { + return "" + } + present := make(map[string]struct{}, len(labels)) + for _, l := range labels { + present[l] = struct{}{} + } + for i, wanted := range m.prLabels { + for l := range wanted { + if _, ok := present[l]; ok { + return m.names[i] + } + } + } + return "" +} + +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*MonorepoOptions, errors.Error) { + var op MonorepoOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, errors.Default.Wrap(err, "error decoding monorepo task options") + } + if op.ProjectName == "" { + return nil, errors.BadInput.New("projectName is required for the monorepo plugin") + } + if len(op.SubProjects) == 0 { + return nil, errors.BadInput.New("at least one entry in subProjects is required for the monorepo plugin") + } + return &op, nil +} diff --git a/backend/plugins/monorepo/tasks/task_data_test.go b/backend/plugins/monorepo/tasks/task_data_test.go new file mode 100644 index 00000000000..2d6affcca05 --- /dev/null +++ b/backend/plugins/monorepo/tasks/task_data_test.go @@ -0,0 +1,220 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// twoServices is the canonical monorepo configuration used across these tests: +// serviceA is declared first, so it wins any tie. +func twoServices() []SubProjectConfig { + return []SubProjectConfig{ + { + Name: "serviceA", + PrLabels: []string{"serviceA"}, + DeployJobPattern: "^deploy-serviceA$", + }, + { + Name: "serviceB", + PrLabels: []string{"serviceB", "svc-b"}, + DeployJobPattern: "^deploy-serviceB$", + }, + } +} + +func TestMatchDeployJob(t *testing.T) { + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + + cases := []struct { + name string + jobName string + expected []string + }{ + {"matches serviceA", "deploy-serviceA", []string{"serviceA"}}, + {"matches serviceB", "deploy-serviceB", []string{"serviceB"}}, + {"build job is not a deployment", "build-serviceA", nil}, + {"unrelated job matches nothing", "run-tests", nil}, + {"anchored pattern rejects a superstring", "deploy-serviceAB", nil}, + {"empty job name matches nothing", "", nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, matcher.MatchDeployJob(tc.jobName)) + }) + } +} + +// A pipeline that runs both services' deploy jobs produces a row for each. The two jobs +// arrive as separate rows, so each resolves to exactly one sub-project. +func TestMatchDeployJob_PipelineDeployingBothServices(t *testing.T) { + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + + assert.Equal(t, []string{"serviceA"}, matcher.MatchDeployJob("deploy-serviceA")) + assert.Equal(t, []string{"serviceB"}, matcher.MatchDeployJob("deploy-serviceB")) +} + +// Overlapping patterns are reported faithfully rather than silently resolved, so a +// misconfiguration is visible in the data instead of hidden. +func TestMatchDeployJob_OverlappingPatterns(t *testing.T) { + matcher, err := NewSubProjectMatcher([]SubProjectConfig{ + {Name: "serviceA", DeployJobPattern: "deploy"}, + {Name: "serviceB", DeployJobPattern: "^deploy-serviceB$"}, + }) + assert.Nil(t, err) + + assert.Equal(t, []string{"serviceA", "serviceB"}, matcher.MatchDeployJob("deploy-serviceB")) +} + +func TestMatchDeployJob_NoPatternNeverMatches(t *testing.T) { + matcher, err := NewSubProjectMatcher([]SubProjectConfig{ + {Name: "labelsOnly", PrLabels: []string{"labelsOnly"}}, + }) + assert.Nil(t, err) + + assert.Nil(t, matcher.MatchDeployJob("deploy-labelsOnly")) +} + +func TestMatchPrLabels(t *testing.T) { + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + + cases := []struct { + name string + labels []string + expected string + }{ + {"single matching label", []string{"serviceA"}, "serviceA"}, + {"alias label resolves to its sub-project", []string{"svc-b"}, "serviceB"}, + {"matching label among unrelated ones", []string{"bug", "serviceB", "urgent"}, "serviceB"}, + {"no matching label", []string{"bug", "urgent"}, ""}, + {"no labels at all", nil, ""}, + {"empty label slice", []string{}, ""}, + {"matching is case sensitive", []string{"servicea"}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, matcher.MatchPrLabels(tc.labels)) + }) + } +} + +// A PR labelled for several sub-projects is assigned to exactly one: the earliest in +// configuration order. Labels carry no size signal, so declaration order is the tie-break. +func TestMatchPrLabels_TieBreakIsConfigOrder(t *testing.T) { + both := []string{"serviceB", "serviceA"} + + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + assert.Equal(t, "serviceA", matcher.MatchPrLabels(both)) + + // Reversing the configuration reverses the winner, proving order drives the result + // rather than the order of labels on the PR. + reversed := []SubProjectConfig{twoServices()[1], twoServices()[0]} + reversedMatcher, err := NewSubProjectMatcher(reversed) + assert.Nil(t, err) + assert.Equal(t, "serviceB", reversedMatcher.MatchPrLabels(both)) +} + +func TestNewSubProjectMatcher_Validation(t *testing.T) { + cases := []struct { + name string + subProjects []SubProjectConfig + expectErr bool + }{ + { + name: "valid configuration", + subProjects: twoServices(), + }, + { + name: "empty configuration is allowed here, rejected by option decoding", + subProjects: nil, + }, + { + name: "missing name", + subProjects: []SubProjectConfig{{PrLabels: []string{"x"}}}, + expectErr: true, + }, + { + name: "duplicate names", + subProjects: []SubProjectConfig{ + {Name: "serviceA", DeployJobPattern: "^a$"}, + {Name: "serviceA", DeployJobPattern: "^b$"}, + }, + expectErr: true, + }, + { + name: "invalid deploy job regex", + subProjects: []SubProjectConfig{{Name: "serviceA", DeployJobPattern: "^deploy-(unclosed"}}, + expectErr: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + matcher, err := NewSubProjectMatcher(tc.subProjects) + if tc.expectErr { + assert.NotNil(t, err) + assert.Nil(t, matcher) + return + } + assert.Nil(t, err) + assert.NotNil(t, matcher) + }) + } +} + +func TestDecodeAndValidateTaskOptions(t *testing.T) { + t.Run("valid options", func(t *testing.T) { + op, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "projectName": "monorepo", + "subProjects": []interface{}{ + map[string]interface{}{ + "name": "serviceA", + "prLabels": []interface{}{"serviceA"}, + "deployJobPattern": "^deploy-serviceA$", + }, + }, + }) + assert.Nil(t, err) + assert.Equal(t, "monorepo", op.ProjectName) + assert.Len(t, op.SubProjects, 1) + assert.Equal(t, "serviceA", op.SubProjects[0].Name) + assert.Equal(t, []string{"serviceA"}, op.SubProjects[0].PrLabels) + assert.Equal(t, "^deploy-serviceA$", op.SubProjects[0].DeployJobPattern) + }) + + t.Run("missing projectName is rejected", func(t *testing.T) { + _, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "subProjects": []interface{}{ + map[string]interface{}{"name": "serviceA"}, + }, + }) + assert.NotNil(t, err) + }) + + t.Run("missing subProjects is rejected", func(t *testing.T) { + _, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "projectName": "monorepo", + }) + assert.NotNil(t, err) + }) +} diff --git a/grafana/dashboards/mysql/monorepo-subprojects.json b/grafana/dashboards/mysql/monorepo-subprojects.json new file mode 100644 index 00000000000..60d1fec5f70 --- /dev/null +++ b/grafana/dashboards/mysql/monorepo-subprojects.json @@ -0,0 +1,340 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "datasource", "uid": "grafana" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Per-sub-project DORA metrics for a monorepo, produced by the monorepo plugin. Deployments are attributed by CI job name, pull requests by label.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { "type": "datasource", "uid": "grafana" }, + "gridPos": { "h": 3, "w": 24, "x": 0, "y": 0 }, + "id": 10, + "options": { + "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, + "content": "## Monorepo Sub-Projects\n\nEach **sub-project** is a logical project inside a single Git repository. Deployments are attributed by **CI job name**, pull requests by **label**. Only Deployment Frequency and Lead Time for Changes are available — Change Failure Rate and Time to Restore require incident data, which this plugin does not attribute.", + "mode": "markdown" + }, + "pluginVersion": "10.1.0", + "title": "", + "type": "text" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Number of successful production deployments per sub-project in the selected time range. A sub-project showing zero usually means its deployJobPattern matches no CI job.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "deployments", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 3 }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { "mode": "single", "sort": "none" }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT sub_project AS 'Sub-Project', COUNT(DISTINCT cicd_deployment_id) AS 'Deployments'\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nGROUP BY sub_project\nORDER BY 2 DESC", + "refId": "A", + "sql": { "columns": [{ "parameters": [], "type": "function" }], "groupBy": [{ "property": { "type": "string" }, "type": "groupBy" }], "limit": 50 } + } + ], + "title": "Deployment Count by Sub-Project", + "type": "barchart" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Successful production deployments over time, one series per sub-project. This is the metric that a plain DevLake setup cannot separate: without attribution every sub-project shows the whole repository's deployment count.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "deployments", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 3 }, + "id": 2, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(finished_date) AS time,\n sub_project AS metric,\n COUNT(DISTINCT cicd_deployment_id) AS value\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nGROUP BY 1, 2\nORDER BY 1", + "refId": "A" + } + ], + "title": "Deployment Frequency over Time", + "type": "timeseries" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Average change lead time per sub-project, broken into its stages. Coding, pickup and review come from DORA unchanged; the deploy leg is recomputed against this sub-project's own deployments. All values in hours.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { + "align": "auto", + "cellOptions": { "type": "auto" }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "noValue": "-", + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Cycle Time (h)" }, + "properties": [ + { "id": "custom.cellOptions", "value": { "type": "color-text" } }, + { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } + ] + } + ] + }, + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 12 }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n sub_project AS 'Sub-Project',\n COUNT(*) AS 'PRs',\n ROUND(AVG(coding_time) / 60, 1) AS 'Coding (h)',\n ROUND(AVG(pickup_time) / 60, 1) AS 'Pickup (h)',\n ROUND(AVG(review_time) / 60, 1) AS 'Review (h)',\n ROUND(AVG(deploy_time) / 60, 1) AS 'Deploy (h)',\n ROUND(AVG(cycle_time) / 60, 1) AS 'Cycle Time (h)'\nFROM monorepo_subproject_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)\nGROUP BY sub_project\nORDER BY 1", + "refId": "A" + } + ], + "title": "Change Lead Time Breakdown by Sub-Project", + "type": "table" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Merged pull requests carrying none of the configured sub-project labels. These are invisible to every metric on this dashboard. A rising number means labelling discipline is slipping, not that the code is wrong.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "#EAB839", "value": 1 }, + { "color": "red", "value": 10 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 12 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS 'Unattributed merged PRs'\nFROM pull_requests pr\nJOIN project_mapping pm\n ON pm.table = 'repos' AND pm.row_id = pr.base_repo_id\nLEFT JOIN monorepo_subproject_pr_metrics m\n ON m.pull_request_id = pr.id AND m.project_name = pm.project_name\nWHERE pr.merged_date IS NOT NULL\n AND m.pull_request_id IS NULL\n AND ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(pr.merged_date)", + "refId": "A" + } + ], + "title": "Unattributed PRs", + "type": "stat" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Merged pull requests that have been attributed to a sub-project but that no deployment has shipped yet, or whose shipping deployment could not be identified.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 12 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS 'Merged, not yet deployed'\nFROM monorepo_subproject_pr_metrics\nWHERE deployment_id = ''\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)", + "refId": "A" + } + ], + "title": "Awaiting Deployment", + "type": "stat" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Every attributed deployment, newest first. The job name column shows which CI job caused the attribution — useful for checking a deployJobPattern is matching what you expect.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true, "inspect": false }, + "mappings": [], + "noValue": "-", + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Result" }, + "properties": [ + { "id": "custom.cellOptions", "value": { "type": "color-text" } }, + { + "id": "mappings", + "value": [ + { "options": { "SUCCESS": { "color": "green", "index": 0 } }, "type": "value" }, + { "options": { "FAILURE": { "color": "red", "index": 1 } }, "type": "value" } + ] + } + ] + } + ] + }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 21 }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Finished" }] + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n finished_date AS 'Finished',\n sub_project AS 'Sub-Project',\n job_name AS 'CI Job',\n environment AS 'Environment',\n result AS 'Result',\n cicd_deployment_id AS 'Deployment',\n commit_sha AS 'Commit'\nFROM monorepo_subproject_deployments\nWHERE ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nORDER BY finished_date DESC\nLIMIT 200", + "refId": "A" + } + ], + "title": "Attributed Deployments", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": ["monorepo", "dora"], + "templating": { + "list": [ + { + "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "definition": "select distinct name from projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "select distinct name from projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { "from": "now-90d", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Monorepo Sub-Projects", + "uid": "monorepo-subprojects", + "version": 1, + "weekStart": "" +} diff --git a/grafana/dashboards/postgresql/monorepo-subprojects.json b/grafana/dashboards/postgresql/monorepo-subprojects.json new file mode 100644 index 00000000000..d9a6f7086b3 --- /dev/null +++ b/grafana/dashboards/postgresql/monorepo-subprojects.json @@ -0,0 +1,628 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Per-sub-project DORA metrics for a monorepo, produced by the monorepo plugin. Deployments are attributed by CI job name, pull requests by label.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "## Monorepo Sub-Projects\n\nEach **sub-project** is a logical project inside a single Git repository. Deployments are attributed by **CI job name**, pull requests by **label**. Only Deployment Frequency and Lead Time for Changes are available \u2014 Change Failure Rate and Time to Restore require incident data, which this plugin does not attribute.", + "mode": "markdown" + }, + "pluginVersion": "10.1.0", + "title": "", + "type": "text" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Number of successful production deployments per sub-project in the selected time range. A sub-project showing zero usually means its deployJobPattern matches no CI job.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "deployments", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT sub_project AS \"Sub-Project\", COUNT(DISTINCT cicd_deployment_id) AS \"Deployments\"\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nGROUP BY sub_project\nORDER BY 2 DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Deployment Count by Sub-Project", + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Successful production deployments over time, one series per sub-project. This is the metric that a plain DevLake setup cannot separate: without attribution every sub-project shows the whole repository's deployment count.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "deployments", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date_trunc('day', finished_date) AS time,\n sub_project AS metric,\n COUNT(DISTINCT cicd_deployment_id) AS value\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nGROUP BY 1, 2\nORDER BY 1", + "refId": "A" + } + ], + "title": "Deployment Frequency over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Average change lead time per sub-project, broken into its stages. Coding, pickup and review come from DORA unchanged; the deploy leg is recomputed against this sub-project's own deployments. All values in hours.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Cycle Time (h)" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n sub_project AS \"Sub-Project\",\n COUNT(*) AS \"PRs\",\n ROUND((AVG(coding_time) / 60)::numeric, 1) AS \"Coding (h)\",\n ROUND((AVG(pickup_time) / 60)::numeric, 1) AS \"Pickup (h)\",\n ROUND((AVG(review_time) / 60)::numeric, 1) AS \"Review (h)\",\n ROUND((AVG(deploy_time) / 60)::numeric, 1) AS \"Deploy (h)\",\n ROUND((AVG(cycle_time) / 60)::numeric, 1) AS \"Cycle Time (h)\"\nFROM monorepo_subproject_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)\nGROUP BY sub_project\nORDER BY 1", + "refId": "A" + } + ], + "title": "Change Lead Time Breakdown by Sub-Project", + "type": "table" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Merged pull requests carrying none of the configured sub-project labels. These are invisible to every metric on this dashboard. A rising number means labelling discipline is slipping, not that the code is wrong.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS \"Unattributed merged PRs\"\nFROM pull_requests pr\nJOIN project_mapping pm\n ON pm.\"table\" = 'repos' AND pm.row_id = pr.base_repo_id\nLEFT JOIN monorepo_subproject_pr_metrics m\n ON m.pull_request_id = pr.id AND m.project_name = pm.project_name\nWHERE pr.merged_date IS NOT NULL\n AND m.pull_request_id IS NULL\n AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr.merged_date)", + "refId": "A" + } + ], + "title": "Unattributed PRs", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Merged pull requests that have been attributed to a sub-project but that no deployment has shipped yet, or whose shipping deployment could not be identified.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS \"Merged, not yet deployed\"\nFROM monorepo_subproject_pr_metrics\nWHERE deployment_id = ''\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)", + "refId": "A" + } + ], + "title": "Awaiting Deployment", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Every attributed deployment, newest first. The job name column shows which CI job caused the attribution \u2014 useful for checking a deployJobPattern is matching what you expect.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Result" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "SUCCESS": { + "color": "green", + "index": 0 + } + }, + "type": "value" + }, + { + "options": { + "FAILURE": { + "color": "red", + "index": 1 + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Finished" + } + ] + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n finished_date AS \"Finished\",\n sub_project AS \"Sub-Project\",\n job_name AS \"CI Job\",\n environment AS \"Environment\",\n result AS \"Result\",\n cicd_deployment_id AS \"Deployment\",\n commit_sha AS \"Commit\"\nFROM monorepo_subproject_deployments\nWHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nORDER BY finished_date DESC\nLIMIT 200", + "refId": "A" + } + ], + "title": "Attributed Deployments", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "monorepo", + "dora" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "definition": "select distinct name from projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "select distinct name from projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Monorepo Sub-Projects", + "uid": "monorepo-subprojects", + "version": 1, + "weekStart": "" +} From 37a0b6e9a0a71f2030e328e97e158b652ae6a5fb Mon Sep 17 00:00:00 2001 From: Uriah Rokach Date: Sun, 16 Aug 2026 10:58:36 +0300 Subject: [PATCH 02/14] feat(monorepo): add config-ui panel for editing sub-project settings Adds a Project Settings panel for the monorepo plugin, letting users define sub-projects (name, PR labels, deploy job pattern) directly in config-ui rather than only via the raw API, following the existing pattern used for the linker plugin's fields. --- .../routes/project/detail/settings-panel.tsx | 109 ++++++++++++++++++ config-ui/src/routes/project/detail/styled.ts | 13 +++ 2 files changed, 122 insertions(+) diff --git a/config-ui/src/routes/project/detail/settings-panel.tsx b/config-ui/src/routes/project/detail/settings-panel.tsx index b7a78946466..4302f226430 100644 --- a/config-ui/src/routes/project/detail/settings-panel.tsx +++ b/config-ui/src/routes/project/detail/settings-panel.tsx @@ -18,6 +18,7 @@ import { useEffect, useState } from 'react'; import { useNavigate } from 'react-router-dom'; +import { CloseOutlined, PlusOutlined } from '@ant-design/icons'; import { Flex, Space, Card, Modal, Input, Checkbox, Button } from 'antd'; import API from '@/api'; @@ -30,6 +31,15 @@ import * as S from './styled'; const RegexPrIssueDefaultValue = '(?mi)(Closes)[\\s]*.*(((and )?#\\d+[ ]*)+)'; +interface ISubProject { + name: string; + // Comma-separated in the UI; split into an array on save. + prLabels: string; + deployJobPattern: string; +} + +const emptySubProject: ISubProject = { name: '', prLabels: '', deployJobPattern: '' }; + interface Props { project: IProject; onRefresh: () => void; @@ -47,6 +57,10 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { const [issueTrace, setIssueTrace] = useState({ enable: false, }); + const [monorepo, setMonorepo] = useState<{ enable: boolean; subProjects: ISubProject[] }>({ + enable: false, + subProjects: [emptySubProject], + }); const [operating, setOperating] = useState(false); const [open, setOpen] = useState(false); @@ -56,6 +70,7 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { const dora = project.metrics.find((ms) => ms.pluginName === 'dora'); const linker = project.metrics.find((ms) => ms.pluginName === 'linker'); const issueTrace = project.metrics.find((ms) => ms.pluginName === 'issue_trace'); + const monorepo = project.metrics.find((ms) => ms.pluginName === 'monorepo'); setName(project.name); setDora({ @@ -68,8 +83,35 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { setIssueTrace({ enable: issueTrace?.enable ?? false, }); + const subProjects = monorepo?.pluginOption?.subProjects; + setMonorepo({ + enable: monorepo?.enable ?? false, + subProjects: + Array.isArray(subProjects) && subProjects.length + ? subProjects.map((sp: any) => ({ + name: sp.name ?? '', + prLabels: Array.isArray(sp.prLabels) ? sp.prLabels.join(',') : '', + deployJobPattern: sp.deployJobPattern ?? '', + })) + : [emptySubProject], + }); }, [project]); + const handleAddSubProject = () => { + setMonorepo({ ...monorepo, subProjects: [...monorepo.subProjects, { ...emptySubProject }] }); + }; + + const handleDeleteSubProject = (index: number) => { + setMonorepo({ ...monorepo, subProjects: monorepo.subProjects.filter((_, i) => i !== index) }); + }; + + const handleUpdateSubProject = (index: number, field: keyof ISubProject, value: string) => { + setMonorepo({ + ...monorepo, + subProjects: monorepo.subProjects.map((sp, i) => (i === index ? { ...sp, [field]: value } : sp)), + }); + }; + const handleUpdate = async () => { const [success] = await operator( () => @@ -94,6 +136,22 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { pluginOption: {}, enable: issueTrace.enable, }, + { + pluginName: 'monorepo', + pluginOption: { + subProjects: monorepo.subProjects + .filter((sp) => sp.name.trim()) + .map((sp) => ({ + name: sp.name.trim(), + prLabels: sp.prLabels + .split(',') + .map((l) => l.trim()) + .filter((l) => l), + deployJobPattern: sp.deployJobPattern.trim(), + })), + }, + enable: monorepo.enable, + }, ], }), { @@ -191,6 +249,57 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { } description="Parse the issue status and assignee history from issue changelogs. Currently, only Jira issues are supported." /> + setMonorepo({ ...monorepo, enable: e.target.checked })} + > + Enable Monorepo Sub-Projects + + } + description={ + + Split a single repository into several logical sub-projects for DORA-style metrics. Deployments are + matched by CI job name, pull requests by label. When a pull request carries more than one + sub-project's label, the first matching sub-project in the list below wins. + + + } + > + {monorepo.enable && ( + + {monorepo.subProjects.map((sp, i) => ( +
+ handleUpdateSubProject(i, 'name', e.target.value)} + /> + handleUpdateSubProject(i, 'prLabels', e.target.value)} + /> + handleUpdateSubProject(i, 'deployJobPattern', e.target.value)} + /> + {monorepo.subProjects.length > 1 && ( +
+ ))} + + + )} + + {reservedSubProjectName && ( + + )} )} - From 80816002fed4cef0e71a8b26d2535eb9648a20a1 Mon Sep 17 00:00:00 2001 From: Yotam Soudry Date: Thu, 27 Aug 2026 23:21:31 +0300 Subject: [PATCH 08/14] feat(dashboards): migrate monorepo-subprojects dashboard to core tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites all 6 panels of monorepo-subprojects.json (MySQL and PostgreSQL) to read from the new core tables (cicd_deployment_commits joined through cicd_deployment_subprojects, and project_pr_metrics) instead of the deprecated monorepo_subproject_deployments/monorepo_subproject_pr_metrics compat tables, per design §5.2's recommendation to migrate this dashboard rather than keep it on the compat tables. Deployment/PR-count panels use COALESCE(sub_project, 'All') so single-repo (non-monorepo) projects still render a sensible group. The "Unattributed merged PRs" panel simplifies to a direct pull_requests.sub_project = 'unattributed' filter now that attribution lives on the PR row itself. The "Attributed Deployments" detail table drops its CI-job-name column, since that information isn't reconstructable from the mapping table alone without re-deriving the regex match in SQL; its description is updated to say so. Note: this is the one dashboard fully migrated in this change. The broader dashboard file list in design §5.2 (Gitlab.json, dora-details-*, engineering-throughput-and-cycle-time*, engineering-overview.json, dora-by-team.json) was surveyed but deliberately NOT modified - see the implementation report for why. Co-Authored-By: Claude Sonnet 5 --- grafana/dashboards/mysql/monorepo-subprojects.json | 14 +++++++------- .../postgresql/monorepo-subprojects.json | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/grafana/dashboards/mysql/monorepo-subprojects.json b/grafana/dashboards/mysql/monorepo-subprojects.json index 60d1fec5f70..0ad2757d998 100644 --- a/grafana/dashboards/mysql/monorepo-subprojects.json +++ b/grafana/dashboards/mysql/monorepo-subprojects.json @@ -76,7 +76,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT sub_project AS 'Sub-Project', COUNT(DISTINCT cicd_deployment_id) AS 'Deployments'\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nGROUP BY sub_project\nORDER BY 2 DESC", + "rawSql": "SELECT COALESCE(ds.sub_project, 'All') AS 'Sub-Project', COUNT(DISTINCT dc.cicd_deployment_id) AS 'Deployments'\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.`table` = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE dc.result = 'SUCCESS'\n AND dc.environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(dc.finished_date)\nGROUP BY 1\nORDER BY 2 DESC", "refId": "A", "sql": { "columns": [{ "parameters": [], "type": "function" }], "groupBy": [{ "property": { "type": "string" }, "type": "groupBy" }], "limit": 50 } } @@ -126,7 +126,7 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE(finished_date) AS time,\n sub_project AS metric,\n COUNT(DISTINCT cicd_deployment_id) AS value\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nGROUP BY 1, 2\nORDER BY 1", + "rawSql": "SELECT\n DATE(dc.finished_date) AS time,\n COALESCE(ds.sub_project, 'All') AS metric,\n COUNT(DISTINCT dc.cicd_deployment_id) AS value\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.`table` = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE dc.result = 'SUCCESS'\n AND dc.environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(dc.finished_date)\nGROUP BY 1, 2\nORDER BY 1", "refId": "A" } ], @@ -172,7 +172,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n sub_project AS 'Sub-Project',\n COUNT(*) AS 'PRs',\n ROUND(AVG(coding_time) / 60, 1) AS 'Coding (h)',\n ROUND(AVG(pickup_time) / 60, 1) AS 'Pickup (h)',\n ROUND(AVG(review_time) / 60, 1) AS 'Review (h)',\n ROUND(AVG(deploy_time) / 60, 1) AS 'Deploy (h)',\n ROUND(AVG(cycle_time) / 60, 1) AS 'Cycle Time (h)'\nFROM monorepo_subproject_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)\nGROUP BY sub_project\nORDER BY 1", + "rawSql": "SELECT\n COALESCE(sub_project, 'All') AS 'Sub-Project',\n COUNT(*) AS 'PRs',\n ROUND(AVG(pr_coding_time) / 60, 1) AS 'Coding (h)',\n ROUND(AVG(pr_pickup_time) / 60, 1) AS 'Pickup (h)',\n ROUND(AVG(pr_review_time) / 60, 1) AS 'Review (h)',\n ROUND(AVG(pr_deploy_time) / 60, 1) AS 'Deploy (h)',\n ROUND(AVG(pr_cycle_time) / 60, 1) AS 'Cycle Time (h)'\nFROM project_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)\nGROUP BY 1\nORDER BY 1", "refId": "A" } ], @@ -214,7 +214,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS 'Unattributed merged PRs'\nFROM pull_requests pr\nJOIN project_mapping pm\n ON pm.table = 'repos' AND pm.row_id = pr.base_repo_id\nLEFT JOIN monorepo_subproject_pr_metrics m\n ON m.pull_request_id = pr.id AND m.project_name = pm.project_name\nWHERE pr.merged_date IS NOT NULL\n AND m.pull_request_id IS NULL\n AND ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(pr.merged_date)", + "rawSql": "SELECT COUNT(*) AS 'Unattributed merged PRs'\nFROM pull_requests pr\nJOIN project_mapping pm ON (pm.`table` = 'repos' AND pm.row_id = pr.base_repo_id)\nWHERE pr.merged_date IS NOT NULL\n AND pr.sub_project = 'unattributed'\n AND ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(pr.merged_date)", "refId": "A" } ], @@ -249,7 +249,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS 'Merged, not yet deployed'\nFROM monorepo_subproject_pr_metrics\nWHERE deployment_id = ''\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)", + "rawSql": "SELECT COUNT(*) AS 'Merged, not yet deployed'\nFROM project_pr_metrics\nWHERE (deployment_commit_id = '' OR deployment_commit_id IS NULL)\n AND sub_project IS NOT NULL\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)", "refId": "A" } ], @@ -258,7 +258,7 @@ }, { "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, - "description": "Every attributed deployment, newest first. The job name column shows which CI job caused the attribution — useful for checking a deployJobPattern is matching what you expect.", + "description": "Every attributed deployment, newest first, sourced from cicd_deployment_commits joined through the core cicd_deployment_subprojects mapping table.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, @@ -297,7 +297,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n finished_date AS 'Finished',\n sub_project AS 'Sub-Project',\n job_name AS 'CI Job',\n environment AS 'Environment',\n result AS 'Result',\n cicd_deployment_id AS 'Deployment',\n commit_sha AS 'Commit'\nFROM monorepo_subproject_deployments\nWHERE ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nORDER BY finished_date DESC\nLIMIT 200", + "rawSql": "SELECT\n dc.finished_date AS 'Finished',\n COALESCE(ds.sub_project, 'All') AS 'Sub-Project',\n dc.environment AS 'Environment',\n dc.result AS 'Result',\n dc.cicd_deployment_id AS 'Deployment',\n dc.commit_sha AS 'Commit'\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.`table` = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(dc.finished_date)\nORDER BY dc.finished_date DESC\nLIMIT 200", "refId": "A" } ], diff --git a/grafana/dashboards/postgresql/monorepo-subprojects.json b/grafana/dashboards/postgresql/monorepo-subprojects.json index d9a6f7086b3..372860d58c1 100644 --- a/grafana/dashboards/postgresql/monorepo-subprojects.json +++ b/grafana/dashboards/postgresql/monorepo-subprojects.json @@ -128,7 +128,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT sub_project AS \"Sub-Project\", COUNT(DISTINCT cicd_deployment_id) AS \"Deployments\"\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nGROUP BY sub_project\nORDER BY 2 DESC", + "rawSql": "SELECT COALESCE(ds.sub_project, 'All') AS \"Sub-Project\", COUNT(DISTINCT dc.cicd_deployment_id) AS \"Deployments\"\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.\"table\" = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE dc.result = 'SUCCESS'\n AND dc.environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(dc.finished_date)\nGROUP BY 1\nORDER BY 2 DESC", "refId": "A", "sql": { "columns": [ @@ -234,7 +234,7 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date_trunc('day', finished_date) AS time,\n sub_project AS metric,\n COUNT(DISTINCT cicd_deployment_id) AS value\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nGROUP BY 1, 2\nORDER BY 1", + "rawSql": "SELECT\n date_trunc('day', dc.finished_date) AS time,\n COALESCE(ds.sub_project, 'All') AS metric,\n COUNT(DISTINCT dc.cicd_deployment_id) AS value\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.\"table\" = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE dc.result = 'SUCCESS'\n AND dc.environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(dc.finished_date)\nGROUP BY 1, 2\nORDER BY 1", "refId": "A" } ], @@ -324,7 +324,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n sub_project AS \"Sub-Project\",\n COUNT(*) AS \"PRs\",\n ROUND((AVG(coding_time) / 60)::numeric, 1) AS \"Coding (h)\",\n ROUND((AVG(pickup_time) / 60)::numeric, 1) AS \"Pickup (h)\",\n ROUND((AVG(review_time) / 60)::numeric, 1) AS \"Review (h)\",\n ROUND((AVG(deploy_time) / 60)::numeric, 1) AS \"Deploy (h)\",\n ROUND((AVG(cycle_time) / 60)::numeric, 1) AS \"Cycle Time (h)\"\nFROM monorepo_subproject_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)\nGROUP BY sub_project\nORDER BY 1", + "rawSql": "SELECT\n COALESCE(sub_project, 'All') AS \"Sub-Project\",\n COUNT(*) AS \"PRs\",\n ROUND((AVG(pr_coding_time) / 60)::numeric, 1) AS \"Coding (h)\",\n ROUND((AVG(pr_pickup_time) / 60)::numeric, 1) AS \"Pickup (h)\",\n ROUND((AVG(pr_review_time) / 60)::numeric, 1) AS \"Review (h)\",\n ROUND((AVG(pr_deploy_time) / 60)::numeric, 1) AS \"Deploy (h)\",\n ROUND((AVG(pr_cycle_time) / 60)::numeric, 1) AS \"Cycle Time (h)\"\nFROM project_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)\nGROUP BY 1\nORDER BY 1", "refId": "A" } ], @@ -394,7 +394,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS \"Unattributed merged PRs\"\nFROM pull_requests pr\nJOIN project_mapping pm\n ON pm.\"table\" = 'repos' AND pm.row_id = pr.base_repo_id\nLEFT JOIN monorepo_subproject_pr_metrics m\n ON m.pull_request_id = pr.id AND m.project_name = pm.project_name\nWHERE pr.merged_date IS NOT NULL\n AND m.pull_request_id IS NULL\n AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr.merged_date)", + "rawSql": "SELECT COUNT(*) AS \"Unattributed merged PRs\"\nFROM pull_requests pr\nJOIN project_mapping pm ON (pm.\"table\" = 'repos' AND pm.row_id = pr.base_repo_id)\nWHERE pr.merged_date IS NOT NULL\n AND pr.sub_project = 'unattributed'\n AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr.merged_date)", "refId": "A" } ], @@ -456,7 +456,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS \"Merged, not yet deployed\"\nFROM monorepo_subproject_pr_metrics\nWHERE deployment_id = ''\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)", + "rawSql": "SELECT COUNT(*) AS \"Merged, not yet deployed\"\nFROM project_pr_metrics\nWHERE (deployment_commit_id = '' OR deployment_commit_id IS NULL)\n AND sub_project IS NOT NULL\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)", "refId": "A" } ], @@ -468,7 +468,7 @@ "type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api" }, - "description": "Every attributed deployment, newest first. The job name column shows which CI job caused the attribution \u2014 useful for checking a deployJobPattern is matching what you expect.", + "description": "Every attributed deployment, newest first, sourced from cicd_deployment_commits joined through the core cicd_deployment_subprojects mapping table.", "fieldConfig": { "defaults": { "color": { @@ -568,7 +568,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n finished_date AS \"Finished\",\n sub_project AS \"Sub-Project\",\n job_name AS \"CI Job\",\n environment AS \"Environment\",\n result AS \"Result\",\n cicd_deployment_id AS \"Deployment\",\n commit_sha AS \"Commit\"\nFROM monorepo_subproject_deployments\nWHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nORDER BY finished_date DESC\nLIMIT 200", + "rawSql": "SELECT\n dc.finished_date AS \"Finished\",\n COALESCE(ds.sub_project, 'All') AS \"Sub-Project\",\n dc.environment AS \"Environment\",\n dc.result AS \"Result\",\n dc.cicd_deployment_id AS \"Deployment\",\n dc.commit_sha AS \"Commit\"\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.\"table\" = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(dc.finished_date)\nORDER BY dc.finished_date DESC\nLIMIT 200", "refId": "A" } ], From c708d1697ed9cbc0401602e1fc5eff486a72ae05 Mon Sep 17 00:00:00 2001 From: Yotam Soudry Date: Thu, 27 Aug 2026 23:40:07 +0300 Subject: [PATCH 09/14] feat(dashboards): add sub_project grouping to single-metric PR time series MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies the mechanical COALESCE(sub_project, 'All') transform (design §5.1/§5.3) to every panel across Gitlab.json, engineering-overview.json and engineering-throughput-and-cycle-time.json (MySQL + PostgreSQL) that is safely convertible: a time series with exactly one time column and one numeric value column, which Grafana's long-format convention already renders as a single line and will now render as one line per sub-project once a text grouping column is added. sub_project is inserted as the second SELECT column (matching the design doc's own example) and added to GROUP BY; original value column names/aliases are preserved. Panels deliberately left untouched, and why: - Grafana "stat" panels (single KPI tiles): their reduceOptions is configured with values:false, meaning they collapse all returned rows into one number via "last non-null". Adding a GROUP BY would silently make the tile show an arbitrary single sub-project's value instead of the whole-project total - fixing that needs a reduceOptions change too, which needs Grafana to verify renders correctly. - Wide-format time series with multiple value columns (e.g. "PRs Opened/Merged" - two metric columns in one row) and barchart/table panels already grouping by something else (e.g. "Top 20 Contributors by Merged PRs" groups by author) - adding sub_project would multiply or reshape the series in ways that need a Grafana-side call, not a mechanical SQL edit. - Every dora-details-*.json and dora-by-team.json panel, and most of engineering-throughput-and-cycle-time-team-view.json: multi-CTE queries with window functions (percent_rank, row_number, calendar-generation CTEs for medians). Retrofitting a grouping dimension into these correctly means restructuring most of the CTEs, which is real engineering work that needs a live Grafana instance to verify - not safe to do blind. Co-Authored-By: Claude Sonnet 5 --- grafana/dashboards/mysql/Gitlab.json | 2 +- grafana/dashboards/mysql/engineering-overview.json | 4 ++-- .../mysql/engineering-throughput-and-cycle-time.json | 4 ++-- grafana/dashboards/postgresql/Gitlab.json | 2 +- grafana/dashboards/postgresql/engineering-overview.json | 4 ++-- .../postgresql/engineering-throughput-and-cycle-time.json | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/grafana/dashboards/mysql/Gitlab.json b/grafana/dashboards/mysql/Gitlab.json index 3a7043b69ec..b0dec184bf6 100644 --- a/grafana/dashboards/mysql/Gitlab.json +++ b/grafana/dashboards/mysql/Gitlab.json @@ -969,7 +969,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "-- The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field `original_status`\nSELECT\n DATE_ADD(date(created_date), INTERVAL -$interval(date(created_date))+1 DAY) as time,\n count(distinct case when status = 'CLOSED' then id else null end)/count(distinct case when status in ('CLOSED', 'MERGED') then id else null end) as ratio\nFROM pull_requests\nWHERE\n $__timeFilter(created_date)\n and base_repo_id in (${repo_id})\ngroup by 1\n", + "rawSql": "-- The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field `original_status`\nSELECT\n DATE_ADD(date(created_date), INTERVAL -$interval(date(created_date))+1 DAY) as time,\n COALESCE(sub_project, 'All') as sub_project,\n count(distinct case when status = 'CLOSED' then id else null end)/count(distinct case when status in ('CLOSED', 'MERGED') then id else null end) as ratio\nFROM pull_requests\nWHERE\n $__timeFilter(created_date)\n and base_repo_id in (${repo_id})\ngroup by 1, 2", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/mysql/engineering-overview.json b/grafana/dashboards/mysql/engineering-overview.json index ad6d5dfefb6..6e4f2be7c07 100644 --- a/grafana/dashboards/mysql/engineering-overview.json +++ b/grafana/dashboards/mysql/engineering-overview.json @@ -1483,7 +1483,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select\n DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY) as time,\n 100*count(distinct case when pr.id in (select pull_request_id from pull_request_issues) then pr.id else null end)/count(distinct pr.id) as unlinked_pr_rate\nfrom pull_requests pr\njoin project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere pm.project_name in (${project})\nand $__timeFilter(created_date)\nand created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -DAY($__timeFrom())+1 DAY), INTERVAL +1 MONTH)\ngroup by time\n\n", + "rawSql": "select\n DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY) as time,\n COALESCE(pr.sub_project, 'All') as sub_project,\n 100*count(distinct case when pr.id in (select pull_request_id from pull_request_issues) then pr.id else null end)/count(distinct pr.id) as unlinked_pr_rate\nfrom pull_requests pr\njoin project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere pm.project_name in (${project})\nand $__timeFilter(created_date)\nand created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -DAY($__timeFrom())+1 DAY), INTERVAL +1 MONTH)\ngroup by time, pr.sub_project", "refId": "A", "select": [ [ @@ -1969,7 +1969,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select\n DATE_ADD(date(pr.created_date), INTERVAL -DAY(date(pr.created_date))+1 DAY) as time,\n AVG(TIMESTAMPDIFF(MINUTE, pr.created_date, pr.merged_date) / 1440) as pr_time_to_merge_in_days\nfrom\n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere\n pm.project_name in (${project}) and\n pr.merged_date is not null\n and $__timeFilter(pr.created_date)\n and pr.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -DAY($__timeFrom())+1 DAY), INTERVAL +1 MONTH)\ngroup by time\norder by time", + "rawSql": "select\n DATE_ADD(date(pr.created_date), INTERVAL -DAY(date(pr.created_date))+1 DAY) as time,\n COALESCE(pr.sub_project, 'All') as sub_project,\n AVG(TIMESTAMPDIFF(MINUTE, pr.created_date, pr.merged_date) / 1440) as pr_time_to_merge_in_days\nfrom\n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere\n pm.project_name in (${project}) and\n pr.merged_date is not null\n and $__timeFilter(pr.created_date)\n and pr.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -DAY($__timeFrom())+1 DAY), INTERVAL +1 MONTH)\ngroup by time, pr.sub_project\norder by time", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json b/grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json index fd62b4e053f..3db70532cd3 100644 --- a/grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json +++ b/grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json @@ -622,7 +622,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n DATE_ADD(date(pr.created_date), INTERVAL -$interval(date(pr.created_date))+1 DAY) as time,\n count(distinct prc.id)/count(distinct pr.id) as \"PR Review Depth\"\nFROM \n pull_requests pr\n left join pull_request_comments prc on pr.id = prc.pull_request_id\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nWHERE\n $__timeFilter(pr.created_date)\n and pm.project_name in (${project})\n and pr.merged_date is not null\ngroup by 1\n", + "rawSql": "SELECT\n DATE_ADD(date(pr.created_date), INTERVAL -$interval(date(pr.created_date))+1 DAY) as time,\n COALESCE(pr.sub_project, 'All') as sub_project,\n count(distinct prc.id)/count(distinct pr.id) as \"PR Review Depth\"\nFROM \n pull_requests pr\n left join pull_request_comments prc on pr.id = prc.pull_request_id\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nWHERE\n $__timeFilter(pr.created_date)\n and pm.project_name in (${project})\n and pr.merged_date is not null\ngroup by 1, 2", "refId": "A", "select": [ [ @@ -1004,7 +1004,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n DATE_ADD(date(pr.created_date), INTERVAL -$interval(date(pr.created_date))+1 DAY) as time,\n sum(case when pr.id not in (SELECT pull_request_id FROM pull_request_comments) then 1 else 0 end) as \"PRs Merged w/o Review\"\nFROM \n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nWHERE\n $__timeFilter(pr.created_date)\n and pm.project_name in (${project})\n and pr.merged_date is not null\nGROUP BY 1\nORDER BY 1", + "rawSql": "SELECT\n DATE_ADD(date(pr.created_date), INTERVAL -$interval(date(pr.created_date))+1 DAY) as time,\n COALESCE(pr.sub_project, 'All') as sub_project,\n sum(case when pr.id not in (SELECT pull_request_id FROM pull_request_comments) then 1 else 0 end) as \"PRs Merged w/o Review\"\nFROM \n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nWHERE\n $__timeFilter(pr.created_date)\n and pm.project_name in (${project})\n and pr.merged_date is not null\nGROUP BY 1, 2\nORDER BY 1", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/postgresql/Gitlab.json b/grafana/dashboards/postgresql/Gitlab.json index af78160fc19..d038db58a23 100644 --- a/grafana/dashboards/postgresql/Gitlab.json +++ b/grafana/dashboards/postgresql/Gitlab.json @@ -969,7 +969,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "/* The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('CLOSED', 'MERGED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1", + "rawSql": "/* The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COALESCE(sub_project, 'All') AS sub_project, CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('CLOSED', 'MERGED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1, 2", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/postgresql/engineering-overview.json b/grafana/dashboards/postgresql/engineering-overview.json index 6d22a879f9c..287973048ca 100644 --- a/grafana/dashboards/postgresql/engineering-overview.json +++ b/grafana/dashboards/postgresql/engineering-overview.json @@ -1483,7 +1483,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, CAST(100 * COUNT(DISTINCT CASE WHEN pr.id IN (SELECT pull_request_id FROM pull_request_issues) THEN pr.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS unlinked_pr_rate FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(created_date) AND created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\"", + "rawSql": "SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, COALESCE(pr.sub_project, 'All') AS sub_project, CAST(100 * COUNT(DISTINCT CASE WHEN pr.id IN (SELECT pull_request_id FROM pull_request_issues) THEN pr.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS unlinked_pr_rate FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(created_date) AND created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\", pr.sub_project", "refId": "A", "select": [ [ @@ -1969,7 +1969,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) + 1) AS time, AVG(CAST((EXTRACT(EPOCH FROM (pr.merged_date - pr.created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS pr_time_to_merge_in_days FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND $__timeFilter(pr.created_date) AND pr.created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\" ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) + 1) AS time, COALESCE(pr.sub_project, 'All') AS sub_project, AVG(CAST((EXTRACT(EPOCH FROM (pr.merged_date - pr.created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS pr_time_to_merge_in_days FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND $__timeFilter(pr.created_date) AND pr.created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\", pr.sub_project ORDER BY time NULLS FIRST", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json b/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json index 174ad11e25e..6abd47e0afd 100644 --- a/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json +++ b/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json @@ -622,7 +622,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT prc.id) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS \"PR Review Depth\" FROM pull_requests AS pr LEFT JOIN pull_request_comments AS prc ON pr.id = prc.pull_request_id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL GROUP BY 1", + "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, COALESCE(pr.sub_project, 'All') AS sub_project, CAST(COUNT(DISTINCT prc.id) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS \"PR Review Depth\" FROM pull_requests AS pr LEFT JOIN pull_request_comments AS prc ON pr.id = prc.pull_request_id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL GROUP BY 1, 2", "refId": "A", "select": [ [ @@ -1004,7 +1004,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, SUM(CASE WHEN NOT pr.id IN (SELECT pull_request_id FROM pull_request_comments) THEN 1 ELSE 0 END) AS \"PRs Merged w/o Review\" FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL GROUP BY 1 ORDER BY 1 NULLS FIRST", + "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, COALESCE(pr.sub_project, 'All') AS sub_project, SUM(CASE WHEN NOT pr.id IN (SELECT pull_request_id FROM pull_request_comments) THEN 1 ELSE 0 END) AS \"PRs Merged w/o Review\" FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL GROUP BY 1, 2 ORDER BY 1 NULLS FIRST", "refId": "A", "select": [ [ From 71cba6b77b8b9495ffa9f6f5c892c11bb24a7612 Mon Sep 17 00:00:00 2001 From: Yotam Soudry Date: Mon, 31 Aug 2026 23:13:21 +0300 Subject: [PATCH 10/14] fix(monorepo): freeze migration script models to satisfy migration linter Migration scripts must not import live model packages (core/migration/linter core/migration/linter/main.go enforces this so migrations stay immutable). 20260809_add_init_tables.go and 20260810_add_cicd_deployment_subprojects.go imported plugins/monorepo/models and core/models/domainlayer/devops respectively; replace with version-frozen local snapshot structs matching the existing convention (see 20260426_add_auth_sessions.go). Co-Authored-By: Claude Sonnet 5 --- ...0260810_add_cicd_deployment_subprojects.go | 24 ++++++--- .../20260809_add_init_tables.go | 53 +++++++++++++++++-- 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/backend/core/models/migrationscripts/20260810_add_cicd_deployment_subprojects.go b/backend/core/models/migrationscripts/20260810_add_cicd_deployment_subprojects.go index 049f4801ac1..de6bc9534e0 100644 --- a/backend/core/models/migrationscripts/20260810_add_cicd_deployment_subprojects.go +++ b/backend/core/models/migrationscripts/20260810_add_cicd_deployment_subprojects.go @@ -20,23 +20,35 @@ package migrationscripts import ( "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/migrationhelper" ) var _ plugin.MigrationScript = (*addCicdDeploymentSubprojects)(nil) +// cicdDeploymentSubproject20260810 is a version-frozen snapshot of +// devops.CicdDeploymentSubproject as it looked when this migration was written. +// Migration scripts must not import live model packages (see core/migration/linter), +// so the shape is duplicated here on purpose rather than imported from the domain layer. +type cicdDeploymentSubproject20260810 struct { + archived.NoPKModel + ProjectName string `gorm:"primaryKey;type:varchar(100)"` + CicdDeploymentId string `gorm:"primaryKey;type:varchar(255);index:idx_cds_deployment"` + SubProject string `gorm:"primaryKey;type:varchar(100)"` +} + +func (cicdDeploymentSubproject20260810) TableName() string { + return "cicd_deployment_subprojects" +} + type addCicdDeploymentSubprojects struct{} -// Up creates the new cicd_deployment_subprojects mapping table. This is a brand new -// table (not an existing one gaining a column), so it is migrated straight from the live -// domain model, matching the precedent set by the monorepo plugin's own -// 20260809_add_init_tables.go rather than a versioned snapshot struct. +// Up creates the new cicd_deployment_subprojects mapping table. func (script *addCicdDeploymentSubprojects) Up(basicRes context.BasicRes) errors.Error { return migrationhelper.AutoMigrateTables( basicRes, - &devops.CicdDeploymentSubproject{}, + &cicdDeploymentSubproject20260810{}, ) } diff --git a/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go b/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go index ce485504013..8e22690a722 100644 --- a/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go +++ b/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go @@ -18,22 +18,69 @@ limitations under the License. package migrationscripts import ( + "time" + "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/migrationhelper" - "github.com/apache/incubator-devlake/plugins/monorepo/models" ) var _ plugin.MigrationScript = (*addInitTables)(nil) +// subProjectDeployment20260809 is a version-frozen snapshot of +// models.SubProjectDeployment as it looked when this migration was written. +// Migration scripts must not import their plugin's live models package +// (see core/migration/linter), so the shape is duplicated here on purpose. +type subProjectDeployment20260809 struct { + archived.NoPKModel + ProjectName string `gorm:"primaryKey;type:varchar(100)"` + SubProject string `gorm:"primaryKey;type:varchar(100)"` + CicdDeploymentId string `gorm:"primaryKey;type:varchar(255)"` + CommitSha string `gorm:"primaryKey;type:varchar(64)"` + JobName string `gorm:"type:varchar(255)"` + Result string `gorm:"type:varchar(100)"` + Environment string `gorm:"type:varchar(255)"` + FinishedDate *time.Time +} + +func (subProjectDeployment20260809) TableName() string { + return "monorepo_subproject_deployments" +} + +// subProjectPrMetric20260809 is a version-frozen snapshot of +// models.SubProjectPrMetric as it looked when this migration was written. +type subProjectPrMetric20260809 struct { + archived.NoPKModel + ProjectName string `gorm:"primaryKey;type:varchar(100)"` + PullRequestId string `gorm:"primaryKey;type:varchar(255)"` + SubProject string `gorm:"index;type:varchar(255)"` + + CodingTime *int64 + PickupTime *int64 + ReviewTime *int64 + DeployTime *int64 + CycleTime *int64 + + DeploymentId string `gorm:"type:varchar(255)"` + + PrCreatedDate *time.Time + PrMergedDate *time.Time + DeployedDate *time.Time +} + +func (subProjectPrMetric20260809) TableName() string { + return "monorepo_subproject_pr_metrics" +} + type addInitTables struct{} func (script *addInitTables) Up(basicRes context.BasicRes) errors.Error { return migrationhelper.AutoMigrateTables( basicRes, - &models.SubProjectDeployment{}, - &models.SubProjectPrMetric{}, + &subProjectDeployment20260809{}, + &subProjectPrMetric20260809{}, ) } From 0dd4a038469e622e88fa2009f6016f26e6647168 Mon Sep 17 00:00:00 2001 From: Yotam Soudry Date: Mon, 31 Aug 2026 23:13:28 +0300 Subject: [PATCH 11/14] fix(monorepo): suppress staticcheck SA1019 for intentional deprecated-table writes golangci-lint's staticcheck flagged four internal uses of models.SubProjectDeployment/SubProjectPrMetric as deprecated. These are the plugin's own writers that must keep populating those tables during the documented one-release backward-compat window, so the warning is expected; scope a //nolint:staticcheck to just those call sites rather than touching the deprecation notice or the shared golangci-lint config. Co-Authored-By: Claude Sonnet 5 --- backend/plugins/monorepo/impl/impl.go | 7 +++++-- backend/plugins/monorepo/tasks/deployment_attributor.go | 4 +++- .../plugins/monorepo/tasks/project_pr_metrics_updater.go | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/backend/plugins/monorepo/impl/impl.go b/backend/plugins/monorepo/impl/impl.go index c753a58ca13..21da42e1129 100644 --- a/backend/plugins/monorepo/impl/impl.go +++ b/backend/plugins/monorepo/impl/impl.go @@ -75,8 +75,11 @@ func (p Monorepo) RequiredDataEntities() (data []map[string]interface{}, err err func (p Monorepo) GetTablesInfo() []dal.Tabler { return []dal.Tabler{ - &models.SubProjectDeployment{}, - &models.SubProjectPrMetric{}, + // These deprecated tables are still owned and populated by this plugin + // during the compat window; they must remain registered here so + // migrations/deletion tooling can find them. + &models.SubProjectDeployment{}, //nolint:staticcheck // SA1019 + &models.SubProjectPrMetric{}, //nolint:staticcheck // SA1019 } } diff --git a/backend/plugins/monorepo/tasks/deployment_attributor.go b/backend/plugins/monorepo/tasks/deployment_attributor.go index 1717d0c8088..dbd56df9fea 100644 --- a/backend/plugins/monorepo/tasks/deployment_attributor.go +++ b/backend/plugins/monorepo/tasks/deployment_attributor.go @@ -123,7 +123,9 @@ func AttributeDeployments(taskCtx plugin.SubTaskContext) errors.Error { CicdDeploymentId: row.CicdDeploymentId, SubProject: subProject, }) - results = append(results, &models.SubProjectDeployment{ + // SubProjectDeployment is deprecated but this plugin still owns and + // populates it during the compat window. + results = append(results, &models.SubProjectDeployment{ //nolint:staticcheck // SA1019 ProjectName: data.Options.ProjectName, SubProject: subProject, CicdDeploymentId: row.CicdDeploymentId, diff --git a/backend/plugins/monorepo/tasks/project_pr_metrics_updater.go b/backend/plugins/monorepo/tasks/project_pr_metrics_updater.go index 7494141e1e7..99e029c7361 100644 --- a/backend/plugins/monorepo/tasks/project_pr_metrics_updater.go +++ b/backend/plugins/monorepo/tasks/project_pr_metrics_updater.go @@ -196,7 +196,9 @@ func backfillSubProjectPrMetrics(taskCtx plugin.SubTaskContext, db dal.Dal, proj Input: cursor, Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { row := inputRow.(*prMetricSubProjectRow) - return []interface{}{&models.SubProjectPrMetric{ + // SubProjectPrMetric is deprecated but this plugin still owns and + // populates it during the compat window. + return []interface{}{&models.SubProjectPrMetric{ //nolint:staticcheck // SA1019 ProjectName: projectName, PullRequestId: row.PullRequestId, SubProject: row.SubProject, From 9d12cb2dfae4cf59d548cb6ab682a5ab8645cdba Mon Sep 17 00:00:00 2001 From: Yotam Soudry Date: Mon, 31 Aug 2026 23:13:35 +0300 Subject: [PATCH 12/14] test: register monorepo plugin in table-info and schema-drift test suites Test_GetPluginTablesInfo (plugins/table_info_test.go) and TestAllGoPluginsListed/TestMigrationSchemaMatchesModels (plugins/schema_e2e/migration_schema_test.go) both maintain an explicit list of every built-in Go plugin and assert it stays in sync with the plugin directories on disk. The new monorepo plugin was never added to either list, so both were failing with a directory-count/registered-count mismatch. Co-Authored-By: Claude Sonnet 5 --- backend/plugins/schema_e2e/migration_schema_test.go | 2 ++ backend/plugins/table_info_test.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/backend/plugins/schema_e2e/migration_schema_test.go b/backend/plugins/schema_e2e/migration_schema_test.go index 0fbc02893bc..3030fe1e12f 100644 --- a/backend/plugins/schema_e2e/migration_schema_test.go +++ b/backend/plugins/schema_e2e/migration_schema_test.go @@ -72,6 +72,7 @@ import ( jira "github.com/apache/incubator-devlake/plugins/jira/impl" linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" + monorepo "github.com/apache/incubator-devlake/plugins/monorepo/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" org "github.com/apache/incubator-devlake/plugins/org/impl" pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl" @@ -123,6 +124,7 @@ func allGoPlugins() []plugin.PluginMeta { jira.Jira{}, linear.Linear{}, linker.Linker{}, + monorepo.Monorepo{}, opsgenie.Opsgenie{}, org.Org{}, pagerduty.PagerDuty{}, diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index c3262153dfc..a398968926e 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -48,6 +48,7 @@ import ( jira "github.com/apache/incubator-devlake/plugins/jira/impl" linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" + monorepo "github.com/apache/incubator-devlake/plugins/monorepo/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" org "github.com/apache/incubator-devlake/plugins/org/impl" pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl" @@ -116,6 +117,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("issue_trace/models", issueTrace.IssueTrace{}.GetTablesInfo) checker.FeedIn("q_dev/models", q_dev.QDev{}.GetTablesInfo) checker.FeedIn("gh-copilot/models", copilot.GhCopilot{}.GetTablesInfo) + checker.FeedIn("monorepo/models", monorepo.Monorepo{}.GetTablesInfo) err := checker.Verify() if err != nil { t.Error(err) From 6b9fa204562638bae52a1e274b79ffc2897f4445 Mon Sep 17 00:00:00 2001 From: Yotam Soudry Date: Mon, 31 Aug 2026 23:13:43 +0300 Subject: [PATCH 13/14] fix(migrations): use CURRENT_TIMESTAMP for cicd_deployment_subprojects backfill on Postgres The backfill migration bound a Go time.Time as a bare parameter in a SELECT list (INSERT ... SELECT DISTINCT ..., ?, ? FROM ...). Postgres's extended query protocol can't infer a type for a parameter that isn't compared against a typed column, defaults it to text, and then rejects the insert into the timestamptz created_at/updated_at columns with "column is of type timestamp with time zone but expression is of type text" (SQLSTATE 42804). MySQL is lenient about the same coercion, so this only failed under e2e-postgres. Replacing the two bound parameters with the SQL-standard CURRENT_TIMESTAMP sidesteps the parameter-typing problem entirely and keeps the statement identical across both supported databases, preserving the migration's existing MySQL/Postgres-portable design. Verified locally against real postgres:18.4 and mysql:8.0 containers. Co-Authored-By: Claude Sonnet 5 --- ...20260810_backfill_sub_project_from_monorepo.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/core/models/migrationscripts/20260810_backfill_sub_project_from_monorepo.go b/backend/core/models/migrationscripts/20260810_backfill_sub_project_from_monorepo.go index 826c510af5c..eb1b33adc06 100644 --- a/backend/core/models/migrationscripts/20260810_backfill_sub_project_from_monorepo.go +++ b/backend/core/models/migrationscripts/20260810_backfill_sub_project_from_monorepo.go @@ -18,8 +18,6 @@ limitations under the License. package migrationscripts import ( - "time" - "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -119,10 +117,17 @@ func backfillPrSubProjects(db dal.Dal) errors.Error { } func backfillDeploymentSubProjects(db dal.Dal) errors.Error { - now := time.Now() + // created_at/updated_at are set via CURRENT_TIMESTAMP rather than a bound + // time.Time parameter: when a parameter appears only in a bare SELECT list + // (not compared against a typed column), Postgres's extended query protocol + // can't infer its type and defaults it to text, which then fails to insert + // into the timestamptz column with "column is of type timestamp with time + // zone but expression is of type text". CURRENT_TIMESTAMP is standard SQL + // supported identically by MySQL and Postgres, so it sidesteps the + // parameter-typing issue entirely without needing a dialect branch. if err := db.Exec(` INSERT INTO cicd_deployment_subprojects (project_name, cicd_deployment_id, sub_project, created_at, updated_at) - SELECT DISTINCT d.project_name, d.cicd_deployment_id, d.sub_project, ?, ? + SELECT DISTINCT d.project_name, d.cicd_deployment_id, d.sub_project, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP FROM monorepo_subproject_deployments d WHERE NOT EXISTS ( SELECT 1 FROM cicd_deployment_subprojects x @@ -130,7 +135,7 @@ func backfillDeploymentSubProjects(db dal.Dal) errors.Error { AND x.cicd_deployment_id = d.cicd_deployment_id AND x.sub_project = d.sub_project ) - `, now, now); err != nil { + `); err != nil { return errors.Default.Wrap(err, "error backfilling cicd_deployment_subprojects") } return nil From 61e96c549e84d82207b14fcb78e1e666683045be Mon Sep 17 00:00:00 2001 From: Yotam Soudry Date: Mon, 31 Aug 2026 23:33:11 +0300 Subject: [PATCH 14/14] fix(dashboards): join pull_request_commits on full PK in PR Details panel The "2. PR Details" panel on the DORA Details - Lead Time for Changes dashboard joined pull_request_commits on commit_sha alone. Since pull_request_commits' primary key is (commit_sha, pull_request_id), a commit shared across multiple PRs (e.g. cherry-picks) causes this join to fan out to multiple rows, forcing a DISTINCT over a larger result set and contributing to the slow/504 query reported for this panel. Add the pull_request_id = pr.id predicate so the join goes through the full primary key instead of just its leading column, restoring a one-row-per-PR lookup for both the MySQL and PostgreSQL dashboards. Co-Authored-By: Claude Sonnet 5 --- grafana/dashboards/mysql/dora-details-lead-timefor-changes.json | 2 +- .../postgresql/dora-details-lead-timefor-changes.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/grafana/dashboards/mysql/dora-details-lead-timefor-changes.json b/grafana/dashboards/mysql/dora-details-lead-timefor-changes.json index 345eb16e2af..ec7274e7c44 100644 --- a/grafana/dashboards/mysql/dora-details-lead-timefor-changes.json +++ b/grafana/dashboards/mysql/dora-details-lead-timefor-changes.json @@ -833,7 +833,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "with _pr_stats as (\n-- get the cycle time of PRs deployed by the deployments finished each month\n\tSELECT\n\t\tdistinct\n\t\tpr.id,\n\t\tpr.title,\n\t\tpr.url,\n\t\tpr.created_date,\n\t\tppm.pr_coding_time,\n\t\tppm.pr_pickup_time,\n\t\tppm.pr_review_time,\n\t\tppm.pr_deploy_time,\n\t\tppm.first_commit_sha,\n\t\tprc.commit_authored_date,\n\t\tcdc.cicd_deployment_id,\n\t\tcdc.name, \n\t\tcdc.finished_date,\n\t\tppm.pr_cycle_time\n\tFROM\n\t\tpull_requests pr\n\t\tjoin project_pr_metrics ppm on ppm.id = pr.id\n\t\tjoin project_mapping pm on pr.base_repo_id = pm.row_id and pm.`table` = 'repos'\n\t\tjoin cicd_deployment_commits cdc on ppm.deployment_commit_id = cdc.id\n\t\tjoin pull_request_commits prc on prc.commit_sha = ppm.first_commit_sha\n\tWHERE\n\t\tpm.project_name in ($project) \n\t\tand pr.merged_date is not null\n\t\tand ppm.pr_cycle_time is not null\n\t\tand $__timeFilter(cdc.finished_date)\n)\n\nSELECT \n -- id as \"PR id\",\n\ttitle as \"PR title\",\n\turl as \"PR url\",\n\turl as metric_hidden,\n -- created_date as \"PR created_date\",\n\tfirst_commit_sha as \"First commit sha\",\n\tcommit_authored_date as \"First commit authored date\",\n\tcicd_deployment_id as \"Deployment id\",\n\t-- name as \"Deployment name\", \n\tfinished_date as \"Deployment finished_date\", \n\tpr_coding_time/60 as \"pr_coding_time\",\n\tpr_pickup_time/60 as \"pr_pickup_time\",\n\tpr_review_time/60 as \"pr_review_time\",\n\tpr_deploy_time/60 as \"pr_deploy_time\",\n pr_cycle_time/60 as change_lead_time\nFROM _pr_stats\n", + "rawSql": "with _pr_stats as (\n-- get the cycle time of PRs deployed by the deployments finished each month\n\tSELECT\n\t\tdistinct\n\t\tpr.id,\n\t\tpr.title,\n\t\tpr.url,\n\t\tpr.created_date,\n\t\tppm.pr_coding_time,\n\t\tppm.pr_pickup_time,\n\t\tppm.pr_review_time,\n\t\tppm.pr_deploy_time,\n\t\tppm.first_commit_sha,\n\t\tprc.commit_authored_date,\n\t\tcdc.cicd_deployment_id,\n\t\tcdc.name, \n\t\tcdc.finished_date,\n\t\tppm.pr_cycle_time\n\tFROM\n\t\tpull_requests pr\n\t\tjoin project_pr_metrics ppm on ppm.id = pr.id\n\t\tjoin project_mapping pm on pr.base_repo_id = pm.row_id and pm.`table` = 'repos'\n\t\tjoin cicd_deployment_commits cdc on ppm.deployment_commit_id = cdc.id\n\t\tjoin pull_request_commits prc on prc.commit_sha = ppm.first_commit_sha and prc.pull_request_id = pr.id\n\tWHERE\n\t\tpm.project_name in ($project) \n\t\tand pr.merged_date is not null\n\t\tand ppm.pr_cycle_time is not null\n\t\tand $__timeFilter(cdc.finished_date)\n)\n\nSELECT \n -- id as \"PR id\",\n\ttitle as \"PR title\",\n\turl as \"PR url\",\n\turl as metric_hidden,\n -- created_date as \"PR created_date\",\n\tfirst_commit_sha as \"First commit sha\",\n\tcommit_authored_date as \"First commit authored date\",\n\tcicd_deployment_id as \"Deployment id\",\n\t-- name as \"Deployment name\", \n\tfinished_date as \"Deployment finished_date\", \n\tpr_coding_time/60 as \"pr_coding_time\",\n\tpr_pickup_time/60 as \"pr_pickup_time\",\n\tpr_review_time/60 as \"pr_review_time\",\n\tpr_deploy_time/60 as \"pr_deploy_time\",\n pr_cycle_time/60 as change_lead_time\nFROM _pr_stats\n", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/postgresql/dora-details-lead-timefor-changes.json b/grafana/dashboards/postgresql/dora-details-lead-timefor-changes.json index 6269f469d4e..071d569115f 100644 --- a/grafana/dashboards/postgresql/dora-details-lead-timefor-changes.json +++ b/grafana/dashboards/postgresql/dora-details-lead-timefor-changes.json @@ -833,7 +833,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "WITH _pr_stats AS (/* get the cycle time of PRs deployed by the deployments finished each month */ SELECT DISTINCT pr.id, pr.title, pr.url, pr.created_date, ppm.pr_coding_time, ppm.pr_pickup_time, ppm.pr_review_time, ppm.pr_deploy_time, ppm.first_commit_sha, prc.commit_authored_date, cdc.cicd_deployment_id, cdc.name, cdc.finished_date, ppm.pr_cycle_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id JOIN pull_request_commits AS prc ON prc.commit_sha = ppm.first_commit_sha WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date)) SELECT title AS \"PR title\" /* id as \"PR id\", */, url AS \"PR url\", url AS metric_hidden, first_commit_sha AS \"First commit sha\" /* created_date as \"PR created_date\", */, commit_authored_date AS \"First commit authored date\", cicd_deployment_id AS \"Deployment id\", finished_date AS \"Deployment finished_date\" /* name as \"Deployment name\", */, CAST(pr_coding_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_coding_time\", CAST(pr_pickup_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_pickup_time\", CAST(pr_review_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_review_time\", CAST(pr_deploy_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_deploy_time\", CAST(pr_cycle_time AS NUMERIC) / NULLIF(60, 0) AS change_lead_time FROM _pr_stats", + "rawSql": "WITH _pr_stats AS (/* get the cycle time of PRs deployed by the deployments finished each month */ SELECT DISTINCT pr.id, pr.title, pr.url, pr.created_date, ppm.pr_coding_time, ppm.pr_pickup_time, ppm.pr_review_time, ppm.pr_deploy_time, ppm.first_commit_sha, prc.commit_authored_date, cdc.cicd_deployment_id, cdc.name, cdc.finished_date, ppm.pr_cycle_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id JOIN pull_request_commits AS prc ON prc.commit_sha = ppm.first_commit_sha AND prc.pull_request_id = pr.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date)) SELECT title AS \"PR title\" /* id as \"PR id\", */, url AS \"PR url\", url AS metric_hidden, first_commit_sha AS \"First commit sha\" /* created_date as \"PR created_date\", */, commit_authored_date AS \"First commit authored date\", cicd_deployment_id AS \"Deployment id\", finished_date AS \"Deployment finished_date\" /* name as \"Deployment name\", */, CAST(pr_coding_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_coding_time\", CAST(pr_pickup_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_pickup_time\", CAST(pr_review_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_review_time\", CAST(pr_deploy_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_deploy_time\", CAST(pr_cycle_time AS NUMERIC) / NULLIF(60, 0) AS change_lead_time FROM _pr_stats", "refId": "A", "select": [ [