From 05f441190edfa4537dfa21e50c2127b2e57ad5e8 Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Thu, 9 Jul 2026 13:05:31 -0600 Subject: [PATCH 01/13] feat(cursor): add plugin for team usage and cost collection Signed-off-by: Joshua Smith --- backend/plugins/cursor/api/blueprint_v200.go | 81 +++++++ backend/plugins/cursor/api/connection.go | 105 ++++++++++ backend/plugins/cursor/api/init.go | 58 +++++ backend/plugins/cursor/api/remote_api.go | 95 +++++++++ backend/plugins/cursor/api/scope.go | 53 +++++ backend/plugins/cursor/api/scope_config.go | 53 +++++ backend/plugins/cursor/api/test_connection.go | 70 +++++++ backend/plugins/cursor/cursor.go | 44 ++++ .../plugins/cursor/impl/connection_helper.go | 25 +++ backend/plugins/cursor/impl/impl.go | 168 +++++++++++++++ backend/plugins/cursor/models/connection.go | 105 ++++++++++ backend/plugins/cursor/models/member.go | 40 ++++ .../migrationscripts/20260709_initialize.go | 174 +++++++++++++++ .../models/migrationscripts/register.go | 27 +++ backend/plugins/cursor/models/models.go | 32 +++ backend/plugins/cursor/models/scope.go | 100 +++++++++ backend/plugins/cursor/models/scope_config.go | 38 ++++ backend/plugins/cursor/models/usage_event.go | 57 +++++ backend/plugins/cursor/models/user_spend.go | 48 +++++ .../cursor/service/connection_test_helper.go | 130 ++++++++++++ backend/plugins/cursor/tasks/api_client.go | 53 +++++ .../plugins/cursor/tasks/collector_utils.go | 198 ++++++++++++++++++ .../plugins/cursor/tasks/members_collector.go | 62 ++++++ .../plugins/cursor/tasks/members_extractor.go | 84 ++++++++ backend/plugins/cursor/tasks/options.go | 24 +++ backend/plugins/cursor/tasks/register.go | 32 +++ backend/plugins/cursor/tasks/retry_after.go | 79 +++++++ backend/plugins/cursor/tasks/subtasks.go | 71 +++++++ backend/plugins/cursor/tasks/task_data.go | 26 +++ .../cursor/tasks/usage_events_collector.go | 113 ++++++++++ .../cursor/tasks/usage_events_extractor.go | 121 +++++++++++ .../cursor/tasks/user_spend_collector.go | 134 ++++++++++++ .../cursor/tasks/user_spend_extractor.go | 99 +++++++++ backend/plugins/table_info_test.go | 2 + .../plugins/register/cursor/assets/icon.svg | 22 ++ .../src/plugins/register/cursor/config.tsx | 59 ++++++ .../cursor/connection-fields/index.ts | 19 ++ .../cursor/connection-fields/styled.ts | 25 +++ .../cursor/connection-fields/token.tsx | 63 ++++++ .../src/plugins/register/cursor/index.ts | 19 ++ config-ui/src/plugins/register/index.ts | 2 + 41 files changed, 2810 insertions(+) create mode 100644 backend/plugins/cursor/api/blueprint_v200.go create mode 100644 backend/plugins/cursor/api/connection.go create mode 100644 backend/plugins/cursor/api/init.go create mode 100644 backend/plugins/cursor/api/remote_api.go create mode 100644 backend/plugins/cursor/api/scope.go create mode 100644 backend/plugins/cursor/api/scope_config.go create mode 100644 backend/plugins/cursor/api/test_connection.go create mode 100644 backend/plugins/cursor/cursor.go create mode 100644 backend/plugins/cursor/impl/connection_helper.go create mode 100644 backend/plugins/cursor/impl/impl.go create mode 100644 backend/plugins/cursor/models/connection.go create mode 100644 backend/plugins/cursor/models/member.go create mode 100644 backend/plugins/cursor/models/migrationscripts/20260709_initialize.go create mode 100644 backend/plugins/cursor/models/migrationscripts/register.go create mode 100644 backend/plugins/cursor/models/models.go create mode 100644 backend/plugins/cursor/models/scope.go create mode 100644 backend/plugins/cursor/models/scope_config.go create mode 100644 backend/plugins/cursor/models/usage_event.go create mode 100644 backend/plugins/cursor/models/user_spend.go create mode 100644 backend/plugins/cursor/service/connection_test_helper.go create mode 100644 backend/plugins/cursor/tasks/api_client.go create mode 100644 backend/plugins/cursor/tasks/collector_utils.go create mode 100644 backend/plugins/cursor/tasks/members_collector.go create mode 100644 backend/plugins/cursor/tasks/members_extractor.go create mode 100644 backend/plugins/cursor/tasks/options.go create mode 100644 backend/plugins/cursor/tasks/register.go create mode 100644 backend/plugins/cursor/tasks/retry_after.go create mode 100644 backend/plugins/cursor/tasks/subtasks.go create mode 100644 backend/plugins/cursor/tasks/task_data.go create mode 100644 backend/plugins/cursor/tasks/usage_events_collector.go create mode 100644 backend/plugins/cursor/tasks/usage_events_extractor.go create mode 100644 backend/plugins/cursor/tasks/user_spend_collector.go create mode 100644 backend/plugins/cursor/tasks/user_spend_extractor.go create mode 100644 config-ui/src/plugins/register/cursor/assets/icon.svg create mode 100644 config-ui/src/plugins/register/cursor/config.tsx create mode 100644 config-ui/src/plugins/register/cursor/connection-fields/index.ts create mode 100644 config-ui/src/plugins/register/cursor/connection-fields/styled.ts create mode 100644 config-ui/src/plugins/register/cursor/connection-fields/token.tsx create mode 100644 config-ui/src/plugins/register/cursor/index.ts diff --git a/backend/plugins/cursor/api/blueprint_v200.go b/backend/plugins/cursor/api/blueprint_v200.go new file mode 100644 index 00000000000..b1431c9bdb0 --- /dev/null +++ b/backend/plugins/cursor/api/blueprint_v200.go @@ -0,0 +1,81 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/cursor/models" + "github.com/apache/incubator-devlake/plugins/cursor/tasks" +) + +// MakeDataSourcePipelinePlanV200 generates the pipeline plan for blueprint v2.0.0. +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + _, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + + plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails) + if err != nil { + return nil, nil, err + } + + return plan, nil, nil +} + +func makeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.CursorScope, models.CursorScopeConfig], +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + + scope := scopeDetail.Scope + task, err := helper.MakePipelinePlanTask( + "cursor", + subtaskMetas, + nil, + tasks.CursorOptions{ + ConnectionId: scope.ConnectionId, + ScopeId: scope.Id, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} diff --git a/backend/plugins/cursor/api/connection.go b/backend/plugins/cursor/api/connection.go new file mode 100644 index 00000000000..594e201f495 --- /dev/null +++ b/backend/plugins/cursor/api/connection.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" +) + +// PostConnections creates a new Cursor connection. +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.CursorConnection{} + if err := helper.Decode(input.Body, connection, vld); err != nil { + return nil, err + } + + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, err + } + + if err := connectionHelper.Create(connection, input); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.CursorConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + if err := (&models.CursorConnection{}).MergeFromRequest(connection, input.Body); err != nil { + return nil, errors.Convert(err) + } + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, err + } + if err := connectionHelper.SaveWithCreateOrUpdate(connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + conn := &models.CursorConnection{} + output, err := connectionHelper.Delete(conn, input) + if err != nil { + return output, err + } + output.Body = conn.Sanitize() + return output, nil +} + +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connections []models.CursorConnection + if err := connectionHelper.List(&connections); err != nil { + return nil, err + } + for i := range connections { + connections[i] = connections[i].Sanitize() + } + return &plugin.ApiResourceOutput{Body: connections}, nil +} + +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.CursorConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +func validateConnection(connection *models.CursorConnection) errors.Error { + if connection == nil { + return errors.BadInput.New("connection is required") + } + if strings.TrimSpace(connection.Token) == "" { + return errors.BadInput.New("token is required") + } + if connection.RateLimitPerHour < 0 { + return errors.BadInput.New("rateLimitPerHour must be non-negative") + } + return nil +} diff --git a/backend/plugins/cursor/api/init.go b/backend/plugins/cursor/api/init.go new file mode 100644 index 00000000000..74d219690be --- /dev/null +++ b/backend/plugins/cursor/api/init.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/go-playground/validator/v10" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" +) + +var ( + basicRes context.BasicRes + vld *validator.Validate + connectionHelper *helper.ConnectionApiHelper + dsHelper *helper.DsHelper[models.CursorConnection, models.CursorScope, models.CursorScopeConfig] + raProxy *helper.DsRemoteApiProxyHelper[models.CursorConnection] + raScopeList *helper.DsRemoteApiScopeListHelper[models.CursorConnection, models.CursorScope, CursorRemotePagination] +) + +// Init stores basic resources and configures shared helpers for API handlers. +func Init(br context.BasicRes, meta plugin.PluginMeta) { + basicRes = br + vld = validator.New() + connectionHelper = helper.NewConnectionHelper(basicRes, vld, meta.Name()) + dsHelper = helper.NewDataSourceHelper[ + models.CursorConnection, models.CursorScope, models.CursorScopeConfig, + ]( + basicRes, + meta.Name(), + []string{"id", "teamId"}, + func(c models.CursorConnection) models.CursorConnection { + c.Normalize() + return c.Sanitize() + }, + func(s models.CursorScope) models.CursorScope { return s }, + nil, + ) + raProxy = helper.NewDsRemoteApiProxyHelper[models.CursorConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = helper.NewDsRemoteApiScopeListHelper[models.CursorConnection, models.CursorScope, CursorRemotePagination](raProxy, listCursorRemoteScopes) +} diff --git a/backend/plugins/cursor/api/remote_api.go b/backend/plugins/cursor/api/remote_api.go new file mode 100644 index 00000000000..fdc63a93757 --- /dev/null +++ b/backend/plugins/cursor/api/remote_api.go @@ -0,0 +1,95 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helperapi "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/helpers/utils" + "github.com/apache/incubator-devlake/plugins/cursor/models" +) + +// CursorRemotePagination is a placeholder for remote scope pagination. +type CursorRemotePagination struct { + Page int `json:"page"` +} + +func listCursorRemoteScopes( + _ *models.CursorConnection, + _ plugin.ApiClient, + _ string, + _ CursorRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.CursorScope], + nextPage *CursorRemotePagination, + err errors.Error, +) { + children = append(children, makeCursorRemoteScopeEntry()) + return children, nil, nil +} + +func makeCursorRemoteScopeEntry() dsmodels.DsRemoteApiScopeListEntry[models.CursorScope] { + return dsmodels.DsRemoteApiScopeListEntry[models.CursorScope]{ + Type: helperapi.RAS_ENTRY_TYPE_SCOPE, + Id: models.DefaultScopeID, + Name: "Team", + FullName: "Cursor Team", + Data: &models.CursorScope{ + Id: models.DefaultScopeID, + Name: "Team", + FullName: "Cursor Team", + }, + } +} + +// RemoteScopes lists all available scopes for this connection. +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// SearchRemoteScopes searches scopes for this connection. +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + params := &dsmodels.DsRemoteApiScopeSearchParams{ + Page: 1, + PageSize: 50, + } + if err := utils.DecodeMapStruct(input.Query, params, true); err != nil { + return nil, err + } + if err := errors.Convert(vld.Struct(params)); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid params") + } + + children := []dsmodels.DsRemoteApiScopeListEntry[models.CursorScope]{} + searchLower := strings.ToLower(strings.TrimSpace(params.Search)) + if searchLower == "" || searchLower == "team" || searchLower == "cursor" { + children = append(children, makeCursorRemoteScopeEntry()) + } + + return &plugin.ApiResourceOutput{ + Body: map[string]interface{}{ + "children": children, + "page": params.Page, + "pageSize": params.PageSize, + }, + }, nil +} diff --git a/backend/plugins/cursor/api/scope.go b/backend/plugins/cursor/api/scope.go new file mode 100644 index 00000000000..ce8dea71797 --- /dev/null +++ b/backend/plugins/cursor/api/scope.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// GetScopeList returns all scopes for a connection. +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// PutScopes creates or updates scopes for a connection. +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// GetScope returns a single scope by id. +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// PatchScope partially updates a scope. +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// DeleteScope removes a scope. +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} + +// GetScopeLatestSyncState returns the latest sync state for a scope. +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/cursor/api/scope_config.go b/backend/plugins/cursor/api/scope_config.go new file mode 100644 index 00000000000..571e49e85ca --- /dev/null +++ b/backend/plugins/cursor/api/scope_config.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// PostScopeConfig creates a new scope config. +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// GetScopeConfigList returns all scope configs for a connection. +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// GetScopeConfig returns a single scope config by id. +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// PatchScopeConfig partially updates a scope config. +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// DeleteScopeConfig removes a scope config. +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} + +// GetProjectsByScopeConfig returns projects associated with a scope config. +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} diff --git a/backend/plugins/cursor/api/test_connection.go b/backend/plugins/cursor/api/test_connection.go new file mode 100644 index 00000000000..c4220d4bca9 --- /dev/null +++ b/backend/plugins/cursor/api/test_connection.go @@ -0,0 +1,70 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + gocontext "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" + "github.com/apache/incubator-devlake/plugins/cursor/service" +) + +// TestConnection validates a Cursor connection before saving it. +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.CursorConnection{} + if err := helper.Decode(input.Body, connection, vld); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + + result, err := service.TestConnection(gocontext.Background(), basicRes, connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection validates a stored Cursor connection with optional overrides. +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.CursorConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, errors.BadInput.Wrap(err, "find connection from db")) + } + if err := (&models.CursorConnection{}).MergeFromRequest(connection, input.Body); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, errors.Convert(err)) + } + + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + + result, err := service.TestConnection(gocontext.Background(), basicRes, connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} diff --git a/backend/plugins/cursor/cursor.go b/backend/plugins/cursor/cursor.go new file mode 100644 index 00000000000..cd6ec904c7a --- /dev/null +++ b/backend/plugins/cursor/cursor.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/cursor/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Cursor + +func main() { + cmd := &cobra.Command{Use: "cursor"} + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "cursor connection id") + scopeId := cmd.Flags().StringP("scopeId", "s", "", "cursor scope id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data created after the specified time") + + _ = cmd.MarkFlagRequired("connectionId") + _ = cmd.MarkFlagRequired("scopeId") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "scopeId": *scopeId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/cursor/impl/connection_helper.go b/backend/plugins/cursor/impl/connection_helper.go new file mode 100644 index 00000000000..43d491afc3d --- /dev/null +++ b/backend/plugins/cursor/impl/connection_helper.go @@ -0,0 +1,25 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import "github.com/apache/incubator-devlake/plugins/cursor/models" + +// NormalizeConnection ensures required defaults are set before use. +func NormalizeConnection(connection *models.CursorConnection) { + connection.Normalize() +} diff --git a/backend/plugins/cursor/impl/impl.go b/backend/plugins/cursor/impl/impl.go new file mode 100644 index 00000000000..0d3b7193071 --- /dev/null +++ b/backend/plugins/cursor/impl/impl.go @@ -0,0 +1,168 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" + "github.com/apache/incubator-devlake/plugins/cursor/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/cursor/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.DataSourcePluginBlueprintV200 + plugin.PluginMigration + plugin.CloseablePluginTask +} = (*Cursor)(nil) + +// Cursor is the plugin entrypoint implementing DevLake interfaces. +type Cursor struct{} + +func (p Cursor) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Cursor) Description() string { + return "Collect Cursor team usage events and billing metrics from the Admin API" +} + +func (p Cursor) Name() string { + return "cursor" +} + +func (p Cursor) Connection() dal.Tabler { + return &models.CursorConnection{} +} + +func (p Cursor) Scope() plugin.ToolLayerScope { + return &models.CursorScope{} +} + +func (p Cursor) ScopeConfig() dal.Tabler { + return &models.CursorScopeConfig{} +} + +func (p Cursor) GetTablesInfo() []dal.Tabler { + return models.GetTablesInfo() +} + +func (p Cursor) SubTaskMetas() []plugin.SubTaskMeta { + return tasks.GetSubTaskMetas() +} + +func (p Cursor) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.CursorOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, err + } + + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + connection := &models.CursorConnection{} + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, err + } + + NormalizeConnection(connection) + + return &tasks.CursorTaskData{ + Options: &op, + Connection: connection, + }, nil +} + +func (p Cursor) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes/:scopeId/latest-sync-state": { + "GET": api.GetScopeLatestSyncState, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "GET": api.GetScopeConfig, + "PATCH": api.PatchScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p Cursor) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Cursor) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/cursor" +} + +func (p Cursor) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Cursor) Close(taskCtx plugin.TaskContext) errors.Error { + return nil +} diff --git a/backend/plugins/cursor/models/connection.go b/backend/plugins/cursor/models/connection.go new file mode 100644 index 00000000000..8bb30c53082 --- /dev/null +++ b/backend/plugins/cursor/models/connection.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const ( + // DefaultEndpoint is the Cursor Admin API base URL. + DefaultEndpoint = "https://api.cursor.com" + // DefaultRateLimitPerHour matches the documented 20 requests/minute Admin API limit. + DefaultRateLimitPerHour = 1200 +) + +// CursorConn stores Cursor Team Admin API connection settings. +type CursorConn struct { + helper.RestConnection `mapstructure:",squash"` + + Token string `mapstructure:"token" json:"token"` +} + +// SetupAuthentication uses HTTP Basic auth with the API key as username and an empty password. +func (conn *CursorConn) SetupAuthentication(request *http.Request) errors.Error { + if conn == nil { + return errors.BadInput.New("connection is required") + } + token := strings.TrimSpace(conn.Token) + if token == "" { + return errors.BadInput.New("token is required") + } + request.SetBasicAuth(token, "") + return nil +} + +func (conn *CursorConn) Sanitize() CursorConn { + if conn == nil { + return CursorConn{} + } + clone := *conn + clone.Token = utils.SanitizeString(clone.Token) + return clone +} + +// CursorConnection persists connection details with metadata required by DevLake. +type CursorConnection struct { + helper.BaseConnection `mapstructure:",squash"` + CursorConn `mapstructure:",squash"` +} + +func (CursorConnection) TableName() string { + return "_tool_cursor_connections" +} + +func (connection CursorConnection) Sanitize() CursorConnection { + connection.CursorConn = connection.CursorConn.Sanitize() + return connection +} + +func (connection *CursorConnection) Normalize() { + if connection == nil { + return + } + if connection.Endpoint == "" { + connection.Endpoint = DefaultEndpoint + } + if connection.RateLimitPerHour <= 0 { + connection.RateLimitPerHour = DefaultRateLimitPerHour + } +} + +func (connection *CursorConnection) MergeFromRequest(target *CursorConnection, body map[string]interface{}) error { + if target == nil { + return nil + } + originalToken := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + sanitizedOriginal := utils.SanitizeString(originalToken) + if target.Token == "" || target.Token == sanitizedOriginal { + target.Token = originalToken + } + return nil +} diff --git a/backend/plugins/cursor/models/member.go b/backend/plugins/cursor/models/member.go new file mode 100644 index 00000000000..63a1f24e6c8 --- /dev/null +++ b/backend/plugins/cursor/models/member.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// CursorMember stores team member roster from /teams/members. +type CursorMember struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` + + Email string `json:"email" gorm:"type:varchar(255);index"` + Name string `json:"name" gorm:"type:varchar(255)"` + Role string `json:"role" gorm:"type:varchar(50)"` + IsRemoved bool `json:"isRemoved"` + + common.NoPKModel +} + +func (CursorMember) TableName() string { + return "_tool_cursor_members" +} diff --git a/backend/plugins/cursor/models/migrationscripts/20260709_initialize.go b/backend/plugins/cursor/models/migrationscripts/20260709_initialize.go new file mode 100644 index 00000000000..0bd85596326 --- /dev/null +++ b/backend/plugins/cursor/models/migrationscripts/20260709_initialize.go @@ -0,0 +1,174 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addCursorInitialTables struct{} + +func (script *addCursorInitialTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &cursorConnection20260709{}, + &cursorScope20260709{}, + &cursorScopeConfig20260709{}, + &cursorUsageEvent20260709{}, + &cursorUserSpend20260709{}, + &cursorMember20260709{}, + &cursorRawUsageEvents20260709{}, + &cursorRawUserSpend20260709{}, + &cursorRawMembers20260709{}, + ) +} + +type cursorConnection20260709 struct { + archived.Model + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + Endpoint string `gorm:"type:varchar(255)" json:"endpoint"` + Proxy string `gorm:"type:varchar(255)" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `json:"token"` +} + +func (cursorConnection20260709) TableName() string { return "_tool_cursor_connections" } + +type cursorScope20260709 struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + Id string `json:"id" gorm:"primaryKey;type:varchar(255)"` + TeamId string `json:"teamId" gorm:"type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + FullName string `json:"fullName" gorm:"type:varchar(255)"` +} + +func (cursorScope20260709) TableName() string { return "_tool_cursor_scopes" } + +type cursorScopeConfig20260709 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities" mapstructure:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index" validate:"required" mapstructure:"connectionId,omitempty"` + Name string `mapstructure:"name" json:"name" gorm:"type:varchar(255);uniqueIndex" validate:"required"` +} + +func (cursorScopeConfig20260709) TableName() string { return "_tool_cursor_scope_configs" } + +type cursorUsageEvent20260709 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + EventId string `gorm:"primaryKey;type:varchar(64)"` + EventTime time.Time `gorm:"index"` + UserEmail string `gorm:"type:varchar(255);index"` + Model string `gorm:"type:varchar(255);index"` + Kind string `gorm:"type:varchar(100)"` + ConversationId string `gorm:"type:varchar(64);index"` + ChargedCents float64 + RequestsCosts int + IsTokenBasedCall bool + IsChargeable bool + MaxMode bool + IsHeadless bool + InputTokens int + OutputTokens int + CacheReadTokens int + CacheWriteTokens int + TotalCents float64 + CursorTokenFee float64 + HostingType string `gorm:"type:varchar(50)"` + ServiceAccountId string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (cursorUsageEvent20260709) TableName() string { return "_tool_cursor_usage_events" } + +type cursorUserSpend20260709 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + UserId string `gorm:"primaryKey;type:varchar(255)"` + BillingCycleStart time.Time `gorm:"primaryKey"` + CollectedAt time.Time `gorm:"index"` + Email string `gorm:"type:varchar(255);index"` + Name string `gorm:"type:varchar(255)"` + Role string `gorm:"type:varchar(50)"` + SpendCents float64 + IncludedSpendCents float64 + FastPremiumRequests int + MonthlyLimitDollars float64 + HardLimitOverrideDollars float64 + archived.NoPKModel +} + +func (cursorUserSpend20260709) TableName() string { return "_tool_cursor_user_spend" } + +type cursorMember20260709 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + UserId string `gorm:"primaryKey;type:varchar(255)"` + Email string `gorm:"type:varchar(255);index"` + Name string `gorm:"type:varchar(255)"` + Role string `gorm:"type:varchar(50)"` + IsRemoved bool + archived.NoPKModel +} + +func (cursorMember20260709) TableName() string { return "_tool_cursor_members" } + +type cursorRawUsageEvents20260709 struct { + ID uint64 `gorm:"primaryKey"` + Params string `gorm:"type:varchar(255);index"` + Data []byte + Url string + Input json.RawMessage `gorm:"type:json"` + CreatedAt time.Time `gorm:"index"` +} + +func (cursorRawUsageEvents20260709) TableName() string { return "_raw_cursor_usage_events" } + +type cursorRawUserSpend20260709 struct { + ID uint64 `gorm:"primaryKey"` + Params string `gorm:"type:varchar(255);index"` + Data []byte + Url string + Input json.RawMessage `gorm:"type:json"` + CreatedAt time.Time `gorm:"index"` +} + +func (cursorRawUserSpend20260709) TableName() string { return "_raw_cursor_user_spend" } + +type cursorRawMembers20260709 struct { + ID uint64 `gorm:"primaryKey"` + Params string `gorm:"type:varchar(255);index"` + Data []byte + Url string + Input json.RawMessage `gorm:"type:json"` + CreatedAt time.Time `gorm:"index"` +} + +func (cursorRawMembers20260709) TableName() string { return "_raw_cursor_members" } + +func (*addCursorInitialTables) Version() uint64 { return 20260709000000 } + +func (*addCursorInitialTables) Name() string { return "cursor init tables" } diff --git a/backend/plugins/cursor/models/migrationscripts/register.go b/backend/plugins/cursor/models/migrationscripts/register.go new file mode 100644 index 00000000000..527301e3394 --- /dev/null +++ b/backend/plugins/cursor/models/migrationscripts/register.go @@ -0,0 +1,27 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import "github.com/apache/incubator-devlake/core/plugin" + +// All returns the ordered list of migration scripts for the Cursor plugin. +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addCursorInitialTables), + } +} diff --git a/backend/plugins/cursor/models/models.go b/backend/plugins/cursor/models/models.go new file mode 100644 index 00000000000..5c37c05b047 --- /dev/null +++ b/backend/plugins/cursor/models/models.go @@ -0,0 +1,32 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "github.com/apache/incubator-devlake/core/dal" + +// GetTablesInfo returns the list of tool-layer tables managed by the Cursor plugin. +func GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &CursorConnection{}, + &CursorScope{}, + &CursorScopeConfig{}, + &CursorUsageEvent{}, + &CursorUserSpend{}, + &CursorMember{}, + } +} diff --git a/backend/plugins/cursor/models/scope.go b/backend/plugins/cursor/models/scope.go new file mode 100644 index 00000000000..00cd8e745b5 --- /dev/null +++ b/backend/plugins/cursor/models/scope.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "strings" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + "gorm.io/gorm" +) + +const ( + // DefaultScopeID is the single team-level scope for Cursor Admin API data. + DefaultScopeID = "team" +) + +// CursorScope represents a team-level collection scope. +type CursorScope struct { + common.Scope `mapstructure:",squash"` + Id string `json:"id" mapstructure:"id" gorm:"primaryKey;type:varchar(255)"` + TeamId string `json:"teamId" mapstructure:"teamId" gorm:"type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + FullName string `json:"fullName" mapstructure:"fullName" gorm:"type:varchar(255)"` +} + +func (CursorScope) TableName() string { + return "_tool_cursor_scopes" +} + +func (s *CursorScope) BeforeSave(tx *gorm.DB) error { + if s == nil { + return nil + } + + s.Id = strings.TrimSpace(s.Id) + s.TeamId = strings.TrimSpace(s.TeamId) + s.Name = strings.TrimSpace(s.Name) + s.FullName = strings.TrimSpace(s.FullName) + + if s.Id == "" { + s.Id = DefaultScopeID + } + if s.Name == "" { + s.Name = s.ScopeName() + } + if s.FullName == "" { + s.FullName = s.ScopeFullName() + } + + return nil +} + +func (s CursorScope) ScopeId() string { + return s.Id +} + +func (s CursorScope) ScopeName() string { + if s.Name != "" { + return s.Name + } + return DefaultScopeID +} + +func (s CursorScope) ScopeFullName() string { + if s.FullName != "" { + return s.FullName + } + return s.ScopeName() +} + +func (s CursorScope) ScopeParams() interface{} { + return &CursorScopeParams{ + ConnectionId: s.ConnectionId, + ScopeId: s.Id, + } +} + +// CursorScopeParams is returned for blueprint configuration. +type CursorScopeParams struct { + ConnectionId uint64 `json:"connectionId"` + ScopeId string `json:"scopeId"` +} + +var _ plugin.ToolLayerScope = (*CursorScope)(nil) diff --git a/backend/plugins/cursor/models/scope_config.go b/backend/plugins/cursor/models/scope_config.go new file mode 100644 index 00000000000..64dcf5e259b --- /dev/null +++ b/backend/plugins/cursor/models/scope_config.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScopeConfig = (*CursorScopeConfig)(nil) + +// CursorScopeConfig contains configuration for Cursor data scope. +type CursorScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` +} + +func (CursorScopeConfig) TableName() string { + return "_tool_cursor_scope_configs" +} + +func (sc CursorScopeConfig) GetConnectionId() uint64 { + return sc.ConnectionId +} diff --git a/backend/plugins/cursor/models/usage_event.go b/backend/plugins/cursor/models/usage_event.go new file mode 100644 index 00000000000..7ee7af57df8 --- /dev/null +++ b/backend/plugins/cursor/models/usage_event.go @@ -0,0 +1,57 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// CursorUsageEvent stores one usage event from /teams/filtered-usage-events. +type CursorUsageEvent struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + EventId string `gorm:"primaryKey;type:varchar(64)" json:"eventId"` + EventTime time.Time `gorm:"index" json:"eventTime"` + + UserEmail string `json:"userEmail" gorm:"type:varchar(255);index"` + Model string `json:"model" gorm:"type:varchar(255);index"` + Kind string `json:"kind" gorm:"type:varchar(100)"` + ConversationId string `json:"conversationId" gorm:"type:varchar(64);index"` + ChargedCents float64 `json:"chargedCents"` + RequestsCosts int `json:"requestsCosts"` + IsTokenBasedCall bool `json:"isTokenBasedCall"` + IsChargeable bool `json:"isChargeable"` + MaxMode bool `json:"maxMode"` + IsHeadless bool `json:"isHeadless"` + InputTokens int `json:"inputTokens"` + OutputTokens int `json:"outputTokens"` + CacheReadTokens int `json:"cacheReadTokens"` + CacheWriteTokens int `json:"cacheWriteTokens"` + TotalCents float64 `json:"totalCents"` + CursorTokenFee float64 `json:"cursorTokenFee"` + HostingType string `json:"hostingType" gorm:"type:varchar(50)"` + ServiceAccountId string `json:"serviceAccountId" gorm:"type:varchar(255)"` + + common.NoPKModel +} + +func (CursorUsageEvent) TableName() string { + return "_tool_cursor_usage_events" +} diff --git a/backend/plugins/cursor/models/user_spend.go b/backend/plugins/cursor/models/user_spend.go new file mode 100644 index 00000000000..c936159482d --- /dev/null +++ b/backend/plugins/cursor/models/user_spend.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// CursorUserSpend stores per-user billing cycle spend from /teams/spend. +type CursorUserSpend struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` + BillingCycleStart time.Time `gorm:"primaryKey" json:"billingCycleStart"` + CollectedAt time.Time `gorm:"index" json:"collectedAt"` + + Email string `json:"email" gorm:"type:varchar(255);index"` + Name string `json:"name" gorm:"type:varchar(255)"` + Role string `json:"role" gorm:"type:varchar(50)"` + SpendCents float64 `json:"spendCents"` + IncludedSpendCents float64 `json:"includedSpendCents"` + FastPremiumRequests int `json:"fastPremiumRequests"` + MonthlyLimitDollars float64 `json:"monthlyLimitDollars"` + HardLimitOverrideDollars float64 `json:"hardLimitOverrideDollars"` + + common.NoPKModel +} + +func (CursorUserSpend) TableName() string { + return "_tool_cursor_user_spend" +} diff --git a/backend/plugins/cursor/service/connection_test_helper.go b/backend/plugins/cursor/service/connection_test_helper.go new file mode 100644 index 00000000000..d88e504be9f --- /dev/null +++ b/backend/plugins/cursor/service/connection_test_helper.go @@ -0,0 +1,130 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + stdctx "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + corectx "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" +) + +// TestConnectionResult represents the payload returned by the connection test endpoints. +type TestConnectionResult struct { + Success bool `json:"success"` + Message string `json:"message"` + MemberCount int `json:"memberCount,omitempty"` + HasEnterpriseAnalytics bool `json:"hasEnterpriseAnalytics,omitempty"` +} + +// TestConnection exercises the Cursor Admin API to validate credentials. +func TestConnection(ctx stdctx.Context, br corectx.BasicRes, connection *models.CursorConnection) (*TestConnectionResult, errors.Error) { + if connection == nil { + return nil, errors.BadInput.New("connection is required") + } + + connection.Normalize() + if strings.TrimSpace(connection.Token) == "" { + return nil, errors.BadInput.New("token is required") + } + + apiClient, err := helper.NewApiClientFromConnection(ctx, br, connection) + if err != nil { + return nil, err + } + + if userKeyErr := detectUserApiKey(apiClient); userKeyErr != nil { + return &TestConnectionResult{ + Success: false, + Message: userKeyErr.Error(), + }, nil + } + + res, err := apiClient.Get("teams/members", nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to reach Cursor Admin API") + } + defer res.Body.Close() + + if res.StatusCode == http.StatusUnauthorized { + return &TestConnectionResult{ + Success: false, + Message: "Invalid Team API key. Create a Team Admin key in Dashboard → API Keys (admin:* scope), not a User API key from Settings → Integrations.", + }, nil + } + + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + return &TestConnectionResult{ + Success: false, + Message: fmt.Sprintf("unexpected status %d: %s", res.StatusCode, string(body)), + }, nil + } + + body, readErr := io.ReadAll(res.Body) + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read response body") + } + + var response struct { + TeamMembers []json.RawMessage `json:"teamMembers"` + } + if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { + return &TestConnectionResult{ + Success: false, + Message: fmt.Sprintf("failed to parse response: %v", jsonErr), + }, nil + } + + hasEnterpriseAnalytics := probeEnterpriseAnalytics(apiClient) + + return &TestConnectionResult{ + Success: true, + Message: "Connection validated successfully", + MemberCount: len(response.TeamMembers), + HasEnterpriseAnalytics: hasEnterpriseAnalytics, + }, nil +} + +func detectUserApiKey(apiClient *helper.ApiClient) errors.Error { + res, err := apiClient.Get("v1/me", nil, nil) + if err != nil || res == nil { + return nil + } + defer res.Body.Close() + if res.StatusCode == http.StatusOK { + return errors.BadInput.New("this is a User API key. Create a Team Admin key in Dashboard → API Keys (admin:* scope)") + } + return nil +} + +func probeEnterpriseAnalytics(apiClient *helper.ApiClient) bool { + res, err := apiClient.Get("analytics/team/dau?startDate=7d&endDate=today", nil, nil) + if err != nil || res == nil { + return false + } + defer res.Body.Close() + return res.StatusCode == http.StatusOK +} diff --git a/backend/plugins/cursor/tasks/api_client.go b/backend/plugins/cursor/tasks/api_client.go new file mode 100644 index 00000000000..975e805dc1c --- /dev/null +++ b/backend/plugins/cursor/tasks/api_client.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" +) + +// CreateApiClient creates an async API client for Cursor Admin API collectors. +func CreateApiClient(taskCtx plugin.TaskContext, connection *models.CursorConnection) (*helper.ApiAsyncClient, errors.Error) { + apiClient, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + + apiClient.SetHeaders(map[string]string{ + "Accept": "application/json", + "Content-Type": "application/json", + }) + + rateLimiter := &helper.ApiRateLimitCalculator{UserRateLimitPerHour: connection.RateLimitPerHour} + asyncClient, err := helper.CreateAsyncApiClient(taskCtx, apiClient, rateLimiter) + if err != nil { + return nil, err + } + + apiClient.SetAfterFunction(func(res *http.Response) errors.Error { + return handleCursorRetryAfter(res, taskCtx.GetLogger(), time.Now, time.Sleep) + }) + + return asyncClient, nil +} diff --git a/backend/plugins/cursor/tasks/collector_utils.go b/backend/plugins/cursor/tasks/collector_utils.go new file mode 100644 index 00000000000..e28bc3e232d --- /dev/null +++ b/backend/plugins/cursor/tasks/collector_utils.go @@ -0,0 +1,198 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" +) + +const ( + rawUsageEventsTable = "cursor_usage_events" + rawUserSpendTable = "cursor_user_spend" + rawMembersTable = "cursor_members" + + cursorApiPageSize = 100 + cursorInitialBackfillDays = 90 +) + +type cursorRawParams struct { + ConnectionId uint64 + ScopeId string + Endpoint string +} + +func (p cursorRawParams) GetParams() any { + return p +} + +type cursorTimeRangeInput struct { + StartDateMs int64 `json:"startDateMs"` + EndDateMs int64 `json:"endDateMs"` +} + +type cursorPageState struct { + Page int `json:"page"` +} + +type cursorPagination struct { + NumPages int `json:"numPages"` + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` + HasNextPage bool `json:"hasNextPage"` + HasPreviousPage bool `json:"hasPreviousPage"` + Page int `json:"page"` + TotalPages int `json:"totalPages"` +} + +func computeUsageTimeRangeMs(since *time.Time, now time.Time) (int64, int64) { + end := now.UTC() + start := end.AddDate(0, 0, -cursorInitialBackfillDays) + if since != nil && !since.IsZero() && since.After(start) { + start = since.UTC() + } + return start.UnixMilli(), end.UnixMilli() +} + +func parseUsageEventsResponse(res *http.Response) ([]json.RawMessage, errors.Error) { + body, err := readResponseBody(res) + if err != nil { + return nil, err + } + var response struct { + UsageEvents []json.RawMessage `json:"usageEvents"` + } + if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { + return nil, errors.Default.Wrap(errors.Convert(jsonErr), "failed to decode usage events response") + } + return response.UsageEvents, nil +} + +func parseSpendMembersResponse(res *http.Response) ([]json.RawMessage, errors.Error) { + body, err := readResponseBody(res) + if err != nil { + return nil, err + } + var response struct { + TeamMemberSpend []json.RawMessage `json:"teamMemberSpend"` + } + if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { + return nil, errors.Default.Wrap(errors.Convert(jsonErr), "failed to decode spend response") + } + return response.TeamMemberSpend, nil +} + +func parseMembersResponse(res *http.Response) ([]json.RawMessage, errors.Error) { + body, err := readResponseBody(res) + if err != nil { + return nil, err + } + var response struct { + TeamMembers []json.RawMessage `json:"teamMembers"` + } + if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { + return nil, errors.Default.Wrap(errors.Convert(jsonErr), "failed to decode members response") + } + return response.TeamMembers, nil +} + +func parseSpendMeta(res *http.Response) (int64, errors.Error) { + body, err := readResponseBody(res) + if err != nil { + return 0, err + } + var response struct { + SubscriptionCycleStart int64 `json:"subscriptionCycleStart"` + } + if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { + return 0, errors.Default.Wrap(errors.Convert(jsonErr), "failed to decode spend metadata") + } + return response.SubscriptionCycleStart, nil +} + +func parsePaginationBody(body []byte) (cursorPagination, errors.Error) { + var response struct { + Pagination cursorPagination `json:"pagination"` + } + if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { + return cursorPagination{}, errors.Default.Wrap(errors.Convert(jsonErr), "failed to decode pagination") + } + return response.Pagination, nil +} + +func parsePagination(res *http.Response) (cursorPagination, errors.Error) { + body, err := readResponseBody(res) + if err != nil { + return cursorPagination{}, err + } + return parsePaginationBody(body) +} + +func readResponseBody(res *http.Response) ([]byte, errors.Error) { + if res == nil || res.Body == nil { + return nil, errors.Default.New("response body is nil") + } + defer res.Body.Close() + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to read response body") + } + return body, nil +} + +func parseEventTimestampMs(raw string) (time.Time, errors.Error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return time.Time{}, errors.BadInput.New("event timestamp is empty") + } + ms, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return time.Time{}, errors.BadInput.Wrap(err, "invalid event timestamp") + } + return time.UnixMilli(ms).UTC(), nil +} + +func computeEventId(timestamp, userEmail, conversationId, model string, chargedCents float64, requestsCosts int) string { + payload := fmt.Sprintf("%s|%s|%s|%s|%v|%d", timestamp, userEmail, conversationId, model, chargedCents, requestsCosts) + sum := sha256.Sum256([]byte(payload)) + return hex.EncodeToString(sum[:]) +} + +func normalizeNullableString(value string) string { + value = strings.TrimSpace(value) + if value == "" || strings.EqualFold(value, "null") { + return "" + } + return value +} + +func billingCycleTime(ms int64) time.Time { + if ms <= 0 { + return time.Time{} + } + return time.UnixMilli(ms).UTC() +} diff --git a/backend/plugins/cursor/tasks/members_collector.go b/backend/plugins/cursor/tasks/members_collector.go new file mode 100644 index 00000000000..d4e41da3aaf --- /dev/null +++ b/backend/plugins/cursor/tasks/members_collector.go @@ -0,0 +1,62 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// CollectMembers collects team member roster from GET /teams/members. +func CollectMembers(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*CursorTaskData) + if !ok { + return errors.Default.New("task data is not CursorTaskData") + } + connection := data.Connection + connection.Normalize() + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawMembersTable, + Options: cursorRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ + RawDataSubTaskArgs: rawArgs, + ApiClient: apiClient, + PageSize: 1, + UrlTemplate: "teams/members", + ResponseParser: parseMembersResponse, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/cursor/tasks/members_extractor.go b/backend/plugins/cursor/tasks/members_extractor.go new file mode 100644 index 00000000000..91477a7e645 --- /dev/null +++ b/backend/plugins/cursor/tasks/members_extractor.go @@ -0,0 +1,84 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" +) + +type memberRecord struct { + Name string `json:"name"` + Email string `json:"email"` + Id string `json:"id"` + Role string `json:"role"` + IsRemoved bool `json:"isRemoved"` +} + +// ExtractMembers parses raw member records into tool-layer tables. +func ExtractMembers(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*CursorTaskData) + if !ok { + return errors.Default.New("task data is not CursorTaskData") + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawMembersTable, + Options: cursorRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + }, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var record memberRecord + if err := errors.Convert(json.Unmarshal(row.Data, &record)); err != nil { + return nil, err + } + + userId := strings.TrimSpace(record.Id) + if userId == "" { + userId = strings.TrimSpace(record.Email) + } + if userId == "" { + return nil, nil + } + + member := &models.CursorMember{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + UserId: userId, + Email: strings.TrimSpace(record.Email), + Name: strings.TrimSpace(record.Name), + Role: strings.TrimSpace(record.Role), + IsRemoved: record.IsRemoved, + } + return []interface{}{member}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/cursor/tasks/options.go b/backend/plugins/cursor/tasks/options.go new file mode 100644 index 00000000000..3676db9443a --- /dev/null +++ b/backend/plugins/cursor/tasks/options.go @@ -0,0 +1,24 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +// CursorOptions defines task-level options passed from pipeline plans. +type CursorOptions struct { + ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"` + ScopeId string `mapstructure:"scopeId" json:"scopeId"` +} diff --git a/backend/plugins/cursor/tasks/register.go b/backend/plugins/cursor/tasks/register.go new file mode 100644 index 00000000000..0a3251c2106 --- /dev/null +++ b/backend/plugins/cursor/tasks/register.go @@ -0,0 +1,32 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import "github.com/apache/incubator-devlake/core/plugin" + +// GetSubTaskMetas returns the ordered list of Cursor subtasks. +func GetSubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + CollectMembersMeta, + ExtractMembersMeta, + CollectUsageEventsMeta, + ExtractUsageEventsMeta, + CollectUserSpendMeta, + ExtractUserSpendMeta, + } +} diff --git a/backend/plugins/cursor/tasks/retry_after.go b/backend/plugins/cursor/tasks/retry_after.go new file mode 100644 index 00000000000..4e8b2357730 --- /dev/null +++ b/backend/plugins/cursor/tasks/retry_after.go @@ -0,0 +1,79 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" +) + +func handleCursorRetryAfter(res *http.Response, logger log.Logger, now nowFunc, sleep sleepFunc) errors.Error { + if res == nil || res.StatusCode != http.StatusTooManyRequests { + return nil + } + if now == nil { + now = time.Now + } + if sleep == nil { + sleep = time.Sleep + } + wait := parseRetryAfter(res.Header.Get("Retry-After"), now().UTC()) + if wait > 0 && logger != nil { + logger.Warn(nil, "Cursor returned 429; sleeping %s per Retry-After", wait.String()) + } + if wait > 0 { + sleep(wait) + } + return errors.HttpStatus(http.StatusTooManyRequests).New("Cursor rate limited the request") +} + +func parseRetryAfter(value string, now time.Time) time.Duration { + value = trimSpace(value) + if value == "" { + return 0 + } + if seconds, err := strconv.Atoi(value); err == nil { + return time.Duration(seconds) * time.Second + } + if retryAt, err := http.ParseTime(value); err == nil { + wait := retryAt.Sub(now) + if wait > 0 { + return wait + } + } + return 0 +} + +func trimSpace(s string) string { + start := 0 + for start < len(s) && (s[start] == ' ' || s[start] == '\t' || s[start] == '\n' || s[start] == '\r') { + start++ + } + end := len(s) + for end > start && (s[end-1] == ' ' || s[end-1] == '\t' || s[end-1] == '\n' || s[end-1] == '\r') { + end-- + } + return s[start:end] +} + +type nowFunc func() time.Time +type sleepFunc func(time.Duration) diff --git a/backend/plugins/cursor/tasks/subtasks.go b/backend/plugins/cursor/tasks/subtasks.go new file mode 100644 index 00000000000..98e0e675814 --- /dev/null +++ b/backend/plugins/cursor/tasks/subtasks.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import "github.com/apache/incubator-devlake/core/plugin" + +var CollectUsageEventsMeta = plugin.SubTaskMeta{ + Name: "collectUsageEvents", + EntryPoint: CollectUsageEvents, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect usage events from the Cursor Admin API", +} + +var ExtractUsageEventsMeta = plugin.SubTaskMeta{ + Name: "extractUsageEvents", + EntryPoint: ExtractUsageEvents, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract usage events into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectUsageEventsMeta}, +} + +var CollectUserSpendMeta = plugin.SubTaskMeta{ + Name: "collectUserSpend", + EntryPoint: CollectUserSpend, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect per-user billing cycle spend from the Cursor Admin API", +} + +var ExtractUserSpendMeta = plugin.SubTaskMeta{ + Name: "extractUserSpend", + EntryPoint: ExtractUserSpend, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract per-user billing cycle spend into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectUserSpendMeta}, +} + +var CollectMembersMeta = plugin.SubTaskMeta{ + Name: "collectMembers", + EntryPoint: CollectMembers, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect team member roster from the Cursor Admin API", +} + +var ExtractMembersMeta = plugin.SubTaskMeta{ + Name: "extractMembers", + EntryPoint: ExtractMembers, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract team member roster into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectMembersMeta}, +} diff --git a/backend/plugins/cursor/tasks/task_data.go b/backend/plugins/cursor/tasks/task_data.go new file mode 100644 index 00000000000..6d0cb9a7aa5 --- /dev/null +++ b/backend/plugins/cursor/tasks/task_data.go @@ -0,0 +1,26 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import "github.com/apache/incubator-devlake/plugins/cursor/models" + +// CursorTaskData stores runtime dependencies for subtasks. +type CursorTaskData struct { + Options *CursorOptions + Connection *models.CursorConnection +} diff --git a/backend/plugins/cursor/tasks/usage_events_collector.go b/backend/plugins/cursor/tasks/usage_events_collector.go new file mode 100644 index 00000000000..bf97dd9535b --- /dev/null +++ b/backend/plugins/cursor/tasks/usage_events_collector.go @@ -0,0 +1,113 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +func newTimeRangeIterator(since *time.Time) *helper.QueueIterator { + iter := helper.NewQueueIterator() + startMs, endMs := computeUsageTimeRangeMs(since, time.Now().UTC()) + iter.Push(cursorTimeRangeInput{StartDateMs: startMs, EndDateMs: endMs}) + return iter +} + +func nextPostPage(prevReqData *helper.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + pagination, err := parsePagination(prevPageResponse) + if err != nil { + return nil, err + } + if !pagination.HasNextPage { + return nil, helper.ErrFinishCollect + } + currentPage := 1 + if state, ok := prevReqData.CustomData.(cursorPageState); ok && state.Page > 0 { + currentPage = state.Page + } else if prevReqData.Pager.Page > 0 { + currentPage = prevReqData.Pager.Page + } + return cursorPageState{Page: currentPage + 1}, nil +} + +func postPageBody(reqData *helper.RequestData) map[string]interface{} { + input := reqData.Input.(cursorTimeRangeInput) + page := 1 + if state, ok := reqData.CustomData.(cursorPageState); ok && state.Page > 0 { + page = state.Page + } else if reqData.Pager.Page > 0 { + page = reqData.Pager.Page + } + return map[string]interface{}{ + "startDate": input.StartDateMs, + "endDate": input.EndDateMs, + "page": page, + "pageSize": reqData.Pager.Size, + } +} + +// CollectUsageEvents collects granular usage events from POST /teams/filtered-usage-events. +func CollectUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*CursorTaskData) + if !ok { + return errors.Default.New("task data is not CursorTaskData") + } + connection := data.Connection + connection.Normalize() + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUsageEventsTable, + Options: cursorRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: newTimeRangeIterator(collector.GetSince()), + Method: http.MethodPost, + UrlTemplate: "teams/filtered-usage-events", + PageSize: cursorApiPageSize, + RequestBody: postPageBody, + GetNextPageCustomData: nextPostPage, + ResponseParser: parseUsageEventsResponse, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/cursor/tasks/usage_events_extractor.go b/backend/plugins/cursor/tasks/usage_events_extractor.go new file mode 100644 index 00000000000..e019b65ddf3 --- /dev/null +++ b/backend/plugins/cursor/tasks/usage_events_extractor.go @@ -0,0 +1,121 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" +) + +type usageEventRecord struct { + Timestamp string `json:"timestamp"` + Model string `json:"model"` + Kind string `json:"kind"` + MaxMode bool `json:"maxMode"` + RequestsCosts int `json:"requestsCosts"` + IsTokenBasedCall bool `json:"isTokenBasedCall"` + TokenUsage *usageTokenUsage `json:"tokenUsage"` + UserEmail string `json:"userEmail"` + IsChargeable bool `json:"isChargeable"` + ServiceAccountId string `json:"serviceAccountId"` + IsHeadless bool `json:"isHeadless"` + ChargedCents float64 `json:"chargedCents"` + CursorTokenFee float64 `json:"cursorTokenFee"` + HostingType string `json:"hostingType"` + ConversationId string `json:"conversationId"` +} + +type usageTokenUsage struct { + InputTokens int `json:"inputTokens"` + OutputTokens int `json:"outputTokens"` + CacheWriteTokens int `json:"cacheWriteTokens"` + CacheReadTokens int `json:"cacheReadTokens"` + TotalCents float64 `json:"totalCents"` +} + +// ExtractUsageEvents parses raw usage event records into tool-layer tables. +func ExtractUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*CursorTaskData) + if !ok { + return errors.Default.New("task data is not CursorTaskData") + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUsageEventsTable, + Options: cursorRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + }, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var record usageEventRecord + if err := errors.Convert(json.Unmarshal(row.Data, &record)); err != nil { + return nil, err + } + + eventTime, err := parseEventTimestampMs(record.Timestamp) + if err != nil { + return nil, err + } + + userEmail := strings.TrimSpace(record.UserEmail) + if userEmail == "" { + return nil, nil + } + + event := &models.CursorUsageEvent{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + EventId: computeEventId(record.Timestamp, userEmail, record.ConversationId, record.Model, record.ChargedCents, record.RequestsCosts), + EventTime: eventTime, + UserEmail: userEmail, + Model: strings.TrimSpace(record.Model), + Kind: strings.TrimSpace(record.Kind), + ConversationId: strings.TrimSpace(record.ConversationId), + ChargedCents: record.ChargedCents, + RequestsCosts: record.RequestsCosts, + IsTokenBasedCall: record.IsTokenBasedCall, + IsChargeable: record.IsChargeable, + MaxMode: record.MaxMode, + IsHeadless: record.IsHeadless, + CursorTokenFee: record.CursorTokenFee, + HostingType: strings.TrimSpace(record.HostingType), + ServiceAccountId: normalizeNullableString(record.ServiceAccountId), + } + if record.TokenUsage != nil { + event.InputTokens = record.TokenUsage.InputTokens + event.OutputTokens = record.TokenUsage.OutputTokens + event.CacheReadTokens = record.TokenUsage.CacheReadTokens + event.CacheWriteTokens = record.TokenUsage.CacheWriteTokens + event.TotalCents = record.TokenUsage.TotalCents + } + return []interface{}{event}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/cursor/tasks/user_spend_collector.go b/backend/plugins/cursor/tasks/user_spend_collector.go new file mode 100644 index 00000000000..0a48628440b --- /dev/null +++ b/backend/plugins/cursor/tasks/user_spend_collector.go @@ -0,0 +1,134 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "io" + "net/http" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +type cursorSpendCollectInput struct { + CollectedAt time.Time `json:"collectedAt"` +} + +type spendRawRecord struct { + SubscriptionCycleStart int64 `json:"subscriptionCycleStart"` + CollectedAt time.Time `json:"collectedAt"` + Member json.RawMessage `json:"member"` +} + +func newSpendIterator() *helper.QueueIterator { + iter := helper.NewQueueIterator() + iter.Push(cursorSpendCollectInput{CollectedAt: time.Now().UTC()}) + return iter +} + +func parseSpendPageResponse(res *http.Response) ([]json.RawMessage, errors.Error) { + if res == nil || res.Body == nil { + return nil, errors.Default.New("response body is nil") + } + body, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + return nil, errors.Default.Wrap(err, "failed to read spend response") + } + + var response struct { + SubscriptionCycleStart int64 `json:"subscriptionCycleStart"` + TeamMemberSpend []json.RawMessage `json:"teamMemberSpend"` + } + if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { + return nil, errors.Default.Wrap(errors.Convert(jsonErr), "failed to decode spend response") + } + + collectedAt := time.Now().UTC() + results := make([]json.RawMessage, 0, len(response.TeamMemberSpend)) + for _, member := range response.TeamMemberSpend { + wrapped, marshalErr := json.Marshal(spendRawRecord{ + SubscriptionCycleStart: response.SubscriptionCycleStart, + CollectedAt: collectedAt, + Member: member, + }) + if marshalErr != nil { + return nil, errors.Default.Wrap(errors.Convert(marshalErr), "failed to wrap spend record") + } + results = append(results, wrapped) + } + return results, nil +} + +func spendPageBody(reqData *helper.RequestData) map[string]interface{} { + page := 1 + if state, ok := reqData.CustomData.(cursorPageState); ok && state.Page > 0 { + page = state.Page + } else if reqData.Pager.Page > 0 { + page = reqData.Pager.Page + } + return map[string]interface{}{ + "page": page, + "pageSize": reqData.Pager.Size, + } +} + +// CollectUserSpend collects per-user billing cycle spend from POST /teams/spend. +func CollectUserSpend(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*CursorTaskData) + if !ok { + return errors.Default.New("task data is not CursorTaskData") + } + connection := data.Connection + connection.Normalize() + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserSpendTable, + Options: cursorRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ + RawDataSubTaskArgs: rawArgs, + ApiClient: apiClient, + Input: newSpendIterator(), + Method: http.MethodPost, + UrlTemplate: "teams/spend", + PageSize: cursorApiPageSize, + RequestBody: spendPageBody, + GetNextPageCustomData: nextPostPage, + ResponseParser: parseSpendPageResponse, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/cursor/tasks/user_spend_extractor.go b/backend/plugins/cursor/tasks/user_spend_extractor.go new file mode 100644 index 00000000000..785ed706904 --- /dev/null +++ b/backend/plugins/cursor/tasks/user_spend_extractor.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" +) + +type spendMemberRecord struct { + UserId string `json:"userId"` + Email string `json:"email"` + Name string `json:"name"` + Role string `json:"role"` + SpendCents float64 `json:"spendCents"` + IncludedSpendCents float64 `json:"includedSpendCents"` + FastPremiumRequests int `json:"fastPremiumRequests"` + MonthlyLimitDollars float64 `json:"monthlyLimitDollars"` + HardLimitOverrideDollars float64 `json:"hardLimitOverrideDollars"` +} + +// ExtractUserSpend parses raw spend records into tool-layer tables. +func ExtractUserSpend(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*CursorTaskData) + if !ok { + return errors.Default.New("task data is not CursorTaskData") + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserSpendTable, + Options: cursorRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + }, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var wrapped spendRawRecord + if err := errors.Convert(json.Unmarshal(row.Data, &wrapped)); err != nil { + return nil, err + } + + var record spendMemberRecord + if err := errors.Convert(json.Unmarshal(wrapped.Member, &record)); err != nil { + return nil, err + } + + userId := strings.TrimSpace(record.UserId) + if userId == "" { + userId = strings.TrimSpace(record.Email) + } + if userId == "" { + return nil, nil + } + + spend := &models.CursorUserSpend{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + UserId: userId, + BillingCycleStart: billingCycleTime(wrapped.SubscriptionCycleStart), + CollectedAt: wrapped.CollectedAt, + Email: strings.TrimSpace(record.Email), + Name: strings.TrimSpace(record.Name), + Role: strings.TrimSpace(record.Role), + SpendCents: record.SpendCents, + IncludedSpendCents: record.IncludedSpendCents, + FastPremiumRequests: record.FastPremiumRequests, + MonthlyLimitDollars: record.MonthlyLimitDollars, + HardLimitOverrideDollars: record.HardLimitOverrideDollars, + } + return []interface{}{spend}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index 2c79feaf26f..5567d4b4a22 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -31,6 +31,7 @@ import ( circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" clickup "github.com/apache/incubator-devlake/plugins/clickup/impl" + cursor "github.com/apache/incubator-devlake/plugins/cursor/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" feishu "github.com/apache/incubator-devlake/plugins/feishu/impl" @@ -107,6 +108,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("webhook/models", webhook.Webhook{}.GetTablesInfo) checker.FeedIn("zentao/models", zentao.Zentao{}.GetTablesInfo) checker.FeedIn("claude_code/models", claudeCode.ClaudeCode{}.GetTablesInfo) + checker.FeedIn("cursor/models", cursor.Cursor{}.GetTablesInfo) checker.FeedIn("circleci/models", circleci.Circleci{}.GetTablesInfo) checker.FeedIn("clickup/models", clickup.ClickUp{}.GetTablesInfo) checker.FeedIn("opsgenie/models", opsgenie.Opsgenie{}.GetTablesInfo) diff --git a/config-ui/src/plugins/register/cursor/assets/icon.svg b/config-ui/src/plugins/register/cursor/assets/icon.svg new file mode 100644 index 00000000000..cbd7c824435 --- /dev/null +++ b/config-ui/src/plugins/register/cursor/assets/icon.svg @@ -0,0 +1,22 @@ + + + + diff --git a/config-ui/src/plugins/register/cursor/config.tsx b/config-ui/src/plugins/register/cursor/config.tsx new file mode 100644 index 00000000000..077a780a2fe --- /dev/null +++ b/config-ui/src/plugins/register/cursor/config.tsx @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; +import { Token } from './connection-fields'; + +export const CursorConfig: IPluginConfig = { + plugin: 'cursor', + name: 'Cursor', + icon: ({ color }) => , + sort: 6.7, + isBeta: true, + connection: { + docLink: 'https://cursor.com/docs/account/teams/admin-api', + initialValues: { + endpoint: 'https://api.cursor.com', + token: '', + rateLimitPerHour: 1200, + }, + fields: [ + 'name', + 'endpoint', + ({ type, initialValues, values, setValues, setErrors }: any) => ( + + ), + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: + 'By default, DevLake uses 1,200 requests/hour for Cursor Admin API collection (documented limit is 20 requests/minute). Adjust this value to throttle collection speed.', + defaultValue: 1200, + }, + ], + }, + dataScope: { + title: 'Team', + }, + scopeConfig: { + entities: ['CROSS'], + transformation: {}, + }, +}; diff --git a/config-ui/src/plugins/register/cursor/connection-fields/index.ts b/config-ui/src/plugins/register/cursor/connection-fields/index.ts new file mode 100644 index 00000000000..9c58cfcc30a --- /dev/null +++ b/config-ui/src/plugins/register/cursor/connection-fields/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './token'; diff --git a/config-ui/src/plugins/register/cursor/connection-fields/styled.ts b/config-ui/src/plugins/register/cursor/connection-fields/styled.ts new file mode 100644 index 00000000000..82ba9f717d0 --- /dev/null +++ b/config-ui/src/plugins/register/cursor/connection-fields/styled.ts @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import styled from 'styled-components'; + +export const ErrorText = styled.div` + margin-top: 4px; + color: #f5222d; + font-size: 12px; +`; diff --git a/config-ui/src/plugins/register/cursor/connection-fields/token.tsx b/config-ui/src/plugins/register/cursor/connection-fields/token.tsx new file mode 100644 index 00000000000..5511bab4417 --- /dev/null +++ b/config-ui/src/plugins/register/cursor/connection-fields/token.tsx @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useEffect, useMemo, type ChangeEvent } from 'react'; +import { Input } from 'antd'; + +import { Block } from '@/components'; + +import * as S from './styled'; + +interface Props { + type: 'create' | 'update'; + initialValues: any; + values: any; + setValues: (value: any) => void; + setErrors: (value: any) => void; +} + +export const Token = ({ type, initialValues, values, setValues, setErrors }: Props) => { + useEffect(() => { + setValues({ token: type === 'create' ? initialValues.token ?? '' : undefined }); + }, [type, initialValues.token]); + + const error = useMemo(() => { + if (type === 'update') return ''; + return values.token?.trim() ? '' : 'Team Admin API Key is required'; + }, [type, values.token]); + + useEffect(() => { + setErrors({ token: error }); + }, [error]); + + return ( + + ) => setValues({ token: e.target.value })} + /> + {error && {error}} + + ); +}; diff --git a/config-ui/src/plugins/register/cursor/index.ts b/config-ui/src/plugins/register/cursor/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/cursor/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index 64a719523e4..94bc90e3c4d 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -27,6 +27,7 @@ import { BitbucketServerConfig } from './bitbucket-server'; import { CircleCIConfig } from './circleci'; import { ClaudeCodeConfig } from './claude-code'; import { ClickUpConfig } from './clickup'; +import { CursorConfig } from './cursor'; import { GitHubConfig } from './github'; import { GhCopilotConfig } from './gh-copilot'; import { GitLabConfig } from './gitlab'; @@ -58,6 +59,7 @@ export const pluginConfigs: IPluginConfig[] = [ CircleCIConfig, ClaudeCodeConfig, ClickUpConfig, + CursorConfig, GitHubConfig, GhCopilotConfig, GitLabConfig, From 1eb797156f84ed1fe44fa0d9134c09c044411358 Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Fri, 10 Jul 2026 11:56:26 -0600 Subject: [PATCH 02/13] fix(cursor): Fix members collector infinite loop * Set collectMembers PageSize to zero so teams/members is fetched once instead of re-inserting the full roster forever. * Extend connection test to probe members, spend, and usage events Admin API permissions before pipelines run. --- .../plugins/cursor/service/connection_test.go | 59 ++++++ .../cursor/service/connection_test_helper.go | 172 +++++++++++++++--- .../plugins/cursor/tasks/members_collector.go | 2 +- 3 files changed, 203 insertions(+), 30 deletions(-) create mode 100644 backend/plugins/cursor/service/connection_test.go diff --git a/backend/plugins/cursor/service/connection_test.go b/backend/plugins/cursor/service/connection_test.go new file mode 100644 index 00000000000..c520351f1e5 --- /dev/null +++ b/backend/plugins/cursor/service/connection_test.go @@ -0,0 +1,59 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "io" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuildPermissionFailureMessage(t *testing.T) { + message := buildPermissionFailureMessage([]string{ + "Cannot read team billing spend (teams/spend): forbidden.", + "Cannot read filtered usage events (teams/filtered-usage-events): unauthorized.", + }) + require.Contains(t, message, "missing required permissions") + require.Contains(t, message, "teams/spend") + require.Contains(t, message, "filtered-usage-events") +} + +func TestBuildAdminApiError_Unauthorized(t *testing.T) { + res := &http.Response{ + StatusCode: http.StatusUnauthorized, + Body: io.NopCloser(strings.NewReader(`{"message":"invalid key"}`)), + } + err := buildAdminApiError(spendEndpoint, "read team billing spend", res) + require.Error(t, err) + require.Contains(t, err.Error(), "Team Admin API key") + require.Contains(t, err.Error(), "teams/spend") +} + +func TestBuildAdminApiError_Forbidden(t *testing.T) { + res := &http.Response{ + StatusCode: http.StatusForbidden, + Body: io.NopCloser(strings.NewReader(`{"message":"forbidden"}`)), + } + err := buildAdminApiError(usageEventsEndpoint, "read filtered usage events", res) + require.Error(t, err) + require.Contains(t, err.Error(), "forbidden") + require.Contains(t, err.Error(), "filtered-usage-events") +} diff --git a/backend/plugins/cursor/service/connection_test_helper.go b/backend/plugins/cursor/service/connection_test_helper.go index d88e504be9f..3ae190865b2 100644 --- a/backend/plugins/cursor/service/connection_test_helper.go +++ b/backend/plugins/cursor/service/connection_test_helper.go @@ -24,6 +24,7 @@ import ( "io" "net/http" "strings" + "time" corectx "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" @@ -31,15 +32,29 @@ import ( "github.com/apache/incubator-devlake/plugins/cursor/models" ) +const ( + membersEndpoint = "teams/members" + spendEndpoint = "teams/spend" + usageEventsEndpoint = "teams/filtered-usage-events" +) + +// AdminApiPermissions reports which Cursor Admin API endpoints the key can access. +type AdminApiPermissions struct { + Members bool `json:"members"` + Spend bool `json:"spend"` + UsageEvents bool `json:"usageEvents"` +} + // TestConnectionResult represents the payload returned by the connection test endpoints. type TestConnectionResult struct { - Success bool `json:"success"` - Message string `json:"message"` - MemberCount int `json:"memberCount,omitempty"` - HasEnterpriseAnalytics bool `json:"hasEnterpriseAnalytics,omitempty"` + Success bool `json:"success"` + Message string `json:"message"` + MemberCount int `json:"memberCount,omitempty"` + Permissions AdminApiPermissions `json:"permissions,omitempty"` + HasEnterpriseAnalytics bool `json:"hasEnterpriseAnalytics,omitempty"` } -// TestConnection exercises the Cursor Admin API to validate credentials. +// TestConnection exercises the Cursor Admin API to validate credentials and permissions. func TestConnection(ctx stdctx.Context, br corectx.BasicRes, connection *models.CursorConnection) (*TestConnectionResult, errors.Error) { if connection == nil { return nil, errors.BadInput.New("connection is required") @@ -54,6 +69,10 @@ func TestConnection(ctx stdctx.Context, br corectx.BasicRes, connection *models. if err != nil { return nil, err } + apiClient.SetHeaders(map[string]string{ + "Accept": "application/json", + "Content-Type": "application/json", + }) if userKeyErr := detectUserApiKey(apiClient); userKeyErr != nil { return &TestConnectionResult{ @@ -62,50 +81,145 @@ func TestConnection(ctx stdctx.Context, br corectx.BasicRes, connection *models. }, nil } - res, err := apiClient.Get("teams/members", nil, nil) - if err != nil { - return nil, errors.Default.Wrap(err, "failed to reach Cursor Admin API") + permissions := AdminApiPermissions{} + var failures []string + + memberCount, membersErr := probeMembers(apiClient) + permissions.Members = membersErr == nil + if membersErr != nil { + failures = append(failures, membersErr.Error()) } - defer res.Body.Close() - if res.StatusCode == http.StatusUnauthorized { + if spendErr := probeSpend(apiClient); spendErr != nil { + permissions.Spend = false + failures = append(failures, spendErr.Error()) + } else { + permissions.Spend = true + } + + if usageErr := probeUsageEvents(apiClient); usageErr != nil { + permissions.UsageEvents = false + failures = append(failures, usageErr.Error()) + } else { + permissions.UsageEvents = true + } + + if len(failures) > 0 { return &TestConnectionResult{ - Success: false, - Message: "Invalid Team API key. Create a Team Admin key in Dashboard → API Keys (admin:* scope), not a User API key from Settings → Integrations.", + Success: false, + Message: buildPermissionFailureMessage(failures), + MemberCount: memberCount, + Permissions: permissions, }, nil } + hasEnterpriseAnalytics := probeEnterpriseAnalytics(apiClient) + + return &TestConnectionResult{ + Success: true, + Message: "Team Admin API key validated. Members, spend, and usage events are accessible.", + MemberCount: memberCount, + Permissions: permissions, + HasEnterpriseAnalytics: hasEnterpriseAnalytics, + }, nil +} + +func probeMembers(apiClient *helper.ApiClient) (int, errors.Error) { + res, err := apiClient.Get(membersEndpoint, nil, nil) + if err != nil { + return 0, errors.Default.Wrap(err, "failed to reach Cursor Admin API") + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { - body, _ := io.ReadAll(res.Body) - return &TestConnectionResult{ - Success: false, - Message: fmt.Sprintf("unexpected status %d: %s", res.StatusCode, string(body)), - }, nil + return 0, buildAdminApiError(membersEndpoint, "list team members", res) } body, readErr := io.ReadAll(res.Body) if readErr != nil { - return nil, errors.Default.Wrap(readErr, "failed to read response body") + return 0, errors.Default.Wrap(readErr, "failed to read members response") } var response struct { TeamMembers []json.RawMessage `json:"teamMembers"` } if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { - return &TestConnectionResult{ - Success: false, - Message: fmt.Sprintf("failed to parse response: %v", jsonErr), - }, nil + return 0, errors.Default.Wrap(errors.Convert(jsonErr), "failed to parse members response") } - hasEnterpriseAnalytics := probeEnterpriseAnalytics(apiClient) + return len(response.TeamMembers), nil +} - return &TestConnectionResult{ - Success: true, - Message: "Connection validated successfully", - MemberCount: len(response.TeamMembers), - HasEnterpriseAnalytics: hasEnterpriseAnalytics, - }, nil +func probeSpend(apiClient *helper.ApiClient) errors.Error { + res, err := apiClient.Post(spendEndpoint, nil, map[string]interface{}{ + "page": 1, + "pageSize": 1, + }, nil) + if err != nil { + return errors.Default.Wrap(err, "failed to reach Cursor spend API") + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return buildAdminApiError(spendEndpoint, "read team billing spend", res) + } + return nil +} + +func probeUsageEvents(apiClient *helper.ApiClient) errors.Error { + endMs := time.Now().UTC().UnixMilli() + startMs := endMs - int64(7*24*time.Hour/time.Millisecond) + + res, err := apiClient.Post(usageEventsEndpoint, nil, map[string]interface{}{ + "startDate": startMs, + "endDate": endMs, + "page": 1, + "pageSize": 1, + }, nil) + if err != nil { + return errors.Default.Wrap(err, "failed to reach Cursor usage events API") + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return buildAdminApiError(usageEventsEndpoint, "read filtered usage events", res) + } + return nil +} + +func buildAdminApiError(endpoint, action string, res *http.Response) errors.Error { + body, _ := io.ReadAll(res.Body) + detail := strings.TrimSpace(string(body)) + if detail != "" && len(detail) > 200 { + detail = detail[:200] + "..." + } + + switch res.StatusCode { + case http.StatusUnauthorized: + return errors.BadInput.New(fmt.Sprintf( + "Cannot %s (%s): unauthorized. Use a Team Admin API key from Dashboard → API Keys (admin:* scope), not a User API key from Settings → Integrations.", + action, endpoint, + )) + case http.StatusForbidden: + msg := fmt.Sprintf("Cannot %s (%s): forbidden. The key may lack admin permissions for this team.", action, endpoint) + if detail != "" { + msg = fmt.Sprintf("%s Details: %s", msg, detail) + } + return errors.BadInput.New(msg) + default: + msg := fmt.Sprintf("Cannot %s (%s): unexpected status %d", action, endpoint, res.StatusCode) + if detail != "" { + msg = fmt.Sprintf("%s. Details: %s", msg, detail) + } + return errors.BadInput.New(msg) + } +} + +func buildPermissionFailureMessage(failures []string) string { + if len(failures) == 1 { + return failures[0] + } + return "Team Admin API key is missing required permissions:\n- " + strings.Join(failures, "\n- ") } func detectUserApiKey(apiClient *helper.ApiClient) errors.Error { diff --git a/backend/plugins/cursor/tasks/members_collector.go b/backend/plugins/cursor/tasks/members_collector.go index d4e41da3aaf..7dd967690f8 100644 --- a/backend/plugins/cursor/tasks/members_collector.go +++ b/backend/plugins/cursor/tasks/members_collector.go @@ -50,7 +50,7 @@ func CollectMembers(taskCtx plugin.SubTaskContext) errors.Error { collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ RawDataSubTaskArgs: rawArgs, ApiClient: apiClient, - PageSize: 1, + PageSize: 0, UrlTemplate: "teams/members", ResponseParser: parseMembersResponse, }) From a700ced1ac17d9b3339e3c4fd7d86ecb0201b51a Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Mon, 13 Jul 2026 14:51:43 -0600 Subject: [PATCH 03/13] fix(cursor): align extractor raw params and handle fractional requestsCosts Signed-off-by: Joshua Smith --- backend/plugins/cursor/e2e/extractors_test.go | 85 +++++++++++++++++++ .../e2e/raw_tables/_raw_cursor_members.csv | 2 + .../raw_tables/_raw_cursor_usage_events.csv | 2 + .../e2e/raw_tables/_raw_cursor_user_spend.csv | 2 + .../snapshot_tables/_tool_cursor_members.csv | 2 + .../_tool_cursor_usage_events.csv | 2 + .../_tool_cursor_user_spend.csv | 2 + .../20260710_requests_costs_float.go | 39 +++++++++ .../models/migrationscripts/register.go | 1 + backend/plugins/cursor/models/usage_event.go | 2 +- .../plugins/cursor/tasks/collector_utils.go | 18 +++- .../cursor/tasks/collector_utils_test.go | 51 +++++++++++ .../plugins/cursor/tasks/members_collector.go | 15 +--- .../plugins/cursor/tasks/members_extractor.go | 5 +- .../cursor/tasks/usage_events_collector.go | 15 +--- .../cursor/tasks/usage_events_extractor.go | 7 +- .../tasks/usage_events_extractor_test.go | 65 ++++++++++++++ .../cursor/tasks/user_spend_collector.go | 15 +--- .../cursor/tasks/user_spend_extractor.go | 5 +- 19 files changed, 286 insertions(+), 49 deletions(-) create mode 100644 backend/plugins/cursor/e2e/extractors_test.go create mode 100644 backend/plugins/cursor/e2e/raw_tables/_raw_cursor_members.csv create mode 100644 backend/plugins/cursor/e2e/raw_tables/_raw_cursor_usage_events.csv create mode 100644 backend/plugins/cursor/e2e/raw_tables/_raw_cursor_user_spend.csv create mode 100644 backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_members.csv create mode 100644 backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_usage_events.csv create mode 100644 backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_user_spend.csv create mode 100644 backend/plugins/cursor/models/migrationscripts/20260710_requests_costs_float.go create mode 100644 backend/plugins/cursor/tasks/collector_utils_test.go create mode 100644 backend/plugins/cursor/tasks/usage_events_extractor_test.go diff --git a/backend/plugins/cursor/e2e/extractors_test.go b/backend/plugins/cursor/e2e/extractors_test.go new file mode 100644 index 00000000000..caebd291e6b --- /dev/null +++ b/backend/plugins/cursor/e2e/extractors_test.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + "time" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/impl" + "github.com/apache/incubator-devlake/plugins/cursor/models" + "github.com/apache/incubator-devlake/plugins/cursor/tasks" +) + +func TestCursorExtractorsDataFlow(t *testing.T) { + cfg := config.GetConfig() + dbUrl := cfg.GetString("E2E_DB_URL") + if dbUrl == "" { + t.Skip("skipping e2e test: E2E_DB_URL is not set") + } + if err := runner.CheckDbConnection(dbUrl, 10*time.Second); err != nil { + t.Skipf("skipping e2e test: cannot connect to E2E_DB_URL: %v", err) + } + + var cursorPlugin impl.Cursor + dataflowTester := e2ehelper.NewDataFlowTester(t, "cursor", cursorPlugin) + + taskData := &tasks.CursorTaskData{ + Options: &tasks.CursorOptions{ + ConnectionId: 1, + ScopeId: "team", + }, + Connection: &models.CursorConnection{ + CursorConn: models.CursorConn{ + RestConnection: helper.RestConnection{Endpoint: models.DefaultEndpoint}, + }, + }, + } + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_cursor_usage_events.csv", "_raw_cursor_usage_events") + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_cursor_members.csv", "_raw_cursor_members") + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_cursor_user_spend.csv", "_raw_cursor_user_spend") + + dataflowTester.FlushTabler(&models.CursorUsageEvent{}) + dataflowTester.FlushTabler(&models.CursorMember{}) + dataflowTester.FlushTabler(&models.CursorUserSpend{}) + + dataflowTester.Subtask(tasks.ExtractMembersMeta, taskData) + dataflowTester.Subtask(tasks.ExtractUsageEventsMeta, taskData) + dataflowTester.Subtask(tasks.ExtractUserSpendMeta, taskData) + + dataflowTester.VerifyTableWithOptions(&models.CursorUsageEvent{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_cursor_usage_events.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + dataflowTester.VerifyTableWithOptions(&models.CursorMember{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_cursor_members.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + dataflowTester.VerifyTableWithOptions(&models.CursorUserSpend{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_cursor_user_spend.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_members.csv b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_members.csv new file mode 100644 index 00000000000..6952772e184 --- /dev/null +++ b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_members.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""name"":""Example User"",""email"":""user@example.com"",""id"":""user_example123"",""role"":""member"",""isRemoved"":false}",https://api.cursor.com/teams/members,null,2026-07-09 17:14:19.000 diff --git a/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_usage_events.csv b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_usage_events.csv new file mode 100644 index 00000000000..66f3948525a --- /dev/null +++ b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_usage_events.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""timestamp"":""1783617259310"",""model"":""composer-2.5-fast"",""kind"":""Usage-based"",""maxMode"":false,""requestsCosts"":2,""isTokenBasedCall"":false,""tokenUsage"":{""inputTokens"":5932,""outputTokens"":4761,""cacheWriteTokens"":0,""cacheReadTokens"":495701,""totalCents"":33.70615},""userEmail"":""user@example.com"",""isChargeable"":true,""serviceAccountId"":""null"",""isHeadless"":false,""chargedCents"":8,""conversationId"":""846952e0-0221-473b-bb1d-e5d016f071b1""}",https://api.cursor.com/teams/filtered-usage-events,null,2026-07-09 17:14:19.000 diff --git a/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_user_spend.csv b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_user_spend.csv new file mode 100644 index 00000000000..d4c4131147f --- /dev/null +++ b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_user_spend.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""subscriptionCycleStart"":1781472188000,""collectedAt"":""2026-07-10T17:31:24.141244Z"",""member"":{""userId"":""user_example123"",""spendCents"":3224,""includedSpendCents"":1984,""fastPremiumRequests"":496,""name"":""Example User"",""email"":""user@example.com"",""profilePictureUrl"":null,""role"":""member"",""monthlyLimitDollars"":null,""hardLimitOverrideDollars"":0}}",https://api.cursor.com/teams/spend,null,2026-07-10 17:31:24.000 diff --git a/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_members.csv b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_members.csv new file mode 100644 index 00000000000..8833567a1f4 --- /dev/null +++ b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_members.csv @@ -0,0 +1,2 @@ +connection_id,scope_id,user_id,email,name,role,is_removed +1,team,user_example123,user@example.com,Example User,member,0 diff --git a/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_usage_events.csv b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_usage_events.csv new file mode 100644 index 00000000000..84b99c6284c --- /dev/null +++ b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_usage_events.csv @@ -0,0 +1,2 @@ +connection_id,scope_id,event_id,event_time,user_email,model,kind,conversation_id,charged_cents,requests_costs,is_token_based_call,is_chargeable,max_mode,is_headless,input_tokens,output_tokens,cache_read_tokens,cache_write_tokens,total_cents,cursor_token_fee,hosting_type,service_account_id +1,team,2fe8f95cb983e68c7e944264bf1cc976c4940e1d3045f8fd5715fb66466d6eb7,2026-07-09T17:14:19.310+00:00,user@example.com,composer-2.5-fast,Usage-based,846952e0-0221-473b-bb1d-e5d016f071b1,8,2,0,1,0,0,5932,4761,495701,0,33.70615,0,, diff --git a/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_user_spend.csv b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_user_spend.csv new file mode 100644 index 00000000000..f1a1bcdbca5 --- /dev/null +++ b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_user_spend.csv @@ -0,0 +1,2 @@ +connection_id,scope_id,user_id,billing_cycle_start,collected_at,email,name,role,spend_cents,included_spend_cents,fast_premium_requests,monthly_limit_dollars,hard_limit_override_dollars +1,team,user_example123,2026-06-14T21:23:08.000+00:00,2026-07-10T17:31:24.141+00:00,user@example.com,Example User,member,3224,1984,496,0,0 diff --git a/backend/plugins/cursor/models/migrationscripts/20260710_requests_costs_float.go b/backend/plugins/cursor/models/migrationscripts/20260710_requests_costs_float.go new file mode 100644 index 00000000000..2b4acf9cb4e --- /dev/null +++ b/backend/plugins/cursor/models/migrationscripts/20260710_requests_costs_float.go @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" +) + +type changeCursorRequestsCostsToFloat struct{} + +func (script *changeCursorRequestsCostsToFloat) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if !db.HasTable("_tool_cursor_usage_events") { + return nil + } + return db.Exec("ALTER TABLE _tool_cursor_usage_events MODIFY COLUMN requests_costs DOUBLE") +} + +func (*changeCursorRequestsCostsToFloat) Version() uint64 { return 20260710120000 } + +func (*changeCursorRequestsCostsToFloat) Name() string { + return "cursor change requests_costs column to double" +} diff --git a/backend/plugins/cursor/models/migrationscripts/register.go b/backend/plugins/cursor/models/migrationscripts/register.go index 527301e3394..f28544150e0 100644 --- a/backend/plugins/cursor/models/migrationscripts/register.go +++ b/backend/plugins/cursor/models/migrationscripts/register.go @@ -23,5 +23,6 @@ import "github.com/apache/incubator-devlake/core/plugin" func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addCursorInitialTables), + new(changeCursorRequestsCostsToFloat), } } diff --git a/backend/plugins/cursor/models/usage_event.go b/backend/plugins/cursor/models/usage_event.go index 7ee7af57df8..5ae4f7e7f8a 100644 --- a/backend/plugins/cursor/models/usage_event.go +++ b/backend/plugins/cursor/models/usage_event.go @@ -35,7 +35,7 @@ type CursorUsageEvent struct { Kind string `json:"kind" gorm:"type:varchar(100)"` ConversationId string `json:"conversationId" gorm:"type:varchar(64);index"` ChargedCents float64 `json:"chargedCents"` - RequestsCosts int `json:"requestsCosts"` + RequestsCosts float64 `json:"requestsCosts"` IsTokenBasedCall bool `json:"isTokenBasedCall"` IsChargeable bool `json:"isChargeable"` MaxMode bool `json:"maxMode"` diff --git a/backend/plugins/cursor/tasks/collector_utils.go b/backend/plugins/cursor/tasks/collector_utils.go index e28bc3e232d..5083f796e83 100644 --- a/backend/plugins/cursor/tasks/collector_utils.go +++ b/backend/plugins/cursor/tasks/collector_utils.go @@ -29,6 +29,7 @@ import ( "time" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/plugins/cursor/models" ) const ( @@ -50,6 +51,19 @@ func (p cursorRawParams) GetParams() any { return p } +func rawParamsFromTaskData(data *CursorTaskData) cursorRawParams { + endpoint := models.DefaultEndpoint + if data.Connection != nil { + data.Connection.Normalize() + endpoint = data.Connection.Endpoint + } + return cursorRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Endpoint: endpoint, + } +} + type cursorTimeRangeInput struct { StartDateMs int64 `json:"startDateMs"` EndDateMs int64 `json:"endDateMs"` @@ -176,8 +190,8 @@ func parseEventTimestampMs(raw string) (time.Time, errors.Error) { return time.UnixMilli(ms).UTC(), nil } -func computeEventId(timestamp, userEmail, conversationId, model string, chargedCents float64, requestsCosts int) string { - payload := fmt.Sprintf("%s|%s|%s|%s|%v|%d", timestamp, userEmail, conversationId, model, chargedCents, requestsCosts) +func computeEventId(timestamp, userEmail, conversationId, model string, chargedCents, requestsCosts float64) string { + payload := fmt.Sprintf("%s|%s|%s|%s|%v|%v", timestamp, userEmail, conversationId, model, chargedCents, requestsCosts) sum := sha256.Sum256([]byte(payload)) return hex.EncodeToString(sum[:]) } diff --git a/backend/plugins/cursor/tasks/collector_utils_test.go b/backend/plugins/cursor/tasks/collector_utils_test.go new file mode 100644 index 00000000000..4b82fcbec7f --- /dev/null +++ b/backend/plugins/cursor/tasks/collector_utils_test.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" +) + +func TestRawParamsFromTaskDataIncludesEndpoint(t *testing.T) { + taskData := &CursorTaskData{ + Options: &CursorOptions{ + ConnectionId: 1, + ScopeId: "team", + }, + Connection: &models.CursorConnection{ + CursorConn: models.CursorConn{ + RestConnection: helper.RestConnection{Endpoint: models.DefaultEndpoint}, + }, + }, + } + + params := rawParamsFromTaskData(taskData) + encoded, err := json.Marshal(params.GetParams()) + if err != nil { + t.Fatalf("marshal params: %v", err) + } + + expected := `{"ConnectionId":1,"ScopeId":"team","Endpoint":"https://api.cursor.com"}` + if string(encoded) != expected { + t.Fatalf("raw params mismatch:\n got: %s\nwant: %s", encoded, expected) + } +} diff --git a/backend/plugins/cursor/tasks/members_collector.go b/backend/plugins/cursor/tasks/members_collector.go index 7dd967690f8..1e741096358 100644 --- a/backend/plugins/cursor/tasks/members_collector.go +++ b/backend/plugins/cursor/tasks/members_collector.go @@ -29,22 +29,15 @@ func CollectMembers(taskCtx plugin.SubTaskContext) errors.Error { if !ok { return errors.Default.New("task data is not CursorTaskData") } - connection := data.Connection - connection.Normalize() - - apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + apiClient, err := CreateApiClient(taskCtx.TaskContext(), data.Connection) if err != nil { return err } rawArgs := helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Table: rawMembersTable, - Options: cursorRawParams{ - ConnectionId: data.Options.ConnectionId, - ScopeId: data.Options.ScopeId, - Endpoint: connection.Endpoint, - }, + Ctx: taskCtx, + Table: rawMembersTable, + Options: rawParamsFromTaskData(data), } collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ diff --git a/backend/plugins/cursor/tasks/members_extractor.go b/backend/plugins/cursor/tasks/members_extractor.go index 91477a7e645..e541b2f3af1 100644 --- a/backend/plugins/cursor/tasks/members_extractor.go +++ b/backend/plugins/cursor/tasks/members_extractor.go @@ -46,10 +46,7 @@ func ExtractMembers(taskCtx plugin.SubTaskContext) errors.Error { RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Ctx: taskCtx, Table: rawMembersTable, - Options: cursorRawParams{ - ConnectionId: data.Options.ConnectionId, - ScopeId: data.Options.ScopeId, - }, + Options: rawParamsFromTaskData(data), }, Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { var record memberRecord diff --git a/backend/plugins/cursor/tasks/usage_events_collector.go b/backend/plugins/cursor/tasks/usage_events_collector.go index bf97dd9535b..a72310deeb0 100644 --- a/backend/plugins/cursor/tasks/usage_events_collector.go +++ b/backend/plugins/cursor/tasks/usage_events_collector.go @@ -72,22 +72,15 @@ func CollectUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { if !ok { return errors.Default.New("task data is not CursorTaskData") } - connection := data.Connection - connection.Normalize() - - apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + apiClient, err := CreateApiClient(taskCtx.TaskContext(), data.Connection) if err != nil { return err } rawArgs := helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Table: rawUsageEventsTable, - Options: cursorRawParams{ - ConnectionId: data.Options.ConnectionId, - ScopeId: data.Options.ScopeId, - Endpoint: connection.Endpoint, - }, + Ctx: taskCtx, + Table: rawUsageEventsTable, + Options: rawParamsFromTaskData(data), } collector, err := helper.NewStatefulApiCollector(rawArgs) diff --git a/backend/plugins/cursor/tasks/usage_events_extractor.go b/backend/plugins/cursor/tasks/usage_events_extractor.go index e019b65ddf3..efc2fab155b 100644 --- a/backend/plugins/cursor/tasks/usage_events_extractor.go +++ b/backend/plugins/cursor/tasks/usage_events_extractor.go @@ -32,7 +32,7 @@ type usageEventRecord struct { Model string `json:"model"` Kind string `json:"kind"` MaxMode bool `json:"maxMode"` - RequestsCosts int `json:"requestsCosts"` + RequestsCosts float64 `json:"requestsCosts"` IsTokenBasedCall bool `json:"isTokenBasedCall"` TokenUsage *usageTokenUsage `json:"tokenUsage"` UserEmail string `json:"userEmail"` @@ -64,10 +64,7 @@ func ExtractUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Ctx: taskCtx, Table: rawUsageEventsTable, - Options: cursorRawParams{ - ConnectionId: data.Options.ConnectionId, - ScopeId: data.Options.ScopeId, - }, + Options: rawParamsFromTaskData(data), }, Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { var record usageEventRecord diff --git a/backend/plugins/cursor/tasks/usage_events_extractor_test.go b/backend/plugins/cursor/tasks/usage_events_extractor_test.go new file mode 100644 index 00000000000..51d62f1a1e0 --- /dev/null +++ b/backend/plugins/cursor/tasks/usage_events_extractor_test.go @@ -0,0 +1,65 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" +) + +func TestUsageEventRecordUnmarshalFractionalRequestsCosts(t *testing.T) { + raw := []byte(`{ + "timestamp":"1783617259310", + "model":"composer-2.5-fast", + "kind":"Usage-based", + "requestsCosts":3.8, + "userEmail":"user@example.com", + "chargedCents":8, + "conversationId":"846952e0-0221-473b-bb1d-e5d016f071b1" + }`) + + var record usageEventRecord + if err := json.Unmarshal(raw, &record); err != nil { + t.Fatalf("unmarshal usage event: %v", err) + } + if record.RequestsCosts != 3.8 { + t.Fatalf("requestsCosts = %v, want 3.8", record.RequestsCosts) + } +} + +func TestComputeEventIdWithFractionalRequestsCosts(t *testing.T) { + fractional := computeEventId( + "1783617259310", + "user@example.com", + "846952e0-0221-473b-bb1d-e5d016f071b1", + "composer-2.5-fast", + 8, + 3.8, + ) + integer := computeEventId( + "1783617259310", + "user@example.com", + "846952e0-0221-473b-bb1d-e5d016f071b1", + "composer-2.5-fast", + 8, + 2, + ) + if fractional == integer { + t.Fatal("expected fractional requestsCosts to change event id") + } +} diff --git a/backend/plugins/cursor/tasks/user_spend_collector.go b/backend/plugins/cursor/tasks/user_spend_collector.go index 0a48628440b..9d619606d3f 100644 --- a/backend/plugins/cursor/tasks/user_spend_collector.go +++ b/backend/plugins/cursor/tasks/user_spend_collector.go @@ -97,22 +97,15 @@ func CollectUserSpend(taskCtx plugin.SubTaskContext) errors.Error { if !ok { return errors.Default.New("task data is not CursorTaskData") } - connection := data.Connection - connection.Normalize() - - apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + apiClient, err := CreateApiClient(taskCtx.TaskContext(), data.Connection) if err != nil { return err } rawArgs := helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Table: rawUserSpendTable, - Options: cursorRawParams{ - ConnectionId: data.Options.ConnectionId, - ScopeId: data.Options.ScopeId, - Endpoint: connection.Endpoint, - }, + Ctx: taskCtx, + Table: rawUserSpendTable, + Options: rawParamsFromTaskData(data), } collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ diff --git a/backend/plugins/cursor/tasks/user_spend_extractor.go b/backend/plugins/cursor/tasks/user_spend_extractor.go index 785ed706904..8f705ada864 100644 --- a/backend/plugins/cursor/tasks/user_spend_extractor.go +++ b/backend/plugins/cursor/tasks/user_spend_extractor.go @@ -50,10 +50,7 @@ func ExtractUserSpend(taskCtx plugin.SubTaskContext) errors.Error { RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ Ctx: taskCtx, Table: rawUserSpendTable, - Options: cursorRawParams{ - ConnectionId: data.Options.ConnectionId, - ScopeId: data.Options.ScopeId, - }, + Options: rawParamsFromTaskData(data), }, Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { var wrapped spendRawRecord From aa3469a339680f8802866b29c1ef8b707517fc7d Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Mon, 13 Jul 2026 14:54:20 -0600 Subject: [PATCH 04/13] feat(grafana): add Cursor usage and cost dashboard Signed-off-by: Joshua Smith --- grafana/dashboards/mysql/cursor-usage.json | 625 +++++++++++++++++++++ 1 file changed, 625 insertions(+) create mode 100644 grafana/dashboards/mysql/cursor-usage.json diff --git a/grafana/dashboards/mysql/cursor-usage.json b/grafana/dashboards/mysql/cursor-usage.json new file mode 100644 index 00000000000..938dd740bd0 --- /dev/null +++ b/grafana/dashboards/mysql/cursor-usage.json @@ -0,0 +1,625 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "datasource", "uid": "grafana" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 100, + "title": "Overview", + "type": "row" + }, + { + "datasource": "mysql", + "description": "Sum of chargedCents for billable usage events in the selected time range", + "fieldConfig": { + "defaults": { + "decimals": 2, + "unit": "currencyUSD", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(charged_cents) / 100 AS \"Charged Amount\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Charged Spend (Period)", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Distinct users with usage events in the selected time range", + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_email) AS \"Active Users\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Active Users (Period)", + "type": "stat" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "purple", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS \"Usage Events\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Usage Events (Period)", + "type": "stat" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS \"Team Members\"\nFROM _tool_cursor_members\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_removed = 0", + "refId": "A" + } + ], + "title": "Team Members", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Current billing cycle on-demand spend from /teams/spend", + "fieldConfig": { + "defaults": { + "decimals": 2, + "unit": "currencyUSD", + "thresholds": { "mode": "absolute", "steps": [{ "color": "orange", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(spend_cents) / 100 AS \"Charged Amount\"\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Billing Cycle Spend", + "type": "stat" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 2 }, + { "color": "red", "value": 7 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(TIMESTAMPDIFF(DAY, MAX(event_time), UTC_TIMESTAMP()), 9999) AS \"Days Since Latest Event\"\nFROM _tool_cursor_usage_events\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Data Freshness", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "id": 101, + "title": "Cost Trends", + "type": "row" + }, + { + "datasource": "mysql", + "description": "Daily charged spend from usage events", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisLabel": "USD", + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "showPoints": "never", + "stacking": { "group": "A", "mode": "none" } + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "id": 10, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT DATE(event_time) AS time,\n SUM(charged_cents) / 100 AS \"Charged Amount\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY DATE(event_time)\nORDER BY 1", + "refId": "A" + } + ], + "title": "Daily Charged Spend", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Daily event count by billing kind", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "stacking": { "group": "A", "mode": "normal" } + } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "id": 11, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" }, + "stacking": { "mode": "normal", "group": "A" } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT DATE(event_time) AS time,\n kind AS metric,\n COUNT(*) AS value\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY DATE(event_time), kind\nORDER BY 1", + "refId": "A" + } + ], + "title": "Events by Kind (Daily)", + "type": "timeseries", + "transformations": [ + { "id": "prepareTimeSeries", "options": { "format": "many" } }, + { "id": "renameByRegex", "options": { "regex": "^value (.+)$", "renamePattern": "$1" } } + ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "id": 102, + "title": "Breakdowns", + "type": "row" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Charged" }, + "properties": [ + { "id": "custom.align", "value": "right" }, + { "id": "custom.width", "value": 90 } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 15 }, + "id": 20, + "options": { + "cellHeight": "sm", + "footer": { "show": false }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(MAX(NULLIF(m.name, '')), e.user_email) AS \"User\",\n CONCAT('$', FORMAT(SUM(e.charged_cents) / 100, 2)) AS \"Charged\"\nFROM _tool_cursor_usage_events e\nLEFT JOIN _tool_cursor_members m\n ON e.user_email = m.email\n AND e.connection_id = m.connection_id\n AND e.scope_id = m.scope_id\n AND m.is_removed = 0\nWHERE $__timeFilter(e.event_time)\n AND e.connection_id = ${connection_id}\n AND e.scope_id = '${scope_id}'\nGROUP BY e.user_email\nORDER BY SUM(e.charged_cents) DESC\nLIMIT 25", + "refId": "A" + } + ], + "title": "Cost by User", + "type": "table" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, + "decimals": 0 + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Model" }, + "properties": [{ "id": "custom.width", "value": 320 }] + }, + { + "matcher": { "id": "byName", "options": "Model Cost" }, + "properties": [ + { "id": "unit", "value": "currencyUSD" }, + { "id": "decimals", "value": 4 }, + { "id": "custom.width", "value": 110 } + ] + }, + { + "matcher": { "id": "byName", "options": "Charged" }, + "properties": [ + { "id": "unit", "value": "currencyUSD" }, + { "id": "decimals", "value": 2 }, + { "id": "custom.width", "value": 90 } + ] + }, + { + "matcher": { "id": "byName", "options": "Events" }, + "properties": [ + { "id": "unit", "value": "short" }, + { "id": "custom.width", "value": 90 } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 10, "x": 6, "y": 15 }, + "id": 21, + "options": { + "cellHeight": "sm", + "footer": { "show": false }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT model AS \"Model\",\n SUM(total_cents) / COUNT(*) / 100 AS \"Model Cost\",\n SUM(charged_cents) / 100 AS \"Charged\",\n COUNT(*) AS \"Events\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY model\nORDER BY SUM(charged_cents) DESC", + "refId": "A" + } + ], + "title": "Cost by Model", + "type": "table" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Charged" }, + "properties": [ + { "id": "unit", "value": "currencyUSD" }, + { "id": "decimals", "value": 2 } + ] + } + ] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 15 }, + "id": 22, + "options": { + "displayLabels": ["name", "percent"], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": ["value"] + }, + "pieType": "donut", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": true }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT kind AS \"Kind\",\n SUM(charged_cents) / 100 AS \"Charged\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY kind\nORDER BY SUM(charged_cents) DESC", + "refId": "A" + } + ], + "title": "Cost by Kind", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "id": 103, + "title": "Billing Cycle & Reconciliation", + "type": "row" + }, + { + "datasource": "mysql", + "description": "Per-user billing cycle snapshot from POST /teams/spend", + "fieldConfig": { + "defaults": { + "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false } + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "On-Demand|Included" }, + "properties": [ + { "id": "unit", "value": "currencyUSD" }, + { "id": "decimals", "value": 2 } + ] + }, + { + "matcher": { "id": "byName", "options": "Premium Reqs" }, + "properties": [{ "id": "unit", "value": "short" }] + } + ] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "id": 30, + "options": { + "cellHeight": "sm", + "footer": { "show": false }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT email AS \"User\",\n name AS \"Name\",\n role AS \"Role\",\n spend_cents / 100 AS \"On-Demand\",\n included_spend_cents / 100 AS \"Included\",\n fast_premium_requests AS \"Premium Reqs\"\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY spend_cents DESC\nLIMIT 50", + "refId": "A" + } + ], + "title": "Billing Cycle Spend by User", + "type": "table" + }, + { + "datasource": "mysql", + "description": "Compare billing cycle spend vs sum(chargedCents) since cycle start. Requires full event backfill for accurate reconciliation.", + "fieldConfig": { + "defaults": { + "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false } + }, + "overrides": [ + { + "matcher": { "id": "byRegexp", "options": "Cycle Spend|Event Charged" }, + "properties": [ + { "id": "unit", "value": "currencyUSD" }, + { "id": "decimals", "value": 2 } + ] + }, + { + "matcher": { "id": "byName", "options": "Events Since Cycle Start" }, + "properties": [{ "id": "unit", "value": "short" }] + } + ] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "id": 31, + "options": { + "cellHeight": "sm", + "footer": { "show": false }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT s.email AS \"User\",\n s.spend_cents / 100 AS \"Cycle Spend\",\n COALESCE(SUM(e.charged_cents), 0) / 100 AS \"Event Charged\",\n COUNT(e.event_id) AS \"Events Since Cycle Start\"\nFROM _tool_cursor_user_spend s\nLEFT JOIN _tool_cursor_usage_events e\n ON s.email = e.user_email\n AND e.event_time >= s.billing_cycle_start\n AND e.connection_id = s.connection_id\n AND e.scope_id = s.scope_id\nWHERE s.connection_id = ${connection_id}\n AND s.scope_id = '${scope_id}'\nGROUP BY s.email, s.spend_cents\nHAVING COUNT(e.event_id) > 0 OR s.spend_cents > 0\nORDER BY s.spend_cents DESC\nLIMIT 50", + "refId": "A" + } + ], + "title": "Spend vs Events Reconciliation", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, + "id": 104, + "title": "Diagnostics", + "type": "row" + }, + { + "datasource": "mysql", + "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 33 }, + "id": 40, + "options": { + "cellHeight": "sm", + "footer": { "show": false }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT '_tool_cursor_usage_events' AS table_name, COUNT(*) AS row_count, MAX(event_time) AS latest\nFROM _tool_cursor_usage_events\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\nUNION ALL\nSELECT '_tool_cursor_user_spend', COUNT(*), MAX(collected_at)\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\nUNION ALL\nSELECT '_tool_cursor_members', COUNT(*), NULL\nFROM _tool_cursor_members\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Data Volume", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": ["cursor", "devlake", "ai"], + "templating": { + "list": [ + { + "current": { "selected": false, "text": "1", "value": "1" }, + "datasource": "mysql", + "definition": "SELECT DISTINCT connection_id FROM _tool_cursor_scopes ORDER BY connection_id", + "hide": 0, + "includeAll": false, + "label": "Connection ID", + "name": "connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_cursor_scopes ORDER BY connection_id", + "refresh": 1, + "regex": "", + "sort": 0, + "type": "query" + }, + { + "current": { "selected": false, "text": "team", "value": "team" }, + "datasource": "mysql", + "definition": "SELECT id AS scope_id FROM _tool_cursor_scopes WHERE connection_id = ${connection_id} ORDER BY id", + "hide": 0, + "includeAll": false, + "label": "Scope ID", + "name": "scope_id", + "options": [], + "query": "SELECT id AS scope_id FROM _tool_cursor_scopes WHERE connection_id = ${connection_id} ORDER BY id", + "refresh": 2, + "regex": "", + "sort": 0, + "type": "query" + } + ] + }, + "time": { "from": "now-90d", "to": "now" }, + "timepicker": {}, + "timezone": "utc", + "title": "Cursor Usage & Cost", + "uid": "cursor_usage", + "version": 3, + "weekStart": "" +} From 8b773d0f6809fc7b043b6787b27145fbd473c2db Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Thu, 16 Jul 2026 11:28:56 -0600 Subject: [PATCH 05/13] feat(cursor): add daily usage and refresh dashboards * Add daily usage collector, extractor, and table for per-user adoption metrics. * Reorganize Cursor Usage overview panels and update related Grafana dashboards. Signed-off-by: Joshua Smith --- backend/plugins/cursor/e2e/extractors_test.go | 8 + .../raw_tables/_raw_cursor_daily_usage.csv | 2 + .../_tool_cursor_daily_usage.csv | 2 + backend/plugins/cursor/models/daily_usage.go | 54 +++ .../20260714_add_daily_usage.go | 78 +++ .../models/migrationscripts/register.go | 1 + backend/plugins/cursor/models/models.go | 1 + .../plugins/cursor/tasks/collector_utils.go | 45 ++ .../cursor/tasks/collector_utils_test.go | 46 ++ .../cursor/tasks/daily_usage_collector.go | 91 ++++ .../cursor/tasks/daily_usage_extractor.go | 111 +++++ backend/plugins/cursor/tasks/register.go | 2 + backend/plugins/cursor/tasks/subtasks.go | 17 + .../dashboards/mysql/ai-cost-efficiency.json | 241 +++++++++- grafana/dashboards/mysql/cursor-usage.json | 444 +++++++++++------- .../dashboards/mysql/multi-ai-comparison.json | 93 +++- 16 files changed, 1050 insertions(+), 186 deletions(-) create mode 100644 backend/plugins/cursor/e2e/raw_tables/_raw_cursor_daily_usage.csv create mode 100644 backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_daily_usage.csv create mode 100644 backend/plugins/cursor/models/daily_usage.go create mode 100644 backend/plugins/cursor/models/migrationscripts/20260714_add_daily_usage.go create mode 100644 backend/plugins/cursor/tasks/daily_usage_collector.go create mode 100644 backend/plugins/cursor/tasks/daily_usage_extractor.go diff --git a/backend/plugins/cursor/e2e/extractors_test.go b/backend/plugins/cursor/e2e/extractors_test.go index caebd291e6b..1dc234f164f 100644 --- a/backend/plugins/cursor/e2e/extractors_test.go +++ b/backend/plugins/cursor/e2e/extractors_test.go @@ -59,14 +59,17 @@ func TestCursorExtractorsDataFlow(t *testing.T) { dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_cursor_usage_events.csv", "_raw_cursor_usage_events") dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_cursor_members.csv", "_raw_cursor_members") dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_cursor_user_spend.csv", "_raw_cursor_user_spend") + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_cursor_daily_usage.csv", "_raw_cursor_daily_usage") dataflowTester.FlushTabler(&models.CursorUsageEvent{}) dataflowTester.FlushTabler(&models.CursorMember{}) dataflowTester.FlushTabler(&models.CursorUserSpend{}) + dataflowTester.FlushTabler(&models.CursorDailyUsage{}) dataflowTester.Subtask(tasks.ExtractMembersMeta, taskData) dataflowTester.Subtask(tasks.ExtractUsageEventsMeta, taskData) dataflowTester.Subtask(tasks.ExtractUserSpendMeta, taskData) + dataflowTester.Subtask(tasks.ExtractDailyUsageMeta, taskData) dataflowTester.VerifyTableWithOptions(&models.CursorUsageEvent{}, e2ehelper.TableOptions{ CSVRelPath: "./snapshot_tables/_tool_cursor_usage_events.csv", @@ -82,4 +85,9 @@ func TestCursorExtractorsDataFlow(t *testing.T) { CSVRelPath: "./snapshot_tables/_tool_cursor_user_spend.csv", IgnoreTypes: []interface{}{common.NoPKModel{}}, }) + + dataflowTester.VerifyTableWithOptions(&models.CursorDailyUsage{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_cursor_daily_usage.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) } diff --git a/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_daily_usage.csv b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_daily_usage.csv new file mode 100644 index 00000000000..193348deea3 --- /dev/null +++ b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_daily_usage.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""day"":""2026-07-08"",""userId"":""user_example123"",""email"":""user@example.com"",""isActive"":true,""completions"":15,""premiumRequests"":3,""agentRequests"":5,""chatRequests"":8,""composerRequests"":2,""totalTabsAccepted"":42,""totalTabsShown"":60,""usageBasedReqs"":7,""subscriptionIncludedReqs"":6,""mostUsedModel"":""composer-2.5-fast"",""clientVersion"":""0.50.3"",""linesAdded"":120,""linesDeleted"":30}",https://api.cursor.com/teams/daily-usage-data,null,2026-07-09 17:14:19.000 diff --git a/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_daily_usage.csv b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_daily_usage.csv new file mode 100644 index 00000000000..cb921061bad --- /dev/null +++ b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_daily_usage.csv @@ -0,0 +1,2 @@ +connection_id,scope_id,user_id,usage_date,email,is_active,completions,premium_requests,agent_requests,chat_requests,composer_requests,tabs_accepted,tabs_shown,usage_based_reqs,subscription_included_reqs,most_used_model,client_version,lines_added,lines_deleted +1,team,user_example123,2026-07-08T00:00:00.000+00:00,user@example.com,1,15,3,5,8,2,42,60,7,6,composer-2.5-fast,0.50.3,120,30 diff --git a/backend/plugins/cursor/models/daily_usage.go b/backend/plugins/cursor/models/daily_usage.go new file mode 100644 index 00000000000..dcb90ef48ef --- /dev/null +++ b/backend/plugins/cursor/models/daily_usage.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// CursorDailyUsage stores per-user per-day adoption metrics from POST /teams/daily-usage-data. +type CursorDailyUsage struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` + UsageDate time.Time `gorm:"primaryKey" json:"usageDate"` + + Email string `json:"email" gorm:"type:varchar(255);index"` + IsActive bool `json:"isActive"` + Completions int `json:"completions"` + PremiumRequests int `json:"premiumRequests"` + AgentRequests int `json:"agentRequests"` + ChatRequests int `json:"chatRequests"` + ComposerRequests int `json:"composerRequests"` + TabsAccepted int `json:"tabsAccepted"` + TabsShown int `json:"tabsShown"` + UsageBasedReqs int `json:"usageBasedReqs"` + SubscriptionIncludedReqs int `json:"subscriptionIncludedReqs"` + MostUsedModel string `json:"mostUsedModel" gorm:"type:varchar(255)"` + ClientVersion string `json:"clientVersion" gorm:"type:varchar(100)"` + LinesAdded int `json:"linesAdded"` + LinesDeleted int `json:"linesDeleted"` + + common.NoPKModel +} + +func (CursorDailyUsage) TableName() string { + return "_tool_cursor_daily_usage" +} diff --git a/backend/plugins/cursor/models/migrationscripts/20260714_add_daily_usage.go b/backend/plugins/cursor/models/migrationscripts/20260714_add_daily_usage.go new file mode 100644 index 00000000000..c41eccec65a --- /dev/null +++ b/backend/plugins/cursor/models/migrationscripts/20260714_add_daily_usage.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addCursorDailyUsage struct{} + +type cursorDailyUsage20260714 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + UserId string `gorm:"primaryKey;type:varchar(255)"` + UsageDate time.Time `gorm:"primaryKey"` + Email string `gorm:"type:varchar(255);index"` + IsActive bool + Completions int + PremiumRequests int + AgentRequests int + ChatRequests int + ComposerRequests int + TabsAccepted int + TabsShown int + UsageBasedReqs int + SubscriptionIncludedReqs int + MostUsedModel string `gorm:"type:varchar(255)"` + ClientVersion string `gorm:"type:varchar(100)"` + LinesAdded int + LinesDeleted int + archived.NoPKModel +} + +func (cursorDailyUsage20260714) TableName() string { return "_tool_cursor_daily_usage" } + +type cursorRawDailyUsage20260714 struct { + ID uint64 `gorm:"primaryKey"` + Params string `gorm:"type:varchar(255);index"` + Data []byte + Url string + Input json.RawMessage `gorm:"type:json"` + CreatedAt time.Time `gorm:"index"` +} + +func (cursorRawDailyUsage20260714) TableName() string { return "_raw_cursor_daily_usage" } + +func (script *addCursorDailyUsage) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &cursorDailyUsage20260714{}, + &cursorRawDailyUsage20260714{}, + ) +} + +func (*addCursorDailyUsage) Version() uint64 { return 20260714000000 } + +func (*addCursorDailyUsage) Name() string { return "cursor add daily usage tables" } diff --git a/backend/plugins/cursor/models/migrationscripts/register.go b/backend/plugins/cursor/models/migrationscripts/register.go index f28544150e0..1c5e9d2b50a 100644 --- a/backend/plugins/cursor/models/migrationscripts/register.go +++ b/backend/plugins/cursor/models/migrationscripts/register.go @@ -24,5 +24,6 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addCursorInitialTables), new(changeCursorRequestsCostsToFloat), + new(addCursorDailyUsage), } } diff --git a/backend/plugins/cursor/models/models.go b/backend/plugins/cursor/models/models.go index 5c37c05b047..6a2ec5d8831 100644 --- a/backend/plugins/cursor/models/models.go +++ b/backend/plugins/cursor/models/models.go @@ -28,5 +28,6 @@ func GetTablesInfo() []dal.Tabler { &CursorUsageEvent{}, &CursorUserSpend{}, &CursorMember{}, + &CursorDailyUsage{}, } } diff --git a/backend/plugins/cursor/tasks/collector_utils.go b/backend/plugins/cursor/tasks/collector_utils.go index 5083f796e83..6be6c1ba583 100644 --- a/backend/plugins/cursor/tasks/collector_utils.go +++ b/backend/plugins/cursor/tasks/collector_utils.go @@ -36,9 +36,12 @@ const ( rawUsageEventsTable = "cursor_usage_events" rawUserSpendTable = "cursor_user_spend" rawMembersTable = "cursor_members" + rawDailyUsageTable = "cursor_daily_usage" cursorApiPageSize = 100 cursorInitialBackfillDays = 90 + // cursorDailyUsageMaxDays is the maximum span allowed by POST /teams/daily-usage-data. + cursorDailyUsageMaxDays = 30 ) type cursorRawParams struct { @@ -92,6 +95,34 @@ func computeUsageTimeRangeMs(since *time.Time, now time.Time) (int64, int64) { return start.UnixMilli(), end.UnixMilli() } +// splitDailyUsageTimeRangeMs splits [startMs, endMs] into chunks of at most maxDays for +// POST /teams/daily-usage-data (API limit: date range cannot exceed 30 days). +func splitDailyUsageTimeRangeMs(startMs, endMs int64, maxDays int) []cursorTimeRangeInput { + if startMs >= endMs || maxDays <= 0 { + return nil + } + maxDuration := time.Duration(maxDays) * 24 * time.Hour + chunkStart := time.UnixMilli(startMs).UTC() + end := time.UnixMilli(endMs).UTC() + + var chunks []cursorTimeRangeInput + for chunkStart.Before(end) { + chunkEnd := chunkStart.Add(maxDuration) + if chunkEnd.After(end) { + chunkEnd = end + } + chunks = append(chunks, cursorTimeRangeInput{ + StartDateMs: chunkStart.UnixMilli(), + EndDateMs: chunkEnd.UnixMilli(), + }) + if !chunkEnd.Before(end) { + break + } + chunkStart = chunkEnd.Add(time.Millisecond) + } + return chunks +} + func parseUsageEventsResponse(res *http.Response) ([]json.RawMessage, errors.Error) { body, err := readResponseBody(res) if err != nil { @@ -120,6 +151,20 @@ func parseSpendMembersResponse(res *http.Response) ([]json.RawMessage, errors.Er return response.TeamMemberSpend, nil } +func parseDailyUsageResponse(res *http.Response) ([]json.RawMessage, errors.Error) { + body, err := readResponseBody(res) + if err != nil { + return nil, err + } + var response struct { + Data []json.RawMessage `json:"data"` + } + if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { + return nil, errors.Default.Wrap(errors.Convert(jsonErr), "failed to decode daily usage response") + } + return response.Data, nil +} + func parseMembersResponse(res *http.Response) ([]json.RawMessage, errors.Error) { body, err := readResponseBody(res) if err != nil { diff --git a/backend/plugins/cursor/tasks/collector_utils_test.go b/backend/plugins/cursor/tasks/collector_utils_test.go index 4b82fcbec7f..c2af757c42f 100644 --- a/backend/plugins/cursor/tasks/collector_utils_test.go +++ b/backend/plugins/cursor/tasks/collector_utils_test.go @@ -20,6 +20,7 @@ package tasks import ( "encoding/json" "testing" + "time" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/cursor/models" @@ -49,3 +50,48 @@ func TestRawParamsFromTaskDataIncludesEndpoint(t *testing.T) { t.Fatalf("raw params mismatch:\n got: %s\nwant: %s", encoded, expected) } } + +func TestSplitDailyUsageTimeRangeMsChunksLongRanges(t *testing.T) { + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2026, 4, 1, 12, 0, 0, 0, time.UTC) + chunks := splitDailyUsageTimeRangeMs(start.UnixMilli(), end.UnixMilli(), cursorDailyUsageMaxDays) + + if len(chunks) != 4 { + t.Fatalf("expected 4 chunks for ~90-day span, got %d", len(chunks)) + } + if chunks[0].StartDateMs != start.UnixMilli() { + t.Fatalf("first chunk start mismatch: got %d want %d", chunks[0].StartDateMs, start.UnixMilli()) + } + if chunks[len(chunks)-1].EndDateMs != end.UnixMilli() { + t.Fatalf("last chunk end mismatch: got %d want %d", chunks[len(chunks)-1].EndDateMs, end.UnixMilli()) + } + for i := 1; i < len(chunks); i++ { + if chunks[i].StartDateMs <= chunks[i-1].StartDateMs { + t.Fatalf("chunk %d does not advance start time", i) + } + maxSpan := time.Duration(cursorDailyUsageMaxDays) * 24 * time.Hour + span := time.UnixMilli(chunks[i].StartDateMs).Sub(time.UnixMilli(chunks[i-1].StartDateMs)) + if span > maxSpan+time.Millisecond { + t.Fatalf("gap between chunk %d and %d exceeds %d days", i-1, i, cursorDailyUsageMaxDays) + } + } +} + +func TestSplitDailyUsageTimeRangeMsEmptyRange(t *testing.T) { + if chunks := splitDailyUsageTimeRangeMs(100, 100, cursorDailyUsageMaxDays); len(chunks) != 0 { + t.Fatalf("expected no chunks for empty range, got %d", len(chunks)) + } +} + +func TestDailyUsagePostBodyUsesEpochMilliseconds(t *testing.T) { + body := dailyUsagePostBody(&helper.RequestData{ + Input: cursorTimeRangeInput{StartDateMs: 1710720000000, EndDateMs: 1710892800000}, + Pager: &helper.Pager{Page: 1, Size: 100}, + }) + if body["startDate"] != int64(1710720000000) { + t.Fatalf("startDate should be epoch ms int64, got %#v", body["startDate"]) + } + if body["endDate"] != int64(1710892800000) { + t.Fatalf("endDate should be epoch ms int64, got %#v", body["endDate"]) + } +} diff --git a/backend/plugins/cursor/tasks/daily_usage_collector.go b/backend/plugins/cursor/tasks/daily_usage_collector.go new file mode 100644 index 00000000000..61a73b68314 --- /dev/null +++ b/backend/plugins/cursor/tasks/daily_usage_collector.go @@ -0,0 +1,91 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +func newDailyUsageDateRangeIterator(since *time.Time) *helper.QueueIterator { + startMs, endMs := computeUsageTimeRangeMs(since, time.Now().UTC()) + iter := helper.NewQueueIterator() + for _, chunk := range splitDailyUsageTimeRangeMs(startMs, endMs, cursorDailyUsageMaxDays) { + iter.Push(chunk) + } + return iter +} + +func dailyUsagePostBody(reqData *helper.RequestData) map[string]interface{} { + input := reqData.Input.(cursorTimeRangeInput) + page := 1 + if state, ok := reqData.CustomData.(cursorPageState); ok && state.Page > 0 { + page = state.Page + } else if reqData.Pager.Page > 0 { + page = reqData.Pager.Page + } + return map[string]interface{}{ + "startDate": input.StartDateMs, + "endDate": input.EndDateMs, + "page": page, + "pageSize": reqData.Pager.Size, + } +} + +// CollectDailyUsage collects per-user per-day adoption metrics from POST /teams/daily-usage-data. +func CollectDailyUsage(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*CursorTaskData) + if !ok { + return errors.Default.New("task data is not CursorTaskData") + } + apiClient, err := CreateApiClient(taskCtx.TaskContext(), data.Connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawDailyUsageTable, + Options: rawParamsFromTaskData(data), + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: newDailyUsageDateRangeIterator(collector.GetSince()), + Method: http.MethodPost, + UrlTemplate: "teams/daily-usage-data", + PageSize: cursorApiPageSize, + RequestBody: dailyUsagePostBody, + GetNextPageCustomData: nextPostPage, + ResponseParser: parseDailyUsageResponse, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/cursor/tasks/daily_usage_extractor.go b/backend/plugins/cursor/tasks/daily_usage_extractor.go new file mode 100644 index 00000000000..80b80538e14 --- /dev/null +++ b/backend/plugins/cursor/tasks/daily_usage_extractor.go @@ -0,0 +1,111 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/cursor/models" +) + +type dailyUsageRecord struct { + Day string `json:"day"` + UserId string `json:"userId"` + Email string `json:"email"` + IsActive bool `json:"isActive"` + Completions int `json:"completions"` + PremiumRequests int `json:"premiumRequests"` + AgentRequests int `json:"agentRequests"` + ChatRequests int `json:"chatRequests"` + ComposerRequests int `json:"composerRequests"` + TotalTabsAccepted int `json:"totalTabsAccepted"` + TotalTabsShown int `json:"totalTabsShown"` + UsageBasedReqs int `json:"usageBasedReqs"` + SubscriptionIncludedReqs int `json:"subscriptionIncludedReqs"` + MostUsedModel string `json:"mostUsedModel"` + ClientVersion string `json:"clientVersion"` + LinesAdded int `json:"linesAdded"` + LinesDeleted int `json:"linesDeleted"` +} + +// ExtractDailyUsage parses raw daily usage records into tool-layer tables. +func ExtractDailyUsage(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*CursorTaskData) + if !ok { + return errors.Default.New("task data is not CursorTaskData") + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawDailyUsageTable, + Options: rawParamsFromTaskData(data), + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var record dailyUsageRecord + if err := errors.Convert(json.Unmarshal(row.Data, &record)); err != nil { + return nil, err + } + + userId := strings.TrimSpace(record.UserId) + if userId == "" { + userId = strings.TrimSpace(record.Email) + } + if userId == "" { + return nil, nil + } + + usageDate, parseErr := time.Parse("2006-01-02", strings.TrimSpace(record.Day)) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid day format in daily usage record") + } + + usage := &models.CursorDailyUsage{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + UserId: userId, + UsageDate: usageDate, + Email: strings.TrimSpace(record.Email), + IsActive: record.IsActive, + Completions: record.Completions, + PremiumRequests: record.PremiumRequests, + AgentRequests: record.AgentRequests, + ChatRequests: record.ChatRequests, + ComposerRequests: record.ComposerRequests, + TabsAccepted: record.TotalTabsAccepted, + TabsShown: record.TotalTabsShown, + UsageBasedReqs: record.UsageBasedReqs, + SubscriptionIncludedReqs: record.SubscriptionIncludedReqs, + MostUsedModel: strings.TrimSpace(record.MostUsedModel), + ClientVersion: strings.TrimSpace(record.ClientVersion), + LinesAdded: record.LinesAdded, + LinesDeleted: record.LinesDeleted, + } + return []interface{}{usage}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/cursor/tasks/register.go b/backend/plugins/cursor/tasks/register.go index 0a3251c2106..e8a3f4ec3cb 100644 --- a/backend/plugins/cursor/tasks/register.go +++ b/backend/plugins/cursor/tasks/register.go @@ -28,5 +28,7 @@ func GetSubTaskMetas() []plugin.SubTaskMeta { ExtractUsageEventsMeta, CollectUserSpendMeta, ExtractUserSpendMeta, + CollectDailyUsageMeta, + ExtractDailyUsageMeta, } } diff --git a/backend/plugins/cursor/tasks/subtasks.go b/backend/plugins/cursor/tasks/subtasks.go index 98e0e675814..8d9462f5232 100644 --- a/backend/plugins/cursor/tasks/subtasks.go +++ b/backend/plugins/cursor/tasks/subtasks.go @@ -69,3 +69,20 @@ var ExtractMembersMeta = plugin.SubTaskMeta{ Description: "Extract team member roster into tool-layer tables", Dependencies: []*plugin.SubTaskMeta{&CollectMembersMeta}, } + +var CollectDailyUsageMeta = plugin.SubTaskMeta{ + Name: "collectDailyUsage", + EntryPoint: CollectDailyUsage, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect per-user per-day adoption metrics from the Cursor Admin API", +} + +var ExtractDailyUsageMeta = plugin.SubTaskMeta{ + Name: "extractDailyUsage", + EntryPoint: ExtractDailyUsage, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract per-user per-day adoption metrics into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectDailyUsageMeta}, +} diff --git a/grafana/dashboards/mysql/ai-cost-efficiency.json b/grafana/dashboards/mysql/ai-cost-efficiency.json index 68cfe802779..0f672003476 100644 --- a/grafana/dashboards/mysql/ai-cost-efficiency.json +++ b/grafana/dashboards/mysql/ai-cost-efficiency.json @@ -27,6 +27,17 @@ "title": "Kiro Usage Dashboard", "type": "link", "url": "/d/kiro_user_report" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Cursor Usage & Cost", + "type": "link", + "url": "/d/cursor_usage" } ], "panels": [ @@ -559,6 +570,206 @@ ], "title": "Weekly AI Activity Volume", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, + "id": 30, + "panels": [], + "title": "Cursor Cost Efficiency", + "type": "row" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "decimals": 2, + "unit": "currencyUSD", + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 25 }, + "id": 31, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(charged_cents) / 100 AS 'Total Spend'\nFROM _tool_cursor_usage_events\nWHERE connection_id = ${cursor_connection_id}\n AND scope_id = '${cursor_scope_id}'\n AND $__timeFilter(event_time)", + "refId": "A" + } + ], + "title": "Cursor Total Spend", + "type": "stat" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "decimals": 2, + "unit": "currencyUSD", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 25 }, + "id": 32, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(e.charged_cents) / 100 / NULLIF(COUNT(DISTINCT pr.id), 0), 2) AS '$ / PR'\nFROM _tool_cursor_usage_events e\nCROSS JOIN (\n SELECT DISTINCT id FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n) pr\nWHERE e.connection_id = ${cursor_connection_id}\n AND e.scope_id = '${cursor_scope_id}'\n AND $__timeFilter(e.event_time)", + "refId": "A" + } + ], + "title": "Cursor $ per PR", + "type": "stat" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "decimals": 2, + "unit": "currencyUSD", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 25 }, + "id": 33, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(e.charged_cents) / 100 / NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 2) AS '$ / Deploy'\nFROM _tool_cursor_usage_events e\nCROSS JOIN (\n SELECT DISTINCT cicd_deployment_id\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n) cdc\nWHERE e.connection_id = ${cursor_connection_id}\n AND e.scope_id = '${cursor_scope_id}'\n AND $__timeFilter(e.event_time)", + "refId": "A" + } + ], + "title": "Cursor $ per Deployment", + "type": "stat" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "decimals": 2, + "unit": "currencyUSD", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 25 }, + "id": 34, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(e.charged_cents) / 100 / NULLIF(COUNT(DISTINCT i.id), 0), 2) AS '$ / Issue'\nFROM _tool_cursor_usage_events e\nCROSS JOIN (\n SELECT DISTINCT id FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n) i\nWHERE e.connection_id = ${cursor_connection_id}\n AND e.scope_id = '${cursor_scope_id}'\n AND $__timeFilter(e.event_time)", + "refId": "A" + } + ], + "title": "Cursor $ per Issue Resolved", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Weekly Cursor spend per merged PR", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 }, + "id": 35, + "options": { + "legend": { "calcs": ["mean", "min"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _spend AS (\n SELECT DATE_SUB(DATE(event_time), INTERVAL WEEKDAY(DATE(event_time)) DAY) AS week_start,\n SUM(charged_cents) / 100 AS spend_usd\n FROM _tool_cursor_usage_events\n WHERE connection_id = ${cursor_connection_id}\n AND scope_id = '${cursor_scope_id}'\n AND $__timeFilter(event_time)\n GROUP BY DATE_SUB(DATE(event_time), INTERVAL WEEKDAY(DATE(event_time)) DAY)\n),\n_prs AS (\n SELECT DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY) AS week_start,\n COUNT(*) AS prs\n FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n GROUP BY DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY)\n)\nSELECT s.week_start AS time,\n ROUND(s.spend_usd / NULLIF(p.prs, 0), 2) AS '$ per PR'\nFROM _spend s\nLEFT JOIN _prs p ON s.week_start = p.week_start\nORDER BY time", + "refId": "A" + } + ], + "title": "Cursor $ per Merged PR (Weekly)", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Weekly Cursor spend per production deployment", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 }, + "id": 36, + "options": { + "legend": { "calcs": ["mean", "min"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _spend AS (\n SELECT DATE_SUB(DATE(event_time), INTERVAL WEEKDAY(DATE(event_time)) DAY) AS week_start,\n SUM(charged_cents) / 100 AS spend_usd\n FROM _tool_cursor_usage_events\n WHERE connection_id = ${cursor_connection_id}\n AND scope_id = '${cursor_scope_id}'\n AND $__timeFilter(event_time)\n GROUP BY DATE_SUB(DATE(event_time), INTERVAL WEEKDAY(DATE(event_time)) DAY)\n),\n_deploys AS (\n SELECT DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(DISTINCT cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n)\nSELECT s.week_start AS time,\n ROUND(s.spend_usd / NULLIF(d.deploys, 0), 2) AS '$ per Deploy'\nFROM _spend s\nLEFT JOIN _deploys d ON s.week_start = d.week_start\nORDER BY time", + "refId": "A" + } + ], + "title": "Cursor $ per Deployment (Weekly)", + "type": "timeseries" } ], "preload": false, @@ -566,11 +777,37 @@ "schemaVersion": 41, "tags": [ "kiro", + "cursor", "cost", "efficiency" ], "templating": { - "list": [] + "list": [ + { + "current": {}, + "datasource": "mysql", + "definition": "SELECT DISTINCT connection_id FROM _tool_cursor_scopes ORDER BY connection_id DESC", + "hide": 0, + "label": "Cursor Connection", + "name": "cursor_connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_cursor_scopes ORDER BY connection_id DESC", + "refresh": 1, + "type": "query" + }, + { + "current": {}, + "datasource": "mysql", + "definition": "SELECT DISTINCT id FROM _tool_cursor_scopes WHERE connection_id = CAST('${cursor_connection_id}' AS UNSIGNED)", + "hide": 0, + "label": "Cursor Scope", + "name": "cursor_scope_id", + "options": [], + "query": "SELECT DISTINCT id FROM _tool_cursor_scopes WHERE connection_id = CAST('${cursor_connection_id}' AS UNSIGNED)", + "refresh": 1, + "type": "query" + } + ] }, "time": { "from": "now-90d", @@ -580,5 +817,5 @@ "timezone": "utc", "title": "AI Cost-Efficiency", "uid": "ai_cost_efficiency", - "version": 1 + "version": 2 } diff --git a/grafana/dashboards/mysql/cursor-usage.json b/grafana/dashboards/mysql/cursor-usage.json index 938dd740bd0..fd550a10843 100644 --- a/grafana/dashboards/mysql/cursor-usage.json +++ b/grafana/dashboards/mysql/cursor-usage.json @@ -14,7 +14,7 @@ }, "editable": true, "fiscalYearStartMonth": 0, - "graphTooltip": 0, + "graphTooltip": 1, "links": [], "liveNow": false, "panels": [ @@ -27,19 +27,19 @@ }, { "datasource": "mysql", - "description": "Sum of chargedCents for billable usage events in the selected time range", + "description": "Total charged amount for billable usage events in the selected time range", "fieldConfig": { "defaults": { "decimals": 2, "unit": "currencyUSD", - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, "id": 1, "options": { - "colorMode": "value", + "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -57,22 +57,24 @@ "refId": "A" } ], - "title": "Charged Spend (Period)", + "title": "Charged Amount (Selected Period)", "type": "stat" }, { "datasource": "mysql", - "description": "Distinct users with usage events in the selected time range", + "description": "Total usage events in selected period from filtered-usage-events API", "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } + "unit": "locale", + "decimals": 0, + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "id": 2, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, + "id": 3, "options": { - "colorMode": "value", + "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -86,25 +88,26 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(DISTINCT user_email) AS \"Active Users\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "rawSql": "SELECT COUNT(*) AS \"Usage Events\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", "refId": "A" } ], - "title": "Active Users (Period)", + "title": "Usage Events (Selected Period)", "type": "stat" }, { "datasource": "mysql", + "description": "Active roster count from /teams/members where is_removed = 0", "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "purple", "value": null }] } + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "id": 3, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, + "id": 4, "options": { - "colorMode": "value", + "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -118,25 +121,26 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS \"Usage Events\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "rawSql": "SELECT COUNT(*) AS \"Team Members\"\nFROM _tool_cursor_members\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_removed = 0", "refId": "A" } ], - "title": "Usage Events (Period)", + "title": "Team Members", "type": "stat" }, { "datasource": "mysql", + "description": "Team members with zero active days in the trailing 7-day window (relative to latest collected usage_date)", "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "id": 4, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, + "id": 303, "options": { - "colorMode": "value", + "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -150,11 +154,11 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS \"Team Members\"\nFROM _tool_cursor_members\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_removed = 0", + "rawSql": "SELECT COUNT(*) AS \"Inactive Users\"\nFROM (\n SELECT user_id\n FROM _tool_cursor_daily_usage\n WHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND usage_date >= DATE_SUB(\n (SELECT MAX(usage_date) FROM _tool_cursor_daily_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'),\n INTERVAL 6 DAY\n )\n GROUP BY user_id\n HAVING MAX(is_active) = 0\n) t", "refId": "A" } ], - "title": "Team Members", + "title": "Inactive Users (Last 7 Days)", "type": "stat" }, { @@ -164,14 +168,84 @@ "defaults": { "decimals": 2, "unit": "currencyUSD", - "thresholds": { "mode": "absolute", "steps": [{ "color": "orange", "value": null }] } + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 5 }, "id": 5, "options": { - "colorMode": "value", + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(SUM(spend_cents), 0) / 100 AS \"On-Demand Spend\"\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "On-Demand Spend (Current Cycle)", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Current billing cycle included (non-on-demand) spend from /teams/spend", + "fieldConfig": { + "defaults": { + "decimals": 2, + "unit": "currencyUSD", + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 5 }, + "id": 301, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(SUM(included_spend_cents), 0) / 100 AS \"Included Spend\"\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Included Spend (Current Cycle)", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Difference between /teams/spend total and sum of chargedCents for events since billing_cycle_start. Positive means spend exceeds collected events (backfill may be incomplete).", + "fieldConfig": { + "defaults": { + "decimals": 2, + "unit": "currencyUSD", + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 5 }, + "id": 302, + "options": { + "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", @@ -185,15 +259,16 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT SUM(spend_cents) / 100 AS \"Charged Amount\"\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "rawSql": "SELECT (\n COALESCE((SELECT SUM(spend_cents) FROM _tool_cursor_user_spend\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'), 0)\n - COALESCE((SELECT SUM(charged_cents) FROM _tool_cursor_usage_events\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n AND event_time >= (SELECT MIN(billing_cycle_start) FROM _tool_cursor_user_spend\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}')), 0)\n) / 100 AS \"Reconciliation Delta\"", "refId": "A" } ], - "title": "Billing Cycle Spend", + "title": "Reconciliation Delta", "type": "stat" }, { "datasource": "mysql", + "description": "Days since most recent usage event; green < 2d, yellow 2-6d, red >= 7d", "fieldConfig": { "defaults": { "thresholds": { @@ -207,7 +282,7 @@ }, "overrides": [] }, - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 5 }, "id": 6, "options": { "colorMode": "value", @@ -233,7 +308,7 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, "id": 101, "title": "Cost Trends", "type": "row" @@ -243,7 +318,7 @@ "description": "Daily charged spend from usage events", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { "fixedColor": "blue", "mode": "fixed" }, "custom": { "axisLabel": "USD", "drawStyle": "bars", @@ -252,11 +327,12 @@ "showPoints": "never", "stacking": { "group": "A", "mode": "none" } }, - "unit": "currencyUSD" + "unit": "currencyUSD", + "decimals": 2 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, "id": 10, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, @@ -287,14 +363,20 @@ "stacking": { "group": "A", "mode": "normal" } } }, - "overrides": [] + "overrides": [ + { "matcher": { "id": "byName", "options": "Usage-based" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Included in Business" }, "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Free" }, "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Errored, Not Charged" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Aborted, Not Charged" }, "properties": [{ "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "User API Key" }, "properties": [{ "id": "color", "value": { "fixedColor": "text", "mode": "fixed" } }] } + ] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, "id": 11, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" }, - "stacking": { "mode": "normal", "group": "A" } + "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "11.0.0", "targets": [ @@ -315,85 +397,58 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, "id": 102, "title": "Breakdowns", "type": "row" }, { "datasource": "mysql", + "description": "Average cost per event, total charged, event count, and share of total events by model for the selected period. Table defaults to charged spend descending. Events and Usage % columns sort numerically; Charged and Avg Cost use formatted USD strings.", "fieldConfig": { "defaults": { "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false } }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "Charged" }, - "properties": [ - { "id": "custom.align", "value": "right" }, - { "id": "custom.width", "value": 90 } - ] - } - ] - }, - "gridPos": { "h": 8, "w": 6, "x": 0, "y": 15 }, - "id": 20, - "options": { - "cellHeight": "sm", - "footer": { "show": false }, - "showHeader": true - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": "mysql", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT COALESCE(MAX(NULLIF(m.name, '')), e.user_email) AS \"User\",\n CONCAT('$', FORMAT(SUM(e.charged_cents) / 100, 2)) AS \"Charged\"\nFROM _tool_cursor_usage_events e\nLEFT JOIN _tool_cursor_members m\n ON e.user_email = m.email\n AND e.connection_id = m.connection_id\n AND e.scope_id = m.scope_id\n AND m.is_removed = 0\nWHERE $__timeFilter(e.event_time)\n AND e.connection_id = ${connection_id}\n AND e.scope_id = '${scope_id}'\nGROUP BY e.user_email\nORDER BY SUM(e.charged_cents) DESC\nLIMIT 25", - "refId": "A" - } - ], - "title": "Cost by User", - "type": "table" - }, - { - "datasource": "mysql", - "fieldConfig": { - "defaults": { - "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, - "decimals": 0 - }, "overrides": [ { "matcher": { "id": "byName", "options": "Model" }, "properties": [{ "id": "custom.width", "value": 320 }] }, { - "matcher": { "id": "byName", "options": "Model Cost" }, + "matcher": { "id": "byName", "options": "Avg Cost" }, "properties": [ - { "id": "unit", "value": "currencyUSD" }, - { "id": "decimals", "value": 4 }, - { "id": "custom.width", "value": 110 } + { "id": "custom.width", "value": 120 }, + { "id": "custom.align", "value": "right" } ] }, { "matcher": { "id": "byName", "options": "Charged" }, "properties": [ - { "id": "unit", "value": "currencyUSD" }, - { "id": "decimals", "value": 2 }, - { "id": "custom.width", "value": 90 } + { "id": "custom.width", "value": 140 }, + { "id": "custom.align", "value": "right" } ] }, { "matcher": { "id": "byName", "options": "Events" }, "properties": [ - { "id": "unit", "value": "short" }, - { "id": "custom.width", "value": 90 } + { "id": "unit", "value": "locale" }, + { "id": "decimals", "value": 0 }, + { "id": "custom.width", "value": 100 }, + { "id": "custom.align", "value": "right" } + ] + }, + { + "matcher": { "id": "byName", "options": "Usage %" }, + "properties": [ + { "id": "unit", "value": "percent" }, + { "id": "decimals", "value": 1 }, + { "id": "custom.width", "value": 80 }, + { "id": "custom.align", "value": "right" } ] } ] }, - "gridPos": { "h": 8, "w": 10, "x": 6, "y": 15 }, + "gridPos": { "h": 8, "w": 14, "x": 0, "y": 19 }, "id": 21, "options": { "cellHeight": "sm", @@ -406,7 +461,7 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT model AS \"Model\",\n SUM(total_cents) / COUNT(*) / 100 AS \"Model Cost\",\n SUM(charged_cents) / 100 AS \"Charged\",\n COUNT(*) AS \"Events\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY model\nORDER BY SUM(charged_cents) DESC", + "rawSql": "SELECT model AS \"Model\",\n CONCAT('$', FORMAT(SUM(total_cents) / COUNT(*) / 100, 4)) AS \"Avg Cost\",\n CONCAT('$', FORMAT(SUM(charged_cents) / 100, 2)) AS \"Charged\",\n COUNT(*) AS \"Events\",\n COUNT(*) * 100.0 / SUM(COUNT(*)) OVER () AS \"Usage %\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY model\nORDER BY SUM(charged_cents) DESC", "refId": "A" } ], @@ -415,35 +470,31 @@ }, { "datasource": "mysql", + "description": "Total charged spend by billing kind for the selected period", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, - "mappings": [], - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + "unit": "currencyUSD", + "decimals": 2, + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, + "color": { "mode": "fixed", "fixedColor": "blue" } }, "overrides": [ - { - "matcher": { "id": "byName", "options": "Charged" }, - "properties": [ - { "id": "unit", "value": "currencyUSD" }, - { "id": "decimals", "value": 2 } - ] - } + { "matcher": { "id": "byName", "options": "Usage-based" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Included in Business" }, "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Free" }, "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Errored, Not Charged" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Aborted, Not Charged" }, "properties": [{ "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "User API Key" }, "properties": [{ "id": "color", "value": { "fixedColor": "text", "mode": "fixed" } }] } ] }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 15 }, + "gridPos": { "h": 8, "w": 10, "x": 14, "y": 19 }, "id": 22, "options": { - "displayLabels": ["name", "percent"], - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": ["value"] - }, - "pieType": "donut", - "reduceOptions": { "calcs": ["sum"], "fields": "", "values": true }, - "tooltip": { "mode": "single", "sort": "none" } + "orientation": "horizontal", + "displayMode": "basic", + "showUnfilled": false, + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": true }, + "text": {} }, "pluginVersion": "11.0.0", "targets": [ @@ -456,42 +507,47 @@ } ], "title": "Cost by Kind", - "type": "piechart" + "type": "bargauge", + "transformations": [ + { + "id": "rowsToFields", + "options": { + "mappings": [ + { "fieldName": "Kind", "handlerKey": "field.name" }, + { "fieldName": "Charged", "handlerKey": "field.value" } + ] + } + } + ] }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, - "id": 103, - "title": "Billing Cycle & Reconciliation", + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "id": 200, + "title": "Adoption (Daily Usage)", "type": "row" }, { "datasource": "mysql", - "description": "Per-user billing cycle snapshot from POST /teams/spend", + "description": "Daily active users based on isActive flag from /teams/daily-usage-data", "fieldConfig": { "defaults": { - "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false } - }, - "overrides": [ - { - "matcher": { "id": "byRegexp", "options": "On-Demand|Included" }, - "properties": [ - { "id": "unit", "value": "currencyUSD" }, - { "id": "decimals", "value": 2 } - ] - }, - { - "matcher": { "id": "byName", "options": "Premium Reqs" }, - "properties": [{ "id": "unit", "value": "short" }] + "color": { "fixedColor": "blue", "mode": "fixed" }, + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 2, + "showPoints": "auto", + "pointSize": 4 } - ] + }, + "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, - "id": 30, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 }, + "id": 201, "options": { - "cellHeight": "sm", - "footer": { "show": false }, - "showHeader": true + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.0.0", "targets": [ @@ -499,40 +555,36 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT email AS \"User\",\n name AS \"Name\",\n role AS \"Role\",\n spend_cents / 100 AS \"On-Demand\",\n included_spend_cents / 100 AS \"Included\",\n fast_premium_requests AS \"Premium Reqs\"\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY spend_cents DESC\nLIMIT 50", + "rawSql": "SELECT usage_date AS time,\n COUNT(DISTINCT user_id) AS \"Active Users\"\nFROM _tool_cursor_daily_usage\nWHERE $__timeFilter(usage_date)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_active = 1\nGROUP BY usage_date\nORDER BY 1", "refId": "A" } ], - "title": "Billing Cycle Spend by User", - "type": "table" + "title": "Daily Active Users (DAU)", + "type": "timeseries" }, { "datasource": "mysql", - "description": "Compare billing cycle spend vs sum(chargedCents) since cycle start. Requires full event backfill for accurate reconciliation.", + "description": "Tab completion acceptance rate: accepted / shown", "fieldConfig": { "defaults": { - "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false } - }, - "overrides": [ - { - "matcher": { "id": "byRegexp", "options": "Cycle Spend|Event Charged" }, - "properties": [ - { "id": "unit", "value": "currencyUSD" }, - { "id": "decimals", "value": 2 } - ] + "color": { "fixedColor": "blue", "mode": "fixed" }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "showPoints": "auto", + "pointSize": 4 }, - { - "matcher": { "id": "byName", "options": "Events Since Cycle Start" }, - "properties": [{ "id": "unit", "value": "short" }] - } - ] + "unit": "percent", + "decimals": 1 + }, + "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, - "id": 31, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 }, + "id": 202, "options": { - "cellHeight": "sm", - "footer": { "show": false }, - "showHeader": true + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.0.0", "targets": [ @@ -540,29 +592,36 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT s.email AS \"User\",\n s.spend_cents / 100 AS \"Cycle Spend\",\n COALESCE(SUM(e.charged_cents), 0) / 100 AS \"Event Charged\",\n COUNT(e.event_id) AS \"Events Since Cycle Start\"\nFROM _tool_cursor_user_spend s\nLEFT JOIN _tool_cursor_usage_events e\n ON s.email = e.user_email\n AND e.event_time >= s.billing_cycle_start\n AND e.connection_id = s.connection_id\n AND e.scope_id = s.scope_id\nWHERE s.connection_id = ${connection_id}\n AND s.scope_id = '${scope_id}'\nGROUP BY s.email, s.spend_cents\nHAVING COUNT(e.event_id) > 0 OR s.spend_cents > 0\nORDER BY s.spend_cents DESC\nLIMIT 50", + "rawSql": "SELECT usage_date AS time,\n ROUND(SUM(tabs_accepted) * 100.0 / NULLIF(SUM(tabs_shown), 0), 1) AS \"Acceptance Rate\"\nFROM _tool_cursor_daily_usage\nWHERE $__timeFilter(usage_date)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_active = 1\nGROUP BY usage_date\nORDER BY 1", "refId": "A" } ], - "title": "Spend vs Events Reconciliation", - "type": "table" - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, - "id": 104, - "title": "Diagnostics", - "type": "row" + "title": "Tab Acceptance Rate", + "type": "timeseries" }, { "datasource": "mysql", - "fieldConfig": { "defaults": {}, "overrides": [] }, - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 33 }, - "id": 40, + "description": "Weekly request volume by feature type (Agent, Chat, Composer)", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "stacking": { "group": "A", "mode": "normal" } + } + }, + "overrides": [ + { "matcher": { "id": "byName", "options": "Agent" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Chat" }, "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] }, + { "matcher": { "id": "byName", "options": "Composer" }, "properties": [{ "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } }] } + ] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 }, + "id": 203, "options": { - "cellHeight": "sm", - "footer": { "show": false }, - "showHeader": true + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "11.0.0", "targets": [ @@ -570,12 +629,45 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT '_tool_cursor_usage_events' AS table_name, COUNT(*) AS row_count, MAX(event_time) AS latest\nFROM _tool_cursor_usage_events\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\nUNION ALL\nSELECT '_tool_cursor_user_spend', COUNT(*), MAX(collected_at)\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\nUNION ALL\nSELECT '_tool_cursor_members', COUNT(*), NULL\nFROM _tool_cursor_members\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'", + "rawSql": "SELECT DATE(DATE_SUB(usage_date, INTERVAL WEEKDAY(usage_date) DAY)) AS time,\n SUM(agent_requests) AS \"Agent\",\n SUM(chat_requests) AS \"Chat\",\n SUM(composer_requests) AS \"Composer\"\nFROM _tool_cursor_daily_usage\nWHERE $__timeFilter(usage_date)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY 1\nORDER BY 1", "refId": "A" } ], - "title": "Data Volume", - "type": "table" + "title": "Feature Adoption (Weekly)", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, + "id": 104, + "title": "Diagnostics", + "type": "row", + "panels": [ + { + "datasource": "mysql", + "description": "Row counts and latest timestamps per _tool_cursor_* table for pipeline health monitoring", + "fieldConfig": { "defaults": {}, "overrides": [] }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 45 }, + "id": 40, + "options": { + "cellHeight": "sm", + "footer": { "show": false }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT '_tool_cursor_usage_events' AS table_name, COUNT(*) AS row_count, MAX(event_time) AS latest\nFROM _tool_cursor_usage_events\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\nUNION ALL\nSELECT '_tool_cursor_user_spend', COUNT(*), MAX(collected_at)\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\nUNION ALL\nSELECT '_tool_cursor_members', COUNT(*), NULL\nFROM _tool_cursor_members\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\nUNION ALL\nSELECT '_tool_cursor_daily_usage', COUNT(*), MAX(usage_date)\nFROM _tool_cursor_daily_usage\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Data Volume", + "type": "table" + } + ] } ], "refresh": "", @@ -620,6 +712,6 @@ "timezone": "utc", "title": "Cursor Usage & Cost", "uid": "cursor_usage", - "version": 3, + "version": 4, "weekStart": "" } diff --git a/grafana/dashboards/mysql/multi-ai-comparison.json b/grafana/dashboards/mysql/multi-ai-comparison.json index 15084441420..a663eba9b49 100644 --- a/grafana/dashboards/mysql/multi-ai-comparison.json +++ b/grafana/dashboards/mysql/multi-ai-comparison.json @@ -38,6 +38,17 @@ "title": "Copilot Adoption", "type": "link", "url": "/d/copilot_adoption" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Cursor Usage", + "type": "link", + "url": "/d/cursor_usage" } ], "panels": [ @@ -59,7 +70,7 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "## Multi-AI Tool Comparison: Copilot vs Kiro\nCompare GitHub Copilot and Kiro adoption and activity side by side.\n\n**Semantic boundary:** Copilot exports code-generation and acceptance events. Kiro exports usage reports and completion responses, but no acceptance event. Returned Kiro lines and Copilot LOC added are displayed with explicit labels and must not be treated as equivalent productivity measures.", + "content": "## Multi-AI Tool Comparison: Copilot vs Kiro vs Cursor\nCompare GitHub Copilot, Kiro, and Cursor adoption and activity side by side.\n\n**Semantic boundary:** Copilot exports code-generation and acceptance events. Kiro exports usage reports and completion responses, but no acceptance event. Cursor metrics come from `_tool_cursor_daily_usage`. Returned Kiro lines, Copilot LOC added, and Cursor LOC added are displayed with explicit labels and must not be treated as equivalent productivity measures.", "mode": "markdown" }, "title": "About", @@ -152,6 +163,13 @@ "rawQuery": true, "rawSql": "SELECT\n DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time,\n MAX(daily_active_users) AS 'Copilot Active Users'\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)\nGROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)\nORDER BY time", "refId": "B" + }, + { + "datasource": "mysql", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(usage_date), INTERVAL WEEKDAY(DATE(usage_date)) DAY) AS time,\n COUNT(DISTINCT user_id) AS 'Cursor Active Users'\nFROM _tool_cursor_daily_usage\nWHERE connection_id = ${cursor_connection_id}\n AND scope_id = '${cursor_scope_id}'\n AND is_active = 1\n AND $__timeFilter(usage_date)\nGROUP BY DATE_SUB(DATE(usage_date), INTERVAL WEEKDAY(DATE(usage_date)) DAY)\nORDER BY time", + "refId": "C" } ], "title": "Weekly Active Users Comparison", @@ -308,13 +326,47 @@ "title": "Copilot: Code Suggestions & Acceptance", "type": "timeseries" }, + { + "datasource": "mysql", + "description": "Weekly Cursor tab completions: shown vs accepted", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, + "id": 13, + "options": { + "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": "mysql", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(usage_date), INTERVAL WEEKDAY(DATE(usage_date)) DAY) AS time,\n SUM(tabs_shown) AS 'Cursor: Suggestions',\n SUM(tabs_accepted) AS 'Cursor: Accepted'\nFROM _tool_cursor_daily_usage\nWHERE connection_id = ${cursor_connection_id}\n AND scope_id = '${cursor_scope_id}'\n AND $__timeFilter(usage_date)\nGROUP BY DATE_SUB(DATE(usage_date), INTERVAL WEEKDAY(DATE(usage_date)) DAY)\nORDER BY time", + "refId": "A" + } + ], + "title": "Cursor: Tab Completions", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 21 + "y": 29 }, "id": 20, "panels": [], @@ -358,7 +410,7 @@ "h": 8, "w": 12, "x": 0, - "y": 22 + "y": 30 }, "id": 21, "options": { @@ -383,7 +435,7 @@ }, "format": "time_series", "rawQuery": true, - "rawSql": "SELECT time, SUM(kiro_lines) AS 'Kiro Returned Lines', SUM(copilot_loc) AS 'Copilot LOC Added' FROM (SELECT DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) AS time, SUM(returned_line_count) AS kiro_lines, 0 AS copilot_loc FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) UNION ALL SELECT DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time, 0 AS kiro_lines, SUM(loc_added_sum) AS copilot_loc FROM _tool_copilot_enterprise_daily_metrics WHERE connection_id = ${copilot_connection_id} AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) GROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)) combined GROUP BY time ORDER BY time", + "rawSql": "SELECT time, SUM(kiro_lines) AS 'Kiro Returned Lines', SUM(copilot_loc) AS 'Copilot LOC Added', SUM(cursor_loc) AS 'Cursor LOC Added' FROM (SELECT DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) AS time, SUM(returned_line_count) AS kiro_lines, 0 AS copilot_loc, 0 AS cursor_loc FROM _tool_kiro_completion_log WHERE $__timeFilter(timestamp) GROUP BY DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) UNION ALL SELECT DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time, 0 AS kiro_lines, SUM(loc_added_sum) AS copilot_loc, 0 AS cursor_loc FROM _tool_copilot_enterprise_daily_metrics WHERE connection_id = ${copilot_connection_id} AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) GROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) UNION ALL SELECT DATE_SUB(DATE(usage_date), INTERVAL WEEKDAY(DATE(usage_date)) DAY) AS time, 0 AS kiro_lines, 0 AS copilot_loc, SUM(lines_added) AS cursor_loc FROM _tool_cursor_daily_usage WHERE connection_id = ${cursor_connection_id} AND scope_id = '${cursor_scope_id}' AND $__timeFilter(usage_date) GROUP BY DATE_SUB(DATE(usage_date), INTERVAL WEEKDAY(DATE(usage_date)) DAY)) combined GROUP BY time ORDER BY time", "refId": "A" } ], @@ -419,7 +471,7 @@ "h": 8, "w": 12, "x": 12, - "y": 22 + "y": 30 }, "id": 22, "options": { @@ -440,7 +492,7 @@ }, "format": "table", "rawQuery": true, - "rawSql": "SELECT 'Kiro (not exported)' AS 'Tool', CAST(NULL AS DECIMAL(10,2)) AS 'Acceptance Rate' UNION ALL SELECT 'Copilot' AS 'Tool', ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) AS 'Acceptance Rate' FROM _tool_copilot_enterprise_daily_metrics WHERE connection_id = ${copilot_connection_id} AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day)", + "rawSql": "SELECT 'Kiro (not exported)' AS 'Tool', CAST(NULL AS DECIMAL(10,2)) AS 'Acceptance Rate' UNION ALL SELECT 'Copilot' AS 'Tool', ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) AS 'Acceptance Rate' FROM _tool_copilot_enterprise_daily_metrics WHERE connection_id = ${copilot_connection_id} AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) UNION ALL SELECT 'Cursor' AS Tool, ROUND(SUM(tabs_accepted) * 100.0 / NULLIF(SUM(tabs_shown), 0), 1) AS 'Acceptance Rate' FROM _tool_cursor_daily_usage WHERE connection_id = ${cursor_connection_id} AND scope_id = '${cursor_scope_id}' AND $__timeFilter(usage_date)", "refId": "A" } ], @@ -454,6 +506,7 @@ "tags": [ "kiro", "copilot", + "cursor", "comparison" ], "templating": { @@ -487,6 +540,30 @@ "query": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${copilot_connection_id}' AS UNSIGNED)", "refresh": 1, "type": "query" + }, + { + "current": {}, + "datasource": "mysql", + "definition": "SELECT DISTINCT connection_id FROM _tool_cursor_scopes ORDER BY connection_id DESC", + "hide": 0, + "label": "Cursor Connection", + "name": "cursor_connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_cursor_scopes ORDER BY connection_id DESC", + "refresh": 1, + "type": "query" + }, + { + "current": {}, + "datasource": "mysql", + "definition": "SELECT DISTINCT id FROM _tool_cursor_scopes WHERE connection_id = CAST('${cursor_connection_id}' AS UNSIGNED)", + "hide": 0, + "label": "Cursor Scope", + "name": "cursor_scope_id", + "options": [], + "query": "SELECT DISTINCT id FROM _tool_cursor_scopes WHERE connection_id = CAST('${cursor_connection_id}' AS UNSIGNED)", + "refresh": 1, + "type": "query" } ] }, @@ -496,7 +573,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Multi-AI Tool Comparison", + "title": "Multi-AI Tool Comparison: Copilot vs Kiro vs Cursor", "uid": "multi_ai_comparison", - "version": 1 + "version": 2 } From ff811cf5cef11065a6f8fca90d875e7e03ff7372 Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Fri, 17 Jul 2026 11:27:13 -0600 Subject: [PATCH 06/13] feat(cursor): add daily usage line metrics * Store accepted, total line, and apply counts from daily usage API with migration. * Add Grafana panels and fix spend reconciliation to include included spend. Signed-off-by: Joshua Smith --- .../raw_tables/_raw_cursor_daily_usage.csv | 2 +- .../_tool_cursor_daily_usage.csv | 4 +- backend/plugins/cursor/models/daily_usage.go | 9 ++ .../20260716_add_daily_usage_line_fields.go | 71 ++++++++++++++++ .../models/migrationscripts/register.go | 1 + .../cursor/tasks/daily_usage_extractor.go | 20 ++++- grafana/dashboards/mysql/cursor-usage.json | 84 +++++++++++++++++-- 7 files changed, 179 insertions(+), 12 deletions(-) create mode 100644 backend/plugins/cursor/models/migrationscripts/20260716_add_daily_usage_line_fields.go diff --git a/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_daily_usage.csv b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_daily_usage.csv index 193348deea3..5150658ba1a 100644 --- a/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_daily_usage.csv +++ b/backend/plugins/cursor/e2e/raw_tables/_raw_cursor_daily_usage.csv @@ -1,2 +1,2 @@ id,params,data,url,input,created_at -1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""day"":""2026-07-08"",""userId"":""user_example123"",""email"":""user@example.com"",""isActive"":true,""completions"":15,""premiumRequests"":3,""agentRequests"":5,""chatRequests"":8,""composerRequests"":2,""totalTabsAccepted"":42,""totalTabsShown"":60,""usageBasedReqs"":7,""subscriptionIncludedReqs"":6,""mostUsedModel"":""composer-2.5-fast"",""clientVersion"":""0.50.3"",""linesAdded"":120,""linesDeleted"":30}",https://api.cursor.com/teams/daily-usage-data,null,2026-07-09 17:14:19.000 +1,"{""ConnectionId"":1,""ScopeId"":""team"",""Endpoint"":""https://api.cursor.com""}","{""day"":""2026-07-08"",""userId"":""user_example123"",""email"":""user@example.com"",""isActive"":true,""completions"":15,""premiumRequests"":3,""agentRequests"":5,""chatRequests"":8,""composerRequests"":2,""totalTabsAccepted"":42,""totalTabsShown"":60,""usageBasedReqs"":7,""subscriptionIncludedReqs"":6,""mostUsedModel"":""composer-2.5-fast"",""clientVersion"":""0.50.3"",""totalLinesAdded"":150,""totalLinesDeleted"":35,""acceptedLinesAdded"":120,""acceptedLinesDeleted"":30,""totalApplies"":10,""totalAccepts"":8,""totalRejects"":2}",https://api.cursor.com/teams/daily-usage-data,null,2026-07-09 17:14:19.000 diff --git a/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_daily_usage.csv b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_daily_usage.csv index cb921061bad..c82f99aef6c 100644 --- a/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_daily_usage.csv +++ b/backend/plugins/cursor/e2e/snapshot_tables/_tool_cursor_daily_usage.csv @@ -1,2 +1,2 @@ -connection_id,scope_id,user_id,usage_date,email,is_active,completions,premium_requests,agent_requests,chat_requests,composer_requests,tabs_accepted,tabs_shown,usage_based_reqs,subscription_included_reqs,most_used_model,client_version,lines_added,lines_deleted -1,team,user_example123,2026-07-08T00:00:00.000+00:00,user@example.com,1,15,3,5,8,2,42,60,7,6,composer-2.5-fast,0.50.3,120,30 +connection_id,scope_id,user_id,usage_date,email,is_active,completions,premium_requests,agent_requests,chat_requests,composer_requests,tabs_accepted,tabs_shown,usage_based_reqs,subscription_included_reqs,most_used_model,client_version,accepted_lines_added,accepted_lines_deleted,total_lines_added,total_lines_deleted,total_applies,total_accepts,total_rejects,lines_added,lines_deleted +1,team,user_example123,2026-07-08T00:00:00.000+00:00,user@example.com,1,15,3,5,8,2,42,60,7,6,composer-2.5-fast,0.50.3,120,30,150,35,10,8,2,120,30 diff --git a/backend/plugins/cursor/models/daily_usage.go b/backend/plugins/cursor/models/daily_usage.go index dcb90ef48ef..d193e374896 100644 --- a/backend/plugins/cursor/models/daily_usage.go +++ b/backend/plugins/cursor/models/daily_usage.go @@ -24,6 +24,8 @@ import ( ) // CursorDailyUsage stores per-user per-day adoption metrics from POST /teams/daily-usage-data. +// Line fields map from acceptedLinesAdded/totalLinesAdded (etc.); LinesAdded/LinesDeleted +// duplicate accepted_lines_added/accepted_lines_deleted for backward compatibility. type CursorDailyUsage struct { ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` @@ -43,6 +45,13 @@ type CursorDailyUsage struct { SubscriptionIncludedReqs int `json:"subscriptionIncludedReqs"` MostUsedModel string `json:"mostUsedModel" gorm:"type:varchar(255)"` ClientVersion string `json:"clientVersion" gorm:"type:varchar(100)"` + AcceptedLinesAdded int `json:"acceptedLinesAdded"` + AcceptedLinesDeleted int `json:"acceptedLinesDeleted"` + TotalLinesAdded int `json:"totalLinesAdded"` + TotalLinesDeleted int `json:"totalLinesDeleted"` + TotalApplies int `json:"totalApplies"` + TotalAccepts int `json:"totalAccepts"` + TotalRejects int `json:"totalRejects"` LinesAdded int `json:"linesAdded"` LinesDeleted int `json:"linesDeleted"` diff --git a/backend/plugins/cursor/models/migrationscripts/20260716_add_daily_usage_line_fields.go b/backend/plugins/cursor/models/migrationscripts/20260716_add_daily_usage_line_fields.go new file mode 100644 index 00000000000..4abedb7e92c --- /dev/null +++ b/backend/plugins/cursor/models/migrationscripts/20260716_add_daily_usage_line_fields.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addCursorDailyUsageLineFields struct{} + +type cursorDailyUsage20260716 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + UserId string `gorm:"primaryKey;type:varchar(255)"` + UsageDate time.Time `gorm:"primaryKey"` + Email string `gorm:"type:varchar(255);index"` + IsActive bool + Completions int + PremiumRequests int + AgentRequests int + ChatRequests int + ComposerRequests int + TabsAccepted int + TabsShown int + UsageBasedReqs int + SubscriptionIncludedReqs int + MostUsedModel string `gorm:"type:varchar(255)"` + ClientVersion string `gorm:"type:varchar(100)"` + AcceptedLinesAdded int + AcceptedLinesDeleted int + TotalLinesAdded int + TotalLinesDeleted int + TotalApplies int + TotalAccepts int + TotalRejects int + LinesAdded int + LinesDeleted int + archived.NoPKModel +} + +func (cursorDailyUsage20260716) TableName() string { return "_tool_cursor_daily_usage" } + +func (script *addCursorDailyUsageLineFields) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &cursorDailyUsage20260716{}) +} + +func (*addCursorDailyUsageLineFields) Version() uint64 { return 20260716120000 } + +func (*addCursorDailyUsageLineFields) Name() string { + return "cursor add daily usage line and apply fields" +} diff --git a/backend/plugins/cursor/models/migrationscripts/register.go b/backend/plugins/cursor/models/migrationscripts/register.go index 1c5e9d2b50a..24e5eedae47 100644 --- a/backend/plugins/cursor/models/migrationscripts/register.go +++ b/backend/plugins/cursor/models/migrationscripts/register.go @@ -25,5 +25,6 @@ func All() []plugin.MigrationScript { new(addCursorInitialTables), new(changeCursorRequestsCostsToFloat), new(addCursorDailyUsage), + new(addCursorDailyUsageLineFields), } } diff --git a/backend/plugins/cursor/tasks/daily_usage_extractor.go b/backend/plugins/cursor/tasks/daily_usage_extractor.go index 80b80538e14..76f4026a1e0 100644 --- a/backend/plugins/cursor/tasks/daily_usage_extractor.go +++ b/backend/plugins/cursor/tasks/daily_usage_extractor.go @@ -44,8 +44,13 @@ type dailyUsageRecord struct { SubscriptionIncludedReqs int `json:"subscriptionIncludedReqs"` MostUsedModel string `json:"mostUsedModel"` ClientVersion string `json:"clientVersion"` - LinesAdded int `json:"linesAdded"` - LinesDeleted int `json:"linesDeleted"` + TotalLinesAdded int `json:"totalLinesAdded"` + TotalLinesDeleted int `json:"totalLinesDeleted"` + AcceptedLinesAdded int `json:"acceptedLinesAdded"` + AcceptedLinesDeleted int `json:"acceptedLinesDeleted"` + TotalApplies int `json:"totalApplies"` + TotalAccepts int `json:"totalAccepts"` + TotalRejects int `json:"totalRejects"` } // ExtractDailyUsage parses raw daily usage records into tool-layer tables. @@ -98,8 +103,15 @@ func ExtractDailyUsage(taskCtx plugin.SubTaskContext) errors.Error { SubscriptionIncludedReqs: record.SubscriptionIncludedReqs, MostUsedModel: strings.TrimSpace(record.MostUsedModel), ClientVersion: strings.TrimSpace(record.ClientVersion), - LinesAdded: record.LinesAdded, - LinesDeleted: record.LinesDeleted, + AcceptedLinesAdded: record.AcceptedLinesAdded, + AcceptedLinesDeleted: record.AcceptedLinesDeleted, + TotalLinesAdded: record.TotalLinesAdded, + TotalLinesDeleted: record.TotalLinesDeleted, + TotalApplies: record.TotalApplies, + TotalAccepts: record.TotalAccepts, + TotalRejects: record.TotalRejects, + LinesAdded: record.AcceptedLinesAdded, + LinesDeleted: record.AcceptedLinesDeleted, } return []interface{}{usage}, nil }, diff --git a/grafana/dashboards/mysql/cursor-usage.json b/grafana/dashboards/mysql/cursor-usage.json index fd550a10843..321c600b48a 100644 --- a/grafana/dashboards/mysql/cursor-usage.json +++ b/grafana/dashboards/mysql/cursor-usage.json @@ -233,7 +233,7 @@ }, { "datasource": "mysql", - "description": "Difference between /teams/spend total and sum of chargedCents for events since billing_cycle_start. Positive means spend exceeds collected events (backfill may be incomplete).", + "description": "Difference between total cycle spend (on-demand + included) from /teams/spend and sum of chargedCents for events since billing_cycle_start. Positive means spend exceeds collected events (backfill may be incomplete).", "fieldConfig": { "defaults": { "decimals": 2, @@ -259,7 +259,7 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT (\n COALESCE((SELECT SUM(spend_cents) FROM _tool_cursor_user_spend\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'), 0)\n - COALESCE((SELECT SUM(charged_cents) FROM _tool_cursor_usage_events\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n AND event_time >= (SELECT MIN(billing_cycle_start) FROM _tool_cursor_user_spend\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}')), 0)\n) / 100 AS \"Reconciliation Delta\"", + "rawSql": "SELECT (\n COALESCE((SELECT SUM(spend_cents + included_spend_cents) FROM _tool_cursor_user_spend\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'), 0)\n - COALESCE((SELECT SUM(charged_cents) FROM _tool_cursor_usage_events\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n AND event_time >= (SELECT MIN(billing_cycle_start) FROM _tool_cursor_user_spend\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}')), 0)\n) / 100 AS \"Reconciliation Delta\"", "refId": "A" } ], @@ -564,7 +564,7 @@ }, { "datasource": "mysql", - "description": "Tab completion acceptance rate: accepted / shown", + "description": "Tab inline-completion acceptance rate only (tabs_accepted / tabs_shown); see AI Line Acceptance Rate for all accepted lines", "fieldConfig": { "defaults": { "color": { "fixedColor": "blue", "mode": "fixed" }, @@ -636,9 +636,83 @@ "title": "Feature Adoption (Weekly)", "type": "timeseries" }, + { + "datasource": "mysql", + "description": "Daily sum of AI-suggested lines accepted into the editor (acceptedLinesAdded from /teams/daily-usage-data)", + "fieldConfig": { + "defaults": { + "color": { "fixedColor": "blue", "mode": "fixed" }, + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "lineWidth": 2, + "showPoints": "auto", + "pointSize": 4 + }, + "unit": "locale", + "decimals": 0 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, + "id": 204, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT usage_date AS time,\n SUM(accepted_lines_added) AS \"Accepted Lines Added\"\nFROM _tool_cursor_daily_usage\nWHERE $__timeFilter(usage_date)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_active = 1\nGROUP BY usage_date\nORDER BY 1", + "refId": "A" + } + ], + "title": "Accepted Lines Added", + "type": "timeseries" + }, + { + "datasource": "mysql", + "description": "Share of all lines added that came from accepted AI suggestions (accepted_lines_added / total_lines_added)", + "fieldConfig": { + "defaults": { + "color": { "fixedColor": "blue", "mode": "fixed" }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 2, + "showPoints": "auto", + "pointSize": 4 + }, + "unit": "percent", + "decimals": 1 + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, + "id": 205, + "options": { + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT usage_date AS time,\n ROUND(SUM(accepted_lines_added) * 100.0 / NULLIF(SUM(total_lines_added), 0), 1) AS \"AI Line Acceptance Rate\"\nFROM _tool_cursor_daily_usage\nWHERE $__timeFilter(usage_date)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_active = 1\nGROUP BY usage_date\nORDER BY 1", + "refId": "A" + } + ], + "title": "AI Line Acceptance Rate", + "type": "timeseries" + }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 44 }, "id": 104, "title": "Diagnostics", "type": "row", @@ -647,7 +721,7 @@ "datasource": "mysql", "description": "Row counts and latest timestamps per _tool_cursor_* table for pipeline health monitoring", "fieldConfig": { "defaults": {}, "overrides": [] }, - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 45 }, + "gridPos": { "h": 6, "w": 12, "x": 0, "y": 53 }, "id": 40, "options": { "cellHeight": "sm", From 4bedce7ab70d7fb46d6d1a9852866a1122383b92 Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Thu, 23 Jul 2026 09:17:37 -0600 Subject: [PATCH 07/13] feat(cursor): add stateful extract and docs * Switch usage events and daily usage extractors to StatefulApiExtractor with versioned one-time full re-extract. * Chunk filtered-usage-events into 30-day ranges and add plugin README. Signed-off-by: Joshua Smith --- backend/plugins/cursor/README.md | 169 ++++++++++++++++++ .../plugins/cursor/tasks/collector_utils.go | 4 +- .../cursor/tasks/collector_utils_test.go | 10 ++ .../cursor/tasks/daily_usage_extractor.go | 18 +- backend/plugins/cursor/tasks/extract_utils.go | 38 ++++ .../cursor/tasks/usage_events_collector.go | 10 +- .../cursor/tasks/usage_events_extractor.go | 18 +- 7 files changed, 233 insertions(+), 34 deletions(-) create mode 100644 backend/plugins/cursor/README.md create mode 100644 backend/plugins/cursor/tasks/extract_utils.go diff --git a/backend/plugins/cursor/README.md b/backend/plugins/cursor/README.md new file mode 100644 index 00000000000..bf71ec79f39 --- /dev/null +++ b/backend/plugins/cursor/README.md @@ -0,0 +1,169 @@ + +# Cursor Plugin (Usage & Cost) + +This plugin ingests **Cursor team usage, billing, and adoption metrics** from the [Cursor Admin API](https://cursor.com/docs/account/teams/admin-api) and stores them in DevLake tool-layer tables for Grafana dashboards and SQL analysis. + +It follows the same structure and patterns as other DevLake AI usage plugins (notably `backend/plugins/gh-copilot`). + +## What it collects + +**Cursor Admin API endpoints:** + +| Endpoint | Method | Data | +|----------|--------|------| +| `/teams/members` | GET | Team roster | +| `/teams/spend` | POST | Per-user billing cycle spend | +| `/teams/filtered-usage-events` | POST | Event-level usage and charges | +| `/teams/daily-usage-data` | POST | Per-user per-day adoption metrics | + +**Stored data (tool layer):** + +| Table | Description | +|-------|-------------| +| `_tool_cursor_members` | Team member roster (email, name, role) | +| `_tool_cursor_usage_events` | Billable usage events with model, tokens, and charged amounts | +| `_tool_cursor_user_spend` | Per-user spend for the current billing cycle (on-demand and included) | +| `_tool_cursor_daily_usage` | Daily adoption metrics: completions, requests by feature, tab acceptance, line edits | + +Data is collected in the **Raw → Tool** layers only. There is no domain-layer converter in this plugin; Grafana dashboards query `_tool_cursor_*` tables directly. + +## Data flow + +```mermaid +flowchart LR + API[Cursor Admin API] + RAW[(Raw tables\n_raw_cursor_*)] + TOOL[(Tool tables\n_tool_cursor_*)] + GRAF[Grafana Dashboards] + + API --> RAW --> TOOL --> GRAF +``` + +**Pipeline subtasks (in order):** + +1. `collectMembers` → `extractMembers` +2. `collectUsageEvents` → `extractUsageEvents` +3. `collectUserSpend` → `extractUserSpend` +4. `collectDailyUsage` → `extractDailyUsage` + +## Repository layout + +- `api/` — REST layer for connections, scopes, and scope configs +- `impl/` — plugin meta, blueprint v200, connection helpers +- `models/` — tool-layer models and migration scripts +- `tasks/` — collectors, extractors, and pipeline registration +- `service/` — connection test logic (Admin API permission probes) +- `e2e/` — E2E fixtures and golden CSV assertions + +## Setup + +### Prerequisites + +- A Cursor **Team or Business** plan with Admin API access +- A **Team Admin API key** from the Cursor dashboard (Dashboard → API Keys) +- Do **not** use a User API key from Settings → Integrations — user keys cannot access `/teams/*` endpoints + +Authentication uses HTTP Basic auth: the API key as username with an empty password. + +### 1) Create a connection + +1. DevLake UI → **Connections → Add Connection → Cursor** +2. Fill in: + - **Name**: e.g. `Cursor Production Team` + - **Endpoint**: defaults to `https://api.cursor.com` + - **Token**: Team Admin API key + - **Rate Limit**: defaults to 1,200 requests/hour (Cursor documents 20 requests/minute) +3. Click **Test Connection**. DevLake probes `/teams/members`, `/teams/spend`, and `/teams/filtered-usage-events` and reports which endpoints the key can access. +4. Save the connection. + +When updating an existing connection, omit the token field to keep the encrypted value already stored in DevLake. + +### 2) Add a scope + +Cursor data is team-level. Add a **Team** scope for the connection. The default scope ID is `team`. + +### 3) Create a blueprint + +Use a blueprint plan like: + +```json +[ + [ + { + "plugin": "cursor", + "options": { + "connectionId": 1, + "scopeId": "team" + } + } + ] +] +``` + +Run the blueprint on a daily schedule to keep usage and cost data current. + +### Collection behavior + +- **Initial backfill**: usage events and daily usage collect up to **90 days** of history on the first run. +- **Incremental runs**: subsequent runs use the pipeline sync policy (`TimeAfter`) to collect only new data. +- **Date range chunking**: both `/teams/daily-usage-data` and `/teams/filtered-usage-events` requests are split into **30-day** chunks (API limit for daily usage; applied to usage events for resilience). +- **Extract**: `extractUsageEvents` and `extractDailyUsage` use **StatefulApiExtractor** (incremental by default). A config version bump triggers a one-time full re-extract after upgrade. +- **Rate limiting**: collectors honor `Retry-After` response headers and respect the configured `rateLimitPerHour`. + +## Dashboards + +Grafana dashboard JSON lives under `grafana/dashboards/mysql/`: + +| Dashboard | File | UID | +|-----------|------|-----| +| Cursor Usage & Cost | `cursor-usage.json` | `cursor_usage` | +| AI Cost Efficiency (Cursor panels) | `ai-cost-efficiency.json` | — | +| Multi-AI Comparison (Cursor panels) | `multi-ai-comparison.json` | — | + +See `grafana/dashboards/mysql/CursorREADME.md` for dashboard prerequisites, variables, and panel descriptions. + +## Error handling + +| Symptom | Likely cause | +|---------|--------------| +| **401 Unauthorized** on test connection | Invalid API key, or a User API key instead of a Team Admin key | +| **403 Forbidden** on spend or usage events | Key lacks permission for that Admin API endpoint | +| **429 Too Many Requests** | Rate limit exceeded — lower `rateLimitPerHour` or wait for `Retry-After` | +| Empty usage events after successful run | Selected time range has no billable events, or sync policy excludes the date range | +| Reconciliation delta on dashboard | Expected when comparing event-level charges to billing-cycle spend snapshots; see dashboard notes | + +Tokens are sanitized before persisting. Connection test results include a `permissions` object showing which Admin API endpoints succeeded. + +## Limitations + +- **Team/Business Admin API only** — Enterprise-only Analytics API endpoints (`/analytics/*`) are not collected in this plugin. +- **Tool layer only** — no domain-layer tables; cross-plugin joins (Jira, GitHub PRs, etc.) are done in Grafana SQL or separate tooling. +- **Team-level scope** — one scope per connection represents the whole team; per-team multi-tenant collection is not supported. +- **Beta** — the plugin is marked beta in Config UI while the Admin API surface continues to evolve. + +## Testing + +```sh +# Unit tests +cd backend && go test ./plugins/cursor/... + +# E2E (requires E2E_DB_URL) +make e2e-test +``` + +E2E fixtures live in `backend/plugins/cursor/e2e/raw_tables/` and `e2e/snapshot_tables/`. diff --git a/backend/plugins/cursor/tasks/collector_utils.go b/backend/plugins/cursor/tasks/collector_utils.go index 6be6c1ba583..612ba5ec774 100644 --- a/backend/plugins/cursor/tasks/collector_utils.go +++ b/backend/plugins/cursor/tasks/collector_utils.go @@ -95,8 +95,8 @@ func computeUsageTimeRangeMs(since *time.Time, now time.Time) (int64, int64) { return start.UnixMilli(), end.UnixMilli() } -// splitDailyUsageTimeRangeMs splits [startMs, endMs] into chunks of at most maxDays for -// POST /teams/daily-usage-data (API limit: date range cannot exceed 30 days). +// splitDailyUsageTimeRangeMs splits [startMs, endMs] into chunks of at most maxDays. +// Used by daily-usage-data (30-day API limit) and filtered-usage-events collectors. func splitDailyUsageTimeRangeMs(startMs, endMs int64, maxDays int) []cursorTimeRangeInput { if startMs >= endMs || maxDays <= 0 { return nil diff --git a/backend/plugins/cursor/tasks/collector_utils_test.go b/backend/plugins/cursor/tasks/collector_utils_test.go index c2af757c42f..7bb9f65d0a6 100644 --- a/backend/plugins/cursor/tasks/collector_utils_test.go +++ b/backend/plugins/cursor/tasks/collector_utils_test.go @@ -83,6 +83,16 @@ func TestSplitDailyUsageTimeRangeMsEmptyRange(t *testing.T) { } } +func TestNewUsageEventsDateRangeIteratorChunksLongRanges(t *testing.T) { + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2026, 4, 1, 12, 0, 0, 0, time.UTC) + chunks := splitDailyUsageTimeRangeMs(start.UnixMilli(), end.UnixMilli(), cursorDailyUsageMaxDays) + + if len(chunks) != 4 { + t.Fatalf("expected 4 chunks for ~90-day usage-events span, got %d", len(chunks)) + } +} + func TestDailyUsagePostBodyUsesEpochMilliseconds(t *testing.T) { body := dailyUsagePostBody(&helper.RequestData{ Input: cursorTimeRangeInput{StartDateMs: 1710720000000, EndDateMs: 1710892800000}, diff --git a/backend/plugins/cursor/tasks/daily_usage_extractor.go b/backend/plugins/cursor/tasks/daily_usage_extractor.go index 76f4026a1e0..1a493f01d64 100644 --- a/backend/plugins/cursor/tasks/daily_usage_extractor.go +++ b/backend/plugins/cursor/tasks/daily_usage_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "strings" "time" @@ -60,18 +59,9 @@ func ExtractDailyUsage(taskCtx plugin.SubTaskContext) errors.Error { return errors.Default.New("task data is not CursorTaskData") } - extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ - RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Table: rawDailyUsageTable, - Options: rawParamsFromTaskData(data), - }, - Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { - var record dailyUsageRecord - if err := errors.Convert(json.Unmarshal(row.Data, &record)); err != nil { - return nil, err - } - + extractor, err := helper.NewStatefulApiExtractor(&helper.StatefulApiExtractorArgs[dailyUsageRecord]{ + SubtaskCommonArgs: cursorSubtaskCommonArgs(taskCtx, data, rawDailyUsageTable), + Extract: func(record *dailyUsageRecord, _ *helper.RawData) ([]any, errors.Error) { userId := strings.TrimSpace(record.UserId) if userId == "" { userId = strings.TrimSpace(record.Email) @@ -113,7 +103,7 @@ func ExtractDailyUsage(taskCtx plugin.SubTaskContext) errors.Error { LinesAdded: record.AcceptedLinesAdded, LinesDeleted: record.AcceptedLinesDeleted, } - return []interface{}{usage}, nil + return []any{usage}, nil }, }) if err != nil { diff --git a/backend/plugins/cursor/tasks/extract_utils.go b/backend/plugins/cursor/tasks/extract_utils.go new file mode 100644 index 00000000000..c755b526960 --- /dev/null +++ b/backend/plugins/cursor/tasks/extract_utils.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// cursorExtractorVersion bumps when extract logic changes in a way that requires +// a one-time full re-extract (SubtaskStateManager compares SubtaskConfig). +const cursorExtractorVersion = 2 + +func cursorSubtaskCommonArgs(taskCtx plugin.SubTaskContext, data *CursorTaskData, rawTable string) *helper.SubtaskCommonArgs { + return &helper.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: rawTable, + Params: rawParamsFromTaskData(data), + SubtaskConfig: map[string]any{ + "extractorVersion": cursorExtractorVersion, + }, + } +} diff --git a/backend/plugins/cursor/tasks/usage_events_collector.go b/backend/plugins/cursor/tasks/usage_events_collector.go index a72310deeb0..fe5a04e68b9 100644 --- a/backend/plugins/cursor/tasks/usage_events_collector.go +++ b/backend/plugins/cursor/tasks/usage_events_collector.go @@ -26,10 +26,12 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) -func newTimeRangeIterator(since *time.Time) *helper.QueueIterator { - iter := helper.NewQueueIterator() +func newUsageEventsDateRangeIterator(since *time.Time) *helper.QueueIterator { startMs, endMs := computeUsageTimeRangeMs(since, time.Now().UTC()) - iter.Push(cursorTimeRangeInput{StartDateMs: startMs, EndDateMs: endMs}) + iter := helper.NewQueueIterator() + for _, chunk := range splitDailyUsageTimeRangeMs(startMs, endMs, cursorDailyUsageMaxDays) { + iter.Push(chunk) + } return iter } @@ -90,7 +92,7 @@ func CollectUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { err = collector.InitCollector(helper.ApiCollectorArgs{ ApiClient: apiClient, - Input: newTimeRangeIterator(collector.GetSince()), + Input: newUsageEventsDateRangeIterator(collector.GetSince()), Method: http.MethodPost, UrlTemplate: "teams/filtered-usage-events", PageSize: cursorApiPageSize, diff --git a/backend/plugins/cursor/tasks/usage_events_extractor.go b/backend/plugins/cursor/tasks/usage_events_extractor.go index efc2fab155b..25ee1071741 100644 --- a/backend/plugins/cursor/tasks/usage_events_extractor.go +++ b/backend/plugins/cursor/tasks/usage_events_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "strings" "github.com/apache/incubator-devlake/core/errors" @@ -60,18 +59,9 @@ func ExtractUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { return errors.Default.New("task data is not CursorTaskData") } - extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ - RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Table: rawUsageEventsTable, - Options: rawParamsFromTaskData(data), - }, - Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { - var record usageEventRecord - if err := errors.Convert(json.Unmarshal(row.Data, &record)); err != nil { - return nil, err - } - + extractor, err := helper.NewStatefulApiExtractor(&helper.StatefulApiExtractorArgs[usageEventRecord]{ + SubtaskCommonArgs: cursorSubtaskCommonArgs(taskCtx, data, rawUsageEventsTable), + Extract: func(record *usageEventRecord, _ *helper.RawData) ([]any, errors.Error) { eventTime, err := parseEventTimestampMs(record.Timestamp) if err != nil { return nil, err @@ -108,7 +98,7 @@ func ExtractUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { event.CacheWriteTokens = record.TokenUsage.CacheWriteTokens event.TotalCents = record.TokenUsage.TotalCents } - return []interface{}{event}, nil + return []any{event}, nil }, }) if err != nil { From 4e387f157f5989fb878baff70d5d3ef9f5baf868 Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Thu, 23 Jul 2026 11:17:01 -0600 Subject: [PATCH 08/13] feat(grafana): refresh Cursor usage dashboard * Align cost panels with currencyUSD and stacked daily on-demand versus included spend. * Document Enterprise AI Code Tracking gaps in the cursor plugin README. Signed-off-by: Joshua Smith --- backend/plugins/cursor/README.md | 25 +- grafana/dashboards/mysql/cursor-usage.json | 955 ++++++++++++++++----- 2 files changed, 782 insertions(+), 198 deletions(-) diff --git a/backend/plugins/cursor/README.md b/backend/plugins/cursor/README.md index bf71ec79f39..8ca44da79f4 100644 --- a/backend/plugins/cursor/README.md +++ b/backend/plugins/cursor/README.md @@ -135,7 +135,7 @@ Grafana dashboard JSON lives under `grafana/dashboards/mysql/`: | AI Cost Efficiency (Cursor panels) | `ai-cost-efficiency.json` | — | | Multi-AI Comparison (Cursor panels) | `multi-ai-comparison.json` | — | -See `grafana/dashboards/mysql/CursorREADME.md` for dashboard prerequisites, variables, and panel descriptions. +See `grafana/dashboards/mysql/cursor-usage.json` for the dashboard definition and [rak_tools/grafana.md](../../../rak_tools/grafana.md) for currency formatting and Grafana version notes. ## Error handling @@ -149,9 +149,30 @@ See `grafana/dashboards/mysql/CursorREADME.md` for dashboard prerequisites, vari Tokens are sanitized before persisting. Connection test results include a `permissions` object showing which Admin API endpoints succeeded. +## Not collected (Enterprise AI Code Tracking) + +The following endpoints from the [AI Code Tracking API](https://cursor.com/docs/account/teams/ai-code-tracking-api) are **Enterprise plan only** and are **not implemented** in this plugin (no collector, extractor, or `_tool_cursor_*` table): + +| Endpoint | Purpose | +|----------|---------| +| `GET /analytics/ai-code/commits` | Per-commit AI line attribution (TAB vs Composer vs non-AI) | +| `GET /analytics/ai-code/changes` | Granular accepted AI changes | +| `GET /analytics/ai-code/commits.csv` / `changes.csv` | Bulk CSV exports of the above | + +Team/Business Admin API keys typically receive **401** on these routes. Connection test optionally probes `GET /analytics/team/dau` (`probeEnterpriseAnalytics`) to detect Enterprise access; it does **not** ingest analytics data. + +**Metrics this would unlock** (shown on the Cursor native dashboard but absent from DevLake today): + +- AI share of **committed** code (not editor line acceptance) +- Per-commit `commitSource`: IDE, CLI, or cloud +- Repo name and primary-branch filters +- TAB vs Composer line attribution on commits + +The **Cursor Usage** Grafana dashboard (`grafana/dashboards/mysql/cursor-usage.json`) uses Admin API proxies from `_tool_cursor_daily_usage` and `_tool_cursor_usage_events` instead; panel descriptions note where metrics are approximate. + ## Limitations -- **Team/Business Admin API only** — Enterprise-only Analytics API endpoints (`/analytics/*`) are not collected in this plugin. +- **Team/Business Admin API only** — Enterprise-only Analytics API endpoints (`/analytics/*`) are not collected in this plugin (see [Not collected (Enterprise AI Code Tracking)](#not-collected-enterprise-ai-code-tracking) above). - **Tool layer only** — no domain-layer tables; cross-plugin joins (Jira, GitHub PRs, etc.) are done in Grafana SQL or separate tooling. - **Team-level scope** — one scope per connection represents the whole team; per-team multi-tenant collection is not supported. - **Beta** — the plugin is marked beta in Config UI while the Admin API surface continues to evolve. diff --git a/grafana/dashboards/mysql/cursor-usage.json b/grafana/dashboards/mysql/cursor-usage.json index 321c600b48a..2331169331e 100644 --- a/grafana/dashboards/mysql/cursor-usage.json +++ b/grafana/dashboards/mysql/cursor-usage.json @@ -3,7 +3,10 @@ "list": [ { "builtIn": 1, - "datasource": { "type": "datasource", "uid": "grafana" }, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", @@ -20,7 +23,12 @@ "panels": [ { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, "id": 100, "title": "Overview", "type": "row" @@ -30,20 +38,39 @@ "description": "Total charged amount for billable usage events in the selected time range", "fieldConfig": { "defaults": { - "decimals": 2, "unit": "currencyUSD", - "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "decimals": 2 }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, "id": 1, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto", "wideLayout": true }, @@ -62,23 +89,42 @@ }, { "datasource": "mysql", - "description": "Total usage events in selected period from filtered-usage-events API", + "description": "Current billing cycle on-demand spend from /teams/spend", "fieldConfig": { "defaults": { - "unit": "locale", - "decimals": 0, - "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + "unit": "currencyUSD", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "decimals": 2 }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, - "id": 3, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 5, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto", "wideLayout": true }, @@ -88,30 +134,51 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS \"Usage Events\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "rawSql": "SELECT COALESCE(SUM(spend_cents), 0) / 100 AS \"On-Demand Spend\"\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", "refId": "A" } ], - "title": "Usage Events (Selected Period)", + "title": "On-Demand Spend (Current Cycle)", "type": "stat" }, { "datasource": "mysql", - "description": "Active roster count from /teams/members where is_removed = 0", + "description": "Current billing cycle included (non-on-demand) spend from /teams/spend", "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + "unit": "currencyUSD", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "decimals": 2 }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, - "id": 4, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 301, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto", "wideLayout": true }, @@ -121,30 +188,51 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS \"Team Members\"\nFROM _tool_cursor_members\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_removed = 0", + "rawSql": "SELECT COALESCE(SUM(included_spend_cents), 0) / 100 AS \"Included Spend\"\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", "refId": "A" } ], - "title": "Team Members", + "title": "Included Spend (Current Cycle)", "type": "stat" }, { "datasource": "mysql", - "description": "Team members with zero active days in the trailing 7-day window (relative to latest collected usage_date)", + "description": "Difference between total cycle spend (on-demand + included) from /teams/spend and sum of chargedCents for events since billing_cycle_start. Positive means spend exceeds collected events (backfill may be incomplete).", "fieldConfig": { "defaults": { - "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + "unit": "currencyUSD", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "decimals": 2 }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, - "id": 303, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 302, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto", "wideLayout": true }, @@ -154,32 +242,51 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS \"Inactive Users\"\nFROM (\n SELECT user_id\n FROM _tool_cursor_daily_usage\n WHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND usage_date >= DATE_SUB(\n (SELECT MAX(usage_date) FROM _tool_cursor_daily_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'),\n INTERVAL 6 DAY\n )\n GROUP BY user_id\n HAVING MAX(is_active) = 0\n) t", + "rawSql": "SELECT (\n COALESCE((SELECT SUM(spend_cents + included_spend_cents) FROM _tool_cursor_user_spend\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'), 0)\n - COALESCE((SELECT SUM(charged_cents) FROM _tool_cursor_usage_events\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n AND event_time >= (SELECT MIN(billing_cycle_start) FROM _tool_cursor_user_spend\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}')), 0)\n) / 100 AS \"Reconciliation Delta\"", "refId": "A" } ], - "title": "Inactive Users (Last 7 Days)", + "title": "Reconciliation Delta", "type": "stat" }, { "datasource": "mysql", - "description": "Current billing cycle on-demand spend from /teams/spend", + "description": "Total usage events in selected period from filtered-usage-events API", "fieldConfig": { "defaults": { - "decimals": 2, - "unit": "currencyUSD", - "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + "unit": "locale", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 0, "y": 5 }, - "id": 5, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto", "wideLayout": true }, @@ -189,32 +296,51 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(SUM(spend_cents), 0) / 100 AS \"On-Demand Spend\"\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "rawSql": "SELECT COUNT(*) AS \"Usage Events\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", "refId": "A" } ], - "title": "On-Demand Spend (Current Cycle)", + "title": "Usage Events (Selected Period)", "type": "stat" }, { "datasource": "mysql", - "description": "Current billing cycle included (non-on-demand) spend from /teams/spend", + "description": "Active roster count from /teams/members where is_removed = 0", "fieldConfig": { "defaults": { - "decimals": 2, - "unit": "currencyUSD", - "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + "unit": "locale", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 6, "y": 5 }, - "id": 301, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto", "wideLayout": true }, @@ -224,32 +350,51 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT COALESCE(SUM(included_spend_cents), 0) / 100 AS \"Included Spend\"\nFROM _tool_cursor_user_spend\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "rawSql": "SELECT COUNT(*) AS \"Team Members\"\nFROM _tool_cursor_members\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_removed = 0", "refId": "A" } ], - "title": "Included Spend (Current Cycle)", + "title": "Team Members", "type": "stat" }, { "datasource": "mysql", - "description": "Difference between total cycle spend (on-demand + included) from /teams/spend and sum of chargedCents for events since billing_cycle_start. Positive means spend exceeds collected events (backfill may be incomplete).", + "description": "Team members with zero active days in the trailing 7-day window (relative to latest collected usage_date)", "fieldConfig": { "defaults": { - "decimals": 2, - "unit": "currencyUSD", - "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + "unit": "locale", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 12, "y": 5 }, - "id": 302, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 303, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto", "wideLayout": true }, @@ -259,11 +404,11 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT (\n COALESCE((SELECT SUM(spend_cents + included_spend_cents) FROM _tool_cursor_user_spend\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'), 0)\n - COALESCE((SELECT SUM(charged_cents) FROM _tool_cursor_usage_events\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n AND event_time >= (SELECT MIN(billing_cycle_start) FROM _tool_cursor_user_spend\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}')), 0)\n) / 100 AS \"Reconciliation Delta\"", + "rawSql": "SELECT COUNT(*) AS \"Inactive Users\"\nFROM (\n SELECT user_id\n FROM _tool_cursor_daily_usage\n WHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND usage_date >= DATE_SUB(\n (SELECT MAX(usage_date) FROM _tool_cursor_daily_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'),\n INTERVAL 6 DAY\n )\n GROUP BY user_id\n HAVING MAX(is_active) = 0\n) t", "refId": "A" } ], - "title": "Reconciliation Delta", + "title": "Inactive Users (Last 7 Days)", "type": "stat" }, { @@ -271,25 +416,47 @@ "description": "Days since most recent usage event; green < 2d, yellow 2-6d, red >= 7d", "fieldConfig": { "defaults": { + "decimals": 0, + "unit": "locale", "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 2 }, - { "color": "red", "value": 7 } + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 7 + } ] } }, "overrides": [] }, - "gridPos": { "h": 4, "w": 6, "x": 18, "y": 5 }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 5 + }, "id": 6, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "textMode": "auto", "wideLayout": true }, @@ -308,75 +475,85 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, "id": 101, "title": "Cost Trends", "type": "row" }, { "datasource": "mysql", - "description": "Daily charged spend from usage events", + "description": "Daily charged spend split by billing kind: on-demand (Usage-based) vs included (Included in Business)", "fieldConfig": { "defaults": { - "color": { "fixedColor": "blue", "mode": "fixed" }, "custom": { "axisLabel": "USD", "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, "showPoints": "never", - "stacking": { "group": "A", "mode": "none" } + "stacking": { + "group": "A", + "mode": "normal" + } }, - "unit": "currencyUSD", - "decimals": 2 - }, - "overrides": [] - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, - "id": 10, - "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "single", "sort": "none" } - }, - "pluginVersion": "11.0.0", - "targets": [ - { - "datasource": "mysql", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT DATE(event_time) AS time,\n SUM(charged_cents) / 100 AS \"Charged Amount\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY DATE(event_time)\nORDER BY 1", - "refId": "A" - } - ], - "title": "Daily Charged Spend", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "Daily event count by billing kind", - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "drawStyle": "bars", - "fillOpacity": 80, - "stacking": { "group": "A", "mode": "normal" } - } + "decimals": 2, + "unit": "currencyUSD" }, "overrides": [ - { "matcher": { "id": "byName", "options": "Usage-based" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Included in Business" }, "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Free" }, "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Errored, Not Charged" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Aborted, Not Charged" }, "properties": [{ "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "User API Key" }, "properties": [{ "id": "color", "value": { "fixedColor": "text", "mode": "fixed" } }] } + { + "matcher": { + "id": "byName", + "options": "On-Demand" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Included" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } ] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, - "id": 11, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 10, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "pluginVersion": "11.0.0", "targets": [ @@ -384,75 +561,161 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT DATE(event_time) AS time,\n kind AS metric,\n COUNT(*) AS value\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY DATE(event_time), kind\nORDER BY 1", + "rawSql": "SELECT DATE(event_time) AS time,\n SUM(CASE WHEN kind = 'Usage-based' THEN charged_cents ELSE 0 END) / 100 AS \"On-Demand\",\n SUM(CASE WHEN kind = 'Included in Business' THEN charged_cents ELSE 0 END) / 100 AS \"Included\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY DATE(event_time)\nORDER BY 1", "refId": "A" } ], - "title": "Events by Kind (Daily)", - "type": "timeseries", - "transformations": [ - { "id": "prepareTimeSeries", "options": { "format": "many" } }, - { "id": "renameByRegex", "options": { "regex": "^value (.+)$", "renamePattern": "$1" } } - ] + "title": "Daily Spend (On-Demand + Included)", + "type": "timeseries" }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, "id": 102, "title": "Breakdowns", "type": "row" }, { "datasource": "mysql", - "description": "Average cost per event, total charged, event count, and share of total events by model for the selected period. Table defaults to charged spend descending. Events and Usage % columns sort numerically; Charged and Avg Cost use formatted USD strings.", + "description": "Average cost per event, total charged, event count, and share of total events by model for the selected period. Table defaults to charged spend descending.", "fieldConfig": { "defaults": { - "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false } + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + } }, "overrides": [ { - "matcher": { "id": "byName", "options": "Model" }, - "properties": [{ "id": "custom.width", "value": 320 }] + "matcher": { + "id": "byName", + "options": "Model" + }, + "properties": [ + { + "id": "custom.width", + "value": 320 + } + ] }, { - "matcher": { "id": "byName", "options": "Avg Cost" }, + "matcher": { + "id": "byName", + "options": "Avg Cost" + }, "properties": [ - { "id": "custom.width", "value": 120 }, - { "id": "custom.align", "value": "right" } + { + "id": "custom.width", + "value": 120 + }, + { + "id": "custom.align", + "value": "right" + }, + { + "id": "decimals", + "value": 4 + }, + { + "id": "unit", + "value": "currencyUSD" + } ] }, { - "matcher": { "id": "byName", "options": "Charged" }, + "matcher": { + "id": "byName", + "options": "Charged" + }, "properties": [ - { "id": "custom.width", "value": 140 }, - { "id": "custom.align", "value": "right" } + { + "id": "custom.width", + "value": 140 + }, + { + "id": "custom.align", + "value": "right" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "unit", + "value": "currencyUSD" + } ] }, { - "matcher": { "id": "byName", "options": "Events" }, + "matcher": { + "id": "byName", + "options": "Events" + }, "properties": [ - { "id": "unit", "value": "locale" }, - { "id": "decimals", "value": 0 }, - { "id": "custom.width", "value": 100 }, - { "id": "custom.align", "value": "right" } + { + "id": "unit", + "value": "locale" + }, + { + "id": "decimals", + "value": 0 + }, + { + "id": "custom.width", + "value": 100 + }, + { + "id": "custom.align", + "value": "right" + } ] }, { - "matcher": { "id": "byName", "options": "Usage %" }, + "matcher": { + "id": "byName", + "options": "Usage %" + }, "properties": [ - { "id": "unit", "value": "percent" }, - { "id": "decimals", "value": 1 }, - { "id": "custom.width", "value": 80 }, - { "id": "custom.align", "value": "right" } + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.width", + "value": 80 + }, + { + "id": "custom.align", + "value": "right" + } ] } ] }, - "gridPos": { "h": 8, "w": 14, "x": 0, "y": 19 }, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 19 + }, "id": 21, "options": { "cellHeight": "sm", - "footer": { "show": false }, + "footer": { + "show": false + }, "showHeader": true }, "pluginVersion": "11.0.0", @@ -461,7 +724,7 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT model AS \"Model\",\n CONCAT('$', FORMAT(SUM(total_cents) / COUNT(*) / 100, 4)) AS \"Avg Cost\",\n CONCAT('$', FORMAT(SUM(charged_cents) / 100, 2)) AS \"Charged\",\n COUNT(*) AS \"Events\",\n COUNT(*) * 100.0 / SUM(COUNT(*)) OVER () AS \"Usage %\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY model\nORDER BY SUM(charged_cents) DESC", + "rawSql": "SELECT model AS \"Model\",\n SUM(total_cents) / COUNT(*) / 100 AS \"Avg Cost\",\n SUM(charged_cents) / 100 AS \"Charged\",\n COUNT(*) AS \"Events\",\n COUNT(*) * 100.0 / SUM(COUNT(*)) OVER () AS \"Usage %\"\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY model\nORDER BY SUM(charged_cents) DESC", "refId": "A" } ], @@ -475,25 +738,131 @@ "defaults": { "unit": "currencyUSD", "decimals": 2, - "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, - "color": { "mode": "fixed", "fixedColor": "blue" } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "color": { + "mode": "fixed", + "fixedColor": "blue" + } }, "overrides": [ - { "matcher": { "id": "byName", "options": "Usage-based" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Included in Business" }, "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Free" }, "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Errored, Not Charged" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Aborted, Not Charged" }, "properties": [{ "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "User API Key" }, "properties": [{ "id": "color", "value": { "fixedColor": "text", "mode": "fixed" } }] } + { + "matcher": { + "id": "byName", + "options": "Usage-based" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Included in Business" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Errored, Not Charged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Aborted, Not Charged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User API Key" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + } ] }, - "gridPos": { "h": 8, "w": 10, "x": 14, "y": 19 }, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 19 + }, "id": 22, "options": { "orientation": "horizontal", "displayMode": "basic", "showUnfilled": false, - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": true }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, "text": {} }, "pluginVersion": "11.0.0", @@ -513,8 +882,14 @@ "id": "rowsToFields", "options": { "mappings": [ - { "fieldName": "Kind", "handlerKey": "field.name" }, - { "fieldName": "Charged", "handlerKey": "field.value" } + { + "fieldName": "Kind", + "handlerKey": "field.name" + }, + { + "fieldName": "Charged", + "handlerKey": "field.value" + } ] } } @@ -522,7 +897,12 @@ }, { "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, "id": 200, "title": "Adoption (Daily Usage)", "type": "row" @@ -532,22 +912,39 @@ "description": "Daily active users based on isActive flag from /teams/daily-usage-data", "fieldConfig": { "defaults": { - "color": { "fixedColor": "blue", "mode": "fixed" }, + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, "custom": { "drawStyle": "line", "fillOpacity": 20, "lineWidth": 2, "showPoints": "auto", "pointSize": 4 - } + }, + "unit": "locale", + "decimals": 0 }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 28 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 28 + }, "id": 201, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "single", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, "pluginVersion": "11.0.0", "targets": [ @@ -567,7 +964,10 @@ "description": "Tab inline-completion acceptance rate only (tabs_accepted / tabs_shown); see AI Line Acceptance Rate for all accepted lines", "fieldConfig": { "defaults": { - "color": { "fixedColor": "blue", "mode": "fixed" }, + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, "custom": { "drawStyle": "line", "fillOpacity": 10, @@ -580,11 +980,23 @@ }, "overrides": [] }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 28 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 28 + }, "id": 202, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "single", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, "pluginVersion": "11.0.0", "targets": [ @@ -592,7 +1004,7 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT usage_date AS time,\n ROUND(SUM(tabs_accepted) * 100.0 / NULLIF(SUM(tabs_shown), 0), 1) AS \"Acceptance Rate\"\nFROM _tool_cursor_daily_usage\nWHERE $__timeFilter(usage_date)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_active = 1\nGROUP BY usage_date\nORDER BY 1", + "rawSql": "SELECT usage_date AS time,\n SUM(tabs_accepted) * 100.0 / NULLIF(SUM(tabs_shown), 0) AS \"Acceptance Rate\"\nFROM _tool_cursor_daily_usage\nWHERE $__timeFilter(usage_date)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_active = 1\nGROUP BY usage_date\nORDER BY 1", "refId": "A" } ], @@ -601,27 +1013,103 @@ }, { "datasource": "mysql", - "description": "Weekly request volume by feature type (Agent, Chat, Composer)", + "description": "Daily usage-event volume by channel from POST /teams/filtered-usage-events. Admin API does not populate hostingType in collected data (Apr–Jul 2026: 0/105k raw events). Mapping: is_headless=1 → Cloud Agent (~1%); empty hosting_type + not headless → IDE; hosting_type=cli → CLI when present. CLI and authoritative IDE/CLI/Cloud commit activity on the Cursor native dashboard use Enterprise GET /analytics/ai-code/commits (commitSource) — not collected.", "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "drawStyle": "bars", "fillOpacity": 80, - "stacking": { "group": "A", "mode": "normal" } - } + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "locale", + "decimals": 0 }, "overrides": [ - { "matcher": { "id": "byName", "options": "Agent" }, "properties": [{ "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Chat" }, "properties": [{ "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }] }, - { "matcher": { "id": "byName", "options": "Composer" }, "properties": [{ "id": "color", "value": { "fixedColor": "purple", "mode": "fixed" } }] } + { + "matcher": { + "id": "byName", + "options": "IDE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CLI" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cloud Agent" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + } ] }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 28 }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 28 + }, "id": 203, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "multi", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "pluginVersion": "11.0.0", "targets": [ @@ -629,19 +1117,37 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT DATE(DATE_SUB(usage_date, INTERVAL WEEKDAY(usage_date) DAY)) AS time,\n SUM(agent_requests) AS \"Agent\",\n SUM(chat_requests) AS \"Chat\",\n SUM(composer_requests) AS \"Composer\"\nFROM _tool_cursor_daily_usage\nWHERE $__timeFilter(usage_date)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY 1\nORDER BY 1", + "rawSql": "SELECT DATE(event_time) AS time,\n CASE\n WHEN is_headless = 1 THEN 'Cloud Agent'\n WHEN LOWER(TRIM(COALESCE(hosting_type, ''))) IN ('cli', 'cursor-cli') THEN 'CLI'\n WHEN LOWER(TRIM(COALESCE(hosting_type, ''))) IN ('cloud', 'cloud_agent', 'cloud-agent') THEN 'Cloud Agent'\n WHEN TRIM(COALESCE(hosting_type, '')) = '' THEN 'IDE'\n ELSE 'Other'\n END AS metric,\n COUNT(*) AS value\nFROM _tool_cursor_usage_events\nWHERE $__timeFilter(event_time)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY 1, 2\nORDER BY 1", "refId": "A" } ], - "title": "Feature Adoption (Weekly)", - "type": "timeseries" + "title": "Activity by Channel", + "type": "timeseries", + "transformations": [ + { + "id": "prepareTimeSeries", + "options": { + "format": "many" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^value (.+)$", + "renamePattern": "$1" + } + } + ] }, { "datasource": "mysql", "description": "Daily sum of AI-suggested lines accepted into the editor (acceptedLinesAdded from /teams/daily-usage-data)", "fieldConfig": { "defaults": { - "color": { "fixedColor": "blue", "mode": "fixed" }, + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, "custom": { "drawStyle": "line", "fillOpacity": 20, @@ -654,11 +1160,23 @@ }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, "id": 204, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "single", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, "pluginVersion": "11.0.0", "targets": [ @@ -675,10 +1193,13 @@ }, { "datasource": "mysql", - "description": "Share of all lines added that came from accepted AI suggestions (accepted_lines_added / total_lines_added)", + "description": "Share of all lines added that came from accepted AI suggestions (accepted_lines_added / total_lines_added). Editor activity only \u2014 not committed-code AI share.", "fieldConfig": { "defaults": { - "color": { "fixedColor": "blue", "mode": "fixed" }, + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, "custom": { "drawStyle": "line", "fillOpacity": 10, @@ -691,11 +1212,23 @@ }, "overrides": [] }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 36 }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, "id": 205, "options": { - "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "mode": "single", "sort": "none" } + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, "pluginVersion": "11.0.0", "targets": [ @@ -703,7 +1236,7 @@ "datasource": "mysql", "format": "table", "rawQuery": true, - "rawSql": "SELECT usage_date AS time,\n ROUND(SUM(accepted_lines_added) * 100.0 / NULLIF(SUM(total_lines_added), 0), 1) AS \"AI Line Acceptance Rate\"\nFROM _tool_cursor_daily_usage\nWHERE $__timeFilter(usage_date)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_active = 1\nGROUP BY usage_date\nORDER BY 1", + "rawSql": "SELECT usage_date AS time,\n SUM(accepted_lines_added) * 100.0 / NULLIF(SUM(total_lines_added), 0) AS \"AI Line Acceptance Rate\"\nFROM _tool_cursor_daily_usage\nWHERE $__timeFilter(usage_date)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND is_active = 1\nGROUP BY usage_date\nORDER BY 1", "refId": "A" } ], @@ -712,7 +1245,12 @@ }, { "collapsed": true, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 44 }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, "id": 104, "title": "Diagnostics", "type": "row", @@ -720,12 +1258,22 @@ { "datasource": "mysql", "description": "Row counts and latest timestamps per _tool_cursor_* table for pipeline health monitoring", - "fieldConfig": { "defaults": {}, "overrides": [] }, - "gridPos": { "h": 6, "w": 12, "x": 0, "y": 53 }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 45 + }, "id": 40, "options": { "cellHeight": "sm", - "footer": { "show": false }, + "footer": { + "show": false + }, "showHeader": true }, "pluginVersion": "11.0.0", @@ -746,11 +1294,19 @@ ], "refresh": "", "schemaVersion": 38, - "tags": ["cursor", "devlake", "ai"], + "tags": [ + "cursor", + "devlake", + "ai" + ], "templating": { "list": [ { - "current": { "selected": false, "text": "1", "value": "1" }, + "current": { + "selected": false, + "text": "1", + "value": "1" + }, "datasource": "mysql", "definition": "SELECT DISTINCT connection_id FROM _tool_cursor_scopes ORDER BY connection_id", "hide": 0, @@ -765,7 +1321,11 @@ "type": "query" }, { - "current": { "selected": false, "text": "team", "value": "team" }, + "current": { + "selected": false, + "text": "team", + "value": "team" + }, "datasource": "mysql", "definition": "SELECT id AS scope_id FROM _tool_cursor_scopes WHERE connection_id = ${connection_id} ORDER BY id", "hide": 0, @@ -781,11 +1341,14 @@ } ] }, - "time": { "from": "now-90d", "to": "now" }, + "time": { + "from": "now-90d", + "to": "now" + }, "timepicker": {}, "timezone": "utc", "title": "Cursor Usage & Cost", "uid": "cursor_usage", - "version": 4, + "version": 14, "weekStart": "" } From 7e72519a834956787e70f2120630bfc1b0c929de Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Tue, 28 Jul 2026 09:34:30 -0600 Subject: [PATCH 09/13] build(plugins): cap concurrent CGO plugin builds * Limit default parallelism to four with DEVLAKE_PLUGIN_PARALLELISM override to reduce OOM on small hosts. * Add Cursor cost_reconciliation.sql queries and remove obsolete README rak_tools reference. Signed-off-by: Joshua Smith --- backend/plugins/cursor/README.md | 2 - .../plugins/cursor/cost_reconciliation.sql | 41 +++++++++++++++++++ backend/scripts/build-plugins.sh | 22 +++++++--- 3 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 backend/plugins/cursor/cost_reconciliation.sql diff --git a/backend/plugins/cursor/README.md b/backend/plugins/cursor/README.md index 8ca44da79f4..38a12307c82 100644 --- a/backend/plugins/cursor/README.md +++ b/backend/plugins/cursor/README.md @@ -135,8 +135,6 @@ Grafana dashboard JSON lives under `grafana/dashboards/mysql/`: | AI Cost Efficiency (Cursor panels) | `ai-cost-efficiency.json` | — | | Multi-AI Comparison (Cursor panels) | `multi-ai-comparison.json` | — | -See `grafana/dashboards/mysql/cursor-usage.json` for the dashboard definition and [rak_tools/grafana.md](../../../rak_tools/grafana.md) for currency formatting and Grafana version notes. - ## Error handling | Symptom | Likely cause | diff --git a/backend/plugins/cursor/cost_reconciliation.sql b/backend/plugins/cursor/cost_reconciliation.sql new file mode 100644 index 00000000000..6ee0588dced --- /dev/null +++ b/backend/plugins/cursor/cost_reconciliation.sql @@ -0,0 +1,41 @@ +-- Cursor cost reconciliation queries +-- Run against the lake database after a full billing-cycle event backfill. +-- +-- Note: SUM(charged_cents) from events only reconciles with /teams/spend when +-- ALL usage events for the billing cycle are collected (paginate filtered-usage-events). + +-- Team-level rollup comparison +SELECT + ROUND(SUM(spend_cents) / 100, 2) AS spend_usd, + ROUND(SUM(included_spend_cents) / 100, 2) AS included_usd, + ROUND(SUM(spend_cents + included_spend_cents) / 100, 2) AS total_cycle_usd +FROM _tool_cursor_user_spend; + +SELECT + ROUND(SUM(charged_cents) / 100, 2) AS event_charged_usd, + ROUND(SUM(total_cents) / 100, 2) AS event_model_usd, + COUNT(*) AS event_count, + MIN(event_time) AS earliest_event, + MAX(event_time) AS latest_event +FROM _tool_cursor_usage_events; + +-- Per-user: billing cycle spend vs event charged (since billing cycle start) +SELECT + s.email, + ROUND(s.spend_cents / 100, 2) AS spend_usd, + ROUND(s.included_spend_cents / 100, 2) AS included_usd, + ROUND((s.spend_cents + s.included_spend_cents) / 100, 2) AS total_cycle_usd, + ROUND(COALESCE(SUM(e.charged_cents), 0) / 100, 2) AS event_charged_usd, + COUNT(e.event_id) AS event_count +FROM _tool_cursor_user_spend s +LEFT JOIN _tool_cursor_usage_events e + ON s.email = e.user_email + AND e.event_time >= s.billing_cycle_start +GROUP BY s.email, s.spend_cents, s.included_spend_cents +ORDER BY total_cycle_usd DESC; + +-- Users with events but no spend row (should be empty) +SELECT DISTINCT e.user_email +FROM _tool_cursor_usage_events e +LEFT JOIN _tool_cursor_user_spend s ON e.user_email = s.email +WHERE s.email IS NULL; diff --git a/backend/scripts/build-plugins.sh b/backend/scripts/build-plugins.sh index e0ef94bb9b7..3619e722653 100755 --- a/backend/scripts/build-plugins.sh +++ b/backend/scripts/build-plugins.sh @@ -65,6 +65,22 @@ fi PIDS="" +# Cap concurrent CGO plugin builds. Unbounded nproc OOMs Docker Desktop / small VMs. +# Override with DEVLAKE_PLUGIN_PARALLELISM=N (default: min(4, nproc)). +if [ -n "$DEVLAKE_PLUGIN_PARALLELISM" ]; then + PARALLELISM="$DEVLAKE_PLUGIN_PARALLELISM" +else + PARALLELISM=4 + if command -v nproc >/dev/null 2>&1; then + PARALLELISM=$(nproc) + elif command -v sysctl >/dev/null 2>&1; then + PARALLELISM=$(sysctl -n hw.ncpu) + fi + if [ "$PARALLELISM" -gt 4 ]; then + PARALLELISM=4 + fi +fi +echo "Plugin build parallelism: $PARALLELISM" for PLUG in $PLUGINS; do NAME=$(basename $PLUG) echo "Building plugin $NAME to bin/plugins/$NAME/$NAME.so with args: $* --gcflags="$GCFLAGS"" @@ -72,12 +88,6 @@ for PLUG in $PLUGINS; do PIDS="$PIDS $!" # avoid too many processes causing signal killed COUNT=$(echo "$PIDS" | wc -w) - PARALLELISM=4 - if command -v nproc >/dev/null 2>&1; then - PARALLELISM=$(nproc) - elif command -v sysctl >/dev/null 2>&1; then - PARALLELISM=$(sysctl -n hw.ncpu) - fi if [ "$COUNT" -ge "$PARALLELISM" ]; then for PID in $PIDS; do wait $PID From 3491f2c79ad46e234715bc5099e856d7730efa1a Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Thu, 30 Jul 2026 15:08:39 -0600 Subject: [PATCH 10/13] fix(cursor): heal stalled usage extracts * Rewind incremental collect by seven days and repair unpromoted raw rows into tool tables * Log collection windows, extract gaps, full sync, and skipped raw rows for pipeline triage Signed-off-by: Joshua Smith --- backend/plugins/cursor/README.md | 6 +- backend/plugins/cursor/e2e/extractors_test.go | 39 +++ .../plugins/cursor/tasks/collector_utils.go | 62 +++- .../cursor/tasks/collector_utils_test.go | 94 ++++++ .../cursor/tasks/daily_usage_collector.go | 7 +- .../cursor/tasks/daily_usage_extractor.go | 18 +- .../plugins/cursor/tasks/extract_stateful.go | 311 ++++++++++++++++++ .../cursor/tasks/extract_stateful_test.go | 118 +++++++ backend/plugins/cursor/tasks/extract_utils.go | 2 +- .../cursor/tasks/usage_events_collector.go | 7 +- .../cursor/tasks/usage_events_extractor.go | 13 +- 11 files changed, 662 insertions(+), 15 deletions(-) create mode 100644 backend/plugins/cursor/tasks/extract_stateful.go create mode 100644 backend/plugins/cursor/tasks/extract_stateful_test.go diff --git a/backend/plugins/cursor/README.md b/backend/plugins/cursor/README.md index 38a12307c82..30e43b8468f 100644 --- a/backend/plugins/cursor/README.md +++ b/backend/plugins/cursor/README.md @@ -120,9 +120,10 @@ Run the blueprint on a daily schedule to keep usage and cost data current. ### Collection behavior - **Initial backfill**: usage events and daily usage collect up to **90 days** of history on the first run. -- **Incremental runs**: subsequent runs use the pipeline sync policy (`TimeAfter`) to collect only new data. +- **Incremental runs**: subsequent runs use the pipeline sync policy / collector state (`LatestSuccessStart`) and **rewind 7 calendar days** so recently-missed or partial days are re-fetched (same idea as the gh-copilot report lookback). - **Date range chunking**: both `/teams/daily-usage-data` and `/teams/filtered-usage-events` requests are split into **30-day** chunks (API limit for daily usage; applied to usage events for resilience). -- **Extract**: `extractUsageEvents` and `extractDailyUsage` use **StatefulApiExtractor** (incremental by default). A config version bump triggers a one-time full re-extract after upgrade. +- **Extract**: `extractUsageEvents` and `extractDailyUsage` use a cursor-local stateful extractor (incremental by default). Incremental runs also promote any raw rows with `id > MAX(_raw_data_id)` already in the tool table, so collected-but-unpromoted raw data is healed without a full refresh. A config version bump (`extractorVersion`) triggers a one-time full re-extract after upgrade. +- **Dashboards**: query `_tool_cursor_*` tables only. Legacy domain tables (`cursor_usage`, `cursor_team_events`) are unrelated and must not be used. - **Rate limiting**: collectors honor `Retry-After` response headers and respect the configured `rateLimitPerHour`. ## Dashboards @@ -143,6 +144,7 @@ Grafana dashboard JSON lives under `grafana/dashboards/mysql/`: | **403 Forbidden** on spend or usage events | Key lacks permission for that Admin API endpoint | | **429 Too Many Requests** | Rate limit exceeded — lower `rateLimitPerHour` or wait for `Retry-After` | | Empty usage events after successful run | Selected time range has no billable events, or sync policy excludes the date range | +| Tool max date stuck while raw advances | Deploy plugin with lookback + extract repair, then full-refresh the blueprint; verify `MAX(_raw_data_id)` in `_tool_cursor_*` catches up to `MAX(id)` in `_raw_cursor_*` | | Reconciliation delta on dashboard | Expected when comparing event-level charges to billing-cycle spend snapshots; see dashboard notes | Tokens are sanitized before persisting. Connection test results include a `permissions` object showing which Admin API endpoints succeeded. diff --git a/backend/plugins/cursor/e2e/extractors_test.go b/backend/plugins/cursor/e2e/extractors_test.go index 1dc234f164f..3f08dfca361 100644 --- a/backend/plugins/cursor/e2e/extractors_test.go +++ b/backend/plugins/cursor/e2e/extractors_test.go @@ -90,4 +90,43 @@ func TestCursorExtractorsDataFlow(t *testing.T) { CSVRelPath: "./snapshot_tables/_tool_cursor_daily_usage.csv", IgnoreTypes: []interface{}{common.NoPKModel{}}, }) + + // Non-degradation: appending newer raw rows and re-extracting must never + // shrink the tool table or move MAX(usage_date) backward. + var beforeCount int64 + if err := dataflowTester.Db.Model(&models.CursorDailyUsage{}).Count(&beforeCount).Error; err != nil { + t.Fatalf("count before: %v", err) + } + var beforeMax time.Time + if err := dataflowTester.Db.Model(&models.CursorDailyUsage{}).Select("MAX(usage_date)").Scan(&beforeMax).Error; err != nil { + t.Fatalf("max before: %v", err) + } + + extraRaw := map[string]interface{}{ + "params": `{"ConnectionId":1,"ScopeId":"team","Endpoint":"https://api.cursor.com"}`, + "data": []byte(`{"day":"2026-07-09","userId":"user_example123","email":"user@example.com","isActive":true,"completions":1,"premiumRequests":0,"agentRequests":1,"chatRequests":0,"composerRequests":0,"totalTabsAccepted":0,"totalTabsShown":0,"usageBasedReqs":0,"subscriptionIncludedReqs":1,"mostUsedModel":"composer-2.5-fast","clientVersion":"0.50.3","totalLinesAdded":0,"totalLinesDeleted":0,"acceptedLinesAdded":0,"acceptedLinesDeleted":0,"totalApplies":0,"totalAccepts":0,"totalRejects":0}`), + "url": "https://api.cursor.com/teams/daily-usage-data", + "input": nil, + "created_at": time.Now().UTC(), + } + if err := dataflowTester.Db.Table("_raw_cursor_daily_usage").Create(extraRaw).Error; err != nil { + t.Fatalf("insert extra raw: %v", err) + } + + dataflowTester.Subtask(tasks.ExtractDailyUsageMeta, taskData) + + var afterCount int64 + if err := dataflowTester.Db.Model(&models.CursorDailyUsage{}).Count(&afterCount).Error; err != nil { + t.Fatalf("count after: %v", err) + } + var afterMax time.Time + if err := dataflowTester.Db.Model(&models.CursorDailyUsage{}).Select("MAX(usage_date)").Scan(&afterMax).Error; err != nil { + t.Fatalf("max after: %v", err) + } + if afterCount < beforeCount { + t.Fatalf("tool row count decreased: before=%d after=%d", beforeCount, afterCount) + } + if afterMax.Before(beforeMax) { + t.Fatalf("MAX(usage_date) moved backward: before=%v after=%v", beforeMax, afterMax) + } } diff --git a/backend/plugins/cursor/tasks/collector_utils.go b/backend/plugins/cursor/tasks/collector_utils.go index 612ba5ec774..4247c0466d4 100644 --- a/backend/plugins/cursor/tasks/collector_utils.go +++ b/backend/plugins/cursor/tasks/collector_utils.go @@ -29,6 +29,7 @@ import ( "time" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/plugins/cursor/models" ) @@ -40,6 +41,9 @@ const ( cursorApiPageSize = 100 cursorInitialBackfillDays = 90 + // cursorLookbackDays rewinds incremental collection so recent calendar days are + // re-fetched (partial API pages / lag). Mirrors gh-copilot reportLookbackDays. + cursorLookbackDays = 7 // cursorDailyUsageMaxDays is the maximum span allowed by POST /teams/daily-usage-data. cursorDailyUsageMaxDays = 30 ) @@ -86,15 +90,69 @@ type cursorPagination struct { TotalPages int `json:"totalPages"` } -func computeUsageTimeRangeMs(since *time.Time, now time.Time) (int64, int64) { +// computeUsageTimeRangeMs returns the [start, end] epoch-ms window for usage +// collectors. Full sync uses up to cursorInitialBackfillDays. Incremental runs +// start at `since` (typically LatestSuccessStart) then rewind by +// cursorLookbackDays so recently-missed or partial calendar days are retried. +func computeUsageTimeRangeMs(since *time.Time, now time.Time, isIncremental bool) (int64, int64) { end := now.UTC() - start := end.AddDate(0, 0, -cursorInitialBackfillDays) + minStart := end.AddDate(0, 0, -cursorInitialBackfillDays) + start := minStart if since != nil && !since.IsZero() && since.After(start) { start = since.UTC() } + if isIncremental { + lookback := end.AddDate(0, 0, -cursorLookbackDays) + if start.After(lookback) { + start = lookback + } + if start.Before(minStart) { + start = minStart + } + } return start.UnixMilli(), end.UnixMilli() } +// usageCollectionWindow is the computed API request window for usage collectors. +type usageCollectionWindow struct { + StartMs int64 + EndMs int64 + ChunkCount int +} + +// usageCollectionWindowFor computes the collection window and 30-day chunk count. +func usageCollectionWindowFor(since *time.Time, now time.Time, isIncremental bool) usageCollectionWindow { + startMs, endMs := computeUsageTimeRangeMs(since, now, isIncremental) + chunks := splitDailyUsageTimeRangeMs(startMs, endMs, cursorDailyUsageMaxDays) + return usageCollectionWindow{ + StartMs: startMs, + EndMs: endMs, + ChunkCount: len(chunks), + } +} + +// logUsageCollectionWindow emits one Info line describing the collection window. +func logUsageCollectionWindow(logger log.Logger, endpoint string, since *time.Time, isIncremental bool) { + if logger == nil { + return + } + window := usageCollectionWindowFor(since, time.Now().UTC(), isIncremental) + sinceStr := "nil" + if since != nil { + sinceStr = since.UTC().Format(time.RFC3339) + } + logger.Info( + "cursor usage collection window: endpoint=%s incremental=%v since=%s start=%s end=%s lookbackDays=%d chunkCount=%d", + endpoint, + isIncremental, + sinceStr, + time.UnixMilli(window.StartMs).UTC().Format(time.RFC3339), + time.UnixMilli(window.EndMs).UTC().Format(time.RFC3339), + cursorLookbackDays, + window.ChunkCount, + ) +} + // splitDailyUsageTimeRangeMs splits [startMs, endMs] into chunks of at most maxDays. // Used by daily-usage-data (30-day API limit) and filtered-usage-events collectors. func splitDailyUsageTimeRangeMs(startMs, endMs int64, maxDays int) []cursorTimeRangeInput { diff --git a/backend/plugins/cursor/tasks/collector_utils_test.go b/backend/plugins/cursor/tasks/collector_utils_test.go index 7bb9f65d0a6..b02617f66cb 100644 --- a/backend/plugins/cursor/tasks/collector_utils_test.go +++ b/backend/plugins/cursor/tasks/collector_utils_test.go @@ -105,3 +105,97 @@ func TestDailyUsagePostBodyUsesEpochMilliseconds(t *testing.T) { t.Fatalf("endDate should be epoch ms int64, got %#v", body["endDate"]) } } + +func TestComputeUsageTimeRangeMsFullSyncUsesBackfill(t *testing.T) { + now := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC) + startMs, endMs := computeUsageTimeRangeMs(nil, now, false) + start := time.UnixMilli(startMs).UTC() + end := time.UnixMilli(endMs).UTC() + if !end.Equal(now) { + t.Fatalf("end mismatch: got %v want %v", end, now) + } + wantStart := now.AddDate(0, 0, -cursorInitialBackfillDays) + if !start.Equal(wantStart) { + t.Fatalf("full sync start mismatch: got %v want %v", start, wantStart) + } +} + +func TestComputeUsageTimeRangeMsIncrementalLookbackBuffer(t *testing.T) { + // since is yesterday: without lookback we'd only request ~1 day. + // With the buffer we look back cursorLookbackDays. + now := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC) + since := time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC) + startMs, endMs := computeUsageTimeRangeMs(&since, now, true) + start := time.UnixMilli(startMs).UTC() + end := time.UnixMilli(endMs).UTC() + if !end.Equal(now) { + t.Fatalf("end mismatch: got %v want %v", end, now) + } + wantStart := now.AddDate(0, 0, -cursorLookbackDays) + if !start.Equal(wantStart) { + t.Fatalf("incremental lookback start mismatch: got %v want %v", start, wantStart) + } +} + +func TestComputeUsageTimeRangeMsIncrementalDoesNotExpandPastBackfill(t *testing.T) { + now := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC) + // since far in the past → start clamped to 90-day min; lookback does not apply. + since := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + startMs, _ := computeUsageTimeRangeMs(&since, now, true) + start := time.UnixMilli(startMs).UTC() + wantStart := now.AddDate(0, 0, -cursorInitialBackfillDays) + if !start.Equal(wantStart) { + t.Fatalf("start should stay at 90-day min: got %v want %v", start, wantStart) + } +} + +func TestComputeUsageTimeRangeMsFullSyncUsesSinceWithoutLookback(t *testing.T) { + now := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC) + since := time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC) + startMs, _ := computeUsageTimeRangeMs(&since, now, false) + start := time.UnixMilli(startMs).UTC() + if !start.Equal(since) { + t.Fatalf("full sync should use since without lookback: got %v want %v", start, since) + } +} + +func TestUsageCollectionWindowForIncrementalLookback(t *testing.T) { + now := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC) + since := time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC) + window := usageCollectionWindowFor(&since, now, true) + wantStart := now.AddDate(0, 0, -cursorLookbackDays).UnixMilli() + if window.StartMs != wantStart { + t.Fatalf("start mismatch: got %d want %d", window.StartMs, wantStart) + } + if window.EndMs != now.UnixMilli() { + t.Fatalf("end mismatch: got %d want %d", window.EndMs, now.UnixMilli()) + } + if window.ChunkCount != 1 { + t.Fatalf("expected 1 chunk for 7-day lookback, got %d", window.ChunkCount) + } +} + +func TestUsageCollectionWindowForFullSyncBackfill(t *testing.T) { + now := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC) + window := usageCollectionWindowFor(nil, now, false) + wantStart := now.AddDate(0, 0, -cursorInitialBackfillDays).UnixMilli() + if window.StartMs != wantStart { + t.Fatalf("start mismatch: got %d want %d", window.StartMs, wantStart) + } + // Exact 90-day span fits in 3×30-day chunks. + if window.ChunkCount != 3 { + t.Fatalf("expected 3 chunks for 90-day backfill, got %d", window.ChunkCount) + } +} + +func TestUsageCollectionWindowForFullSyncUsesSinceWithoutLookback(t *testing.T) { + now := time.Date(2026, 7, 30, 0, 0, 0, 0, time.UTC) + since := time.Date(2026, 7, 29, 0, 0, 0, 0, time.UTC) + window := usageCollectionWindowFor(&since, now, false) + if window.StartMs != since.UnixMilli() { + t.Fatalf("full sync should use since without lookback: got %d want %d", window.StartMs, since.UnixMilli()) + } + if window.ChunkCount != 1 { + t.Fatalf("expected 1 chunk, got %d", window.ChunkCount) + } +} diff --git a/backend/plugins/cursor/tasks/daily_usage_collector.go b/backend/plugins/cursor/tasks/daily_usage_collector.go index 61a73b68314..1b7d57f0196 100644 --- a/backend/plugins/cursor/tasks/daily_usage_collector.go +++ b/backend/plugins/cursor/tasks/daily_usage_collector.go @@ -26,8 +26,8 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) -func newDailyUsageDateRangeIterator(since *time.Time) *helper.QueueIterator { - startMs, endMs := computeUsageTimeRangeMs(since, time.Now().UTC()) +func newDailyUsageDateRangeIterator(since *time.Time, isIncremental bool) *helper.QueueIterator { + startMs, endMs := computeUsageTimeRangeMs(since, time.Now().UTC(), isIncremental) iter := helper.NewQueueIterator() for _, chunk := range splitDailyUsageTimeRangeMs(startMs, endMs, cursorDailyUsageMaxDays) { iter.Push(chunk) @@ -75,7 +75,7 @@ func CollectDailyUsage(taskCtx plugin.SubTaskContext) errors.Error { err = collector.InitCollector(helper.ApiCollectorArgs{ ApiClient: apiClient, - Input: newDailyUsageDateRangeIterator(collector.GetSince()), + Input: newDailyUsageDateRangeIterator(collector.GetSince(), collector.IsIncremental()), Method: http.MethodPost, UrlTemplate: "teams/daily-usage-data", PageSize: cursorApiPageSize, @@ -87,5 +87,6 @@ func CollectDailyUsage(taskCtx plugin.SubTaskContext) errors.Error { return err } + logUsageCollectionWindow(taskCtx.GetLogger(), "teams/daily-usage-data", collector.GetSince(), collector.IsIncremental()) return collector.Execute() } diff --git a/backend/plugins/cursor/tasks/daily_usage_extractor.go b/backend/plugins/cursor/tasks/daily_usage_extractor.go index 1a493f01d64..21a4349ae03 100644 --- a/backend/plugins/cursor/tasks/daily_usage_extractor.go +++ b/backend/plugins/cursor/tasks/daily_usage_extractor.go @@ -58,20 +58,34 @@ func ExtractDailyUsage(taskCtx plugin.SubTaskContext) errors.Error { if !ok { return errors.Default.New("task data is not CursorTaskData") } + logger := taskCtx.GetLogger() - extractor, err := helper.NewStatefulApiExtractor(&helper.StatefulApiExtractorArgs[dailyUsageRecord]{ + extractor, err := newCursorStatefulExtractor(&cursorStatefulExtractorArgs[dailyUsageRecord]{ SubtaskCommonArgs: cursorSubtaskCommonArgs(taskCtx, data, rawDailyUsageTable), - Extract: func(record *dailyUsageRecord, _ *helper.RawData) ([]any, errors.Error) { + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + ToolTable: models.CursorDailyUsage{}.TableName(), + Extract: func(record *dailyUsageRecord, row *helper.RawData) ([]any, errors.Error) { userId := strings.TrimSpace(record.UserId) if userId == "" { userId = strings.TrimSpace(record.Email) } if userId == "" { + rawID := uint64(0) + if row != nil { + rawID = row.ID + } + logger.Warn(nil, "skipping daily usage raw row id=%d: missing userId and email", rawID) return nil, nil } usageDate, parseErr := time.Parse("2006-01-02", strings.TrimSpace(record.Day)) if parseErr != nil { + rawID := uint64(0) + if row != nil { + rawID = row.ID + } + logger.Warn(nil, "invalid day in daily usage raw row id=%d: %v", rawID, parseErr) return nil, errors.BadInput.Wrap(parseErr, "invalid day format in daily usage record") } diff --git a/backend/plugins/cursor/tasks/extract_stateful.go b/backend/plugins/cursor/tasks/extract_stateful.go new file mode 100644 index 00000000000..57b3616c9a7 --- /dev/null +++ b/backend/plugins/cursor/tasks/extract_stateful.go @@ -0,0 +1,311 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// cursorExtractRepairGapWarnThreshold is the unpromoted raw-id gap above which +// extract logs a Warn (small gaps are normal right after collect in the same pipeline). +const cursorExtractRepairGapWarnThreshold = 100 + +// cursorStatefulExtractorArgs mirrors helper.StatefulApiExtractorArgs with an +// extra tool-table repair path for unpromoted raw rows. +type cursorStatefulExtractorArgs[InputType any] struct { + *helper.SubtaskCommonArgs + Extract func(body *InputType, row *helper.RawData) ([]any, errors.Error) + ConnectionId uint64 + ScopeId string + ToolTable string +} + +type cursorStatefulExtractor[InputType any] struct { + *cursorStatefulExtractorArgs[InputType] + *helper.SubtaskStateManager +} + +func newCursorStatefulExtractor[InputType any](args *cursorStatefulExtractorArgs[InputType]) (*cursorStatefulExtractor[InputType], errors.Error) { + stateManager, err := helper.NewSubtaskStateManager(args.SubtaskCommonArgs) + if err != nil { + return nil, err + } + return &cursorStatefulExtractor[InputType]{ + cursorStatefulExtractorArgs: args, + SubtaskStateManager: stateManager, + }, nil +} + +// maxPromotedRawId returns COALESCE(MAX(_raw_data_id), 0) for the tool table +// scoped to connection/scope. Used to repair raw rows that were collected but +// never promoted to the tool layer. +func maxPromotedRawId(db dal.Dal, toolTable string, connectionId uint64, scopeId string) (uint64, errors.Error) { + if toolTable == "" || !db.HasTable(toolTable) { + return 0, nil + } + var maxIds []uint64 + err := db.Pluck( + "COALESCE(MAX(_raw_data_id), 0)", + &maxIds, + dal.From(toolTable), + dal.Where("connection_id = ? AND scope_id = ?", connectionId, scopeId), + ) + if err != nil { + return 0, errors.Default.Wrap(err, "failed to query max promoted raw id") + } + if len(maxIds) == 0 { + return 0, nil + } + return maxIds[0], nil +} + +// maxRawId returns COALESCE(MAX(id), 0) for raw rows matching params. +func maxRawId(db dal.Dal, rawTable, params string) (uint64, errors.Error) { + if rawTable == "" || !db.HasTable(rawTable) { + return 0, nil + } + var maxIds []uint64 + err := db.Pluck( + "COALESCE(MAX(id), 0)", + &maxIds, + dal.From(rawTable), + dal.Where("params = ?", params), + ) + if err != nil { + return 0, errors.Default.Wrap(err, "failed to query max raw id") + } + if len(maxIds) == 0 { + return 0, nil + } + return maxIds[0], nil +} + +// extractRepairGap returns how far raw MAX(id) is ahead of the tool's MAX(_raw_data_id). +func extractRepairGap(maxRaw, maxPromoted uint64) uint64 { + if maxRaw <= maxPromoted { + return 0 + } + return maxRaw - maxPromoted +} + +// countIncrementalRepairBreakdown returns counts for repair-only backlog rows +// (id > maxPromoted AND created_at < since) and normal since-window rows. +func countIncrementalRepairBreakdown(db dal.Dal, rawTable, params string, since *time.Time, until *time.Time, maxPromoted uint64) (repairOnlyRows, sinceRows int64, err errors.Error) { + base := []dal.Clause{ + dal.From(rawTable), + dal.Where("params = ?", params), + } + if until != nil { + base = append(base, dal.Where("created_at < ?", *until)) + } + + if since != nil { + sinceClauses := append([]dal.Clause{}, base...) + sinceClauses = append(sinceClauses, dal.Where("created_at >= ?", *since)) + sinceRows, err = db.Count(sinceClauses...) + if err != nil { + return 0, 0, errors.Default.Wrap(err, "failed to count since-window raw rows") + } + + repairClauses := append([]dal.Clause{}, base...) + repairClauses = append(repairClauses, dal.Where("id > ? AND created_at < ?", maxPromoted, *since)) + repairOnlyRows, err = db.Count(repairClauses...) + if err != nil { + return 0, 0, errors.Default.Wrap(err, "failed to count repair-only raw rows") + } + return repairOnlyRows, sinceRows, nil + } + + // No since: repair path is id > maxPromoted; sinceRows is 0. + repairClauses := append([]dal.Clause{}, base...) + repairClauses = append(repairClauses, dal.Where("id > ?", maxPromoted)) + repairOnlyRows, err = db.Count(repairClauses...) + if err != nil { + return 0, 0, errors.Default.Wrap(err, "failed to count repair-only raw rows") + } + return repairOnlyRows, 0, nil +} + +// incrementalRawWhere builds the incremental raw-row filter: +// created_at >= since OR id > maxPromotedRawId (repair gap). +// Exported for unit tests via the package-level function below. +func incrementalRawWhere(since *time.Time, maxPromoted uint64) (clause string, args []any) { + if since != nil { + return "(created_at >= ? OR id > ?)", []any{*since, maxPromoted} + } + return "id > ?", []any{maxPromoted} +} + +func setCursorRawDataOrigin(result any, table, params string, id uint64) { + if origin, ok := result.(common.GetRawDataOrigin); ok { + o := origin.GetRawDataOrigin() + o.RawDataTable = table + o.RawDataParams = params + o.RawDataId = id + } +} + +func (extractor *cursorStatefulExtractor[InputType]) Execute() errors.Error { + db := extractor.GetDal() + logger := extractor.GetLogger() + table := extractor.GetRawDataTable() + params := extractor.GetRawDataParams() + if !db.HasTable(table) { + return nil + } + + if !extractor.IsIncremental() { + logger.Info( + "cursor extract full sync: tool=%s raw=%s params=%s (first run, blueprint full refresh, or subtask config change)", + extractor.ToolTable, + table, + params, + ) + } + + clauses := []dal.Clause{ + dal.Select("id"), + dal.From(table), + dal.Where("params = ?", params), + dal.Orderby("id ASC"), + } + + var maxPromoted uint64 + if extractor.IsIncremental() { + var err errors.Error + maxPromoted, err = maxPromotedRawId(db, extractor.ToolTable, extractor.ConnectionId, extractor.ScopeId) + if err != nil { + return err + } + whereSQL, whereArgs := incrementalRawWhere(extractor.GetSince(), maxPromoted) + clauses = append(clauses, dal.Where(whereSQL, whereArgs...)) + } + clauses = append(clauses, dal.Where("created_at < ? ", extractor.GetUntil())) + + count, err := db.Count(clauses...) + if err != nil { + return errors.Default.Wrap(err, "error getting count of records") + } + logger.Info("get data from %s where params=%s and got %d with clauses %+v", table, params, count, clauses) + + if extractor.IsIncremental() { + rawMax, rawErr := maxRawId(db, table, params) + if rawErr != nil { + return rawErr + } + gap := extractRepairGap(rawMax, maxPromoted) + repairOnlyRows, sinceRows, breakdownErr := countIncrementalRepairBreakdown( + db, table, params, extractor.GetSince(), extractor.GetUntil(), maxPromoted, + ) + if breakdownErr != nil { + return breakdownErr + } + sinceStr := "nil" + if since := extractor.GetSince(); since != nil { + sinceStr = since.UTC().Format(time.RFC3339) + } + logger.Info( + "cursor incremental extract: tool=%s maxPromotedRawId=%d maxRawId=%d gap=%d since=%s rowsToProcess=%d repairOnlyRows=%d sinceRows=%d", + extractor.ToolTable, + maxPromoted, + rawMax, + gap, + sinceStr, + count, + repairOnlyRows, + sinceRows, + ) + if gap >= cursorExtractRepairGapWarnThreshold { + logger.Warn( + nil, + "cursor extract repair gap: tool=%s maxPromoted=%d maxRawId=%d unpromoted=%d (threshold=%d)", + extractor.ToolTable, + maxPromoted, + rawMax, + gap, + cursorExtractRepairGapWarnThreshold, + ) + } + } + + var ids []uint64 + err = db.Pluck("id", &ids, clauses...) + if err != nil { + return errors.Default.Wrap(err, "error getting IDs") + } + + divider := helper.NewBatchSaveDivider(extractor.SubTaskContext, extractor.GetBatchSize(), table, params) + divider.SetIncrementalMode(extractor.IsIncremental()) + + extractor.SetProgress(0, -1) + ctx := extractor.GetContext() + + for _, id := range ids { + select { + case <-ctx.Done(): + return errors.Convert(ctx.Err()) + default: + } + + row := &helper.RawData{} + err := db.First(row, dal.From(table), dal.Where("id = ?", id)) + if err != nil { + return errors.Default.Wrap(err, "error loading full row by ID") + } + + body := new(InputType) + err = errors.Convert(json.Unmarshal(row.Data, body)) + if err != nil { + return err + } + + results, err := extractor.Extract(body, row) + if err != nil { + return errors.Default.Wrap(err, "error calling plugin Extract implementation") + } + + for _, result := range results { + batch, err := divider.ForType(reflect.TypeOf(result)) + if err != nil { + return errors.Default.Wrap(err, "error getting batch from result") + } + setCursorRawDataOrigin(result, table, params, row.ID) + err = batch.Add(result) + if err != nil { + return errors.Default.Wrap(err, "error adding result to batch") + } + } + extractor.IncProgress(1) + } + + err = divider.Close() + if err != nil { + return err + } + return extractor.SubtaskStateManager.Close() +} + +var _ plugin.SubTask = (*cursorStatefulExtractor[any])(nil) diff --git a/backend/plugins/cursor/tasks/extract_stateful_test.go b/backend/plugins/cursor/tasks/extract_stateful_test.go new file mode 100644 index 00000000000..cf0ec1185ec --- /dev/null +++ b/backend/plugins/cursor/tasks/extract_stateful_test.go @@ -0,0 +1,118 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" +) + +func TestIncrementalRawWhereWithSince(t *testing.T) { + since := time.Date(2026, 7, 29, 0, 6, 3, 0, time.UTC) + clause, args := incrementalRawWhere(&since, 18002) + want := "(created_at >= ? OR id > ?)" + if clause != want { + t.Fatalf("clause mismatch:\n got: %s\nwant: %s", clause, want) + } + if len(args) != 2 { + t.Fatalf("expected 2 args, got %d", len(args)) + } + gotSince, ok := args[0].(time.Time) + if !ok || !gotSince.Equal(since) { + t.Fatalf("since arg mismatch: %#v", args[0]) + } + gotMax, ok := args[1].(uint64) + if !ok || gotMax != 18002 { + t.Fatalf("maxPromoted arg mismatch: %#v", args[1]) + } +} + +func TestIncrementalRawWhereWithoutSince(t *testing.T) { + clause, args := incrementalRawWhere(nil, 100) + want := "id > ?" + if clause != want { + t.Fatalf("clause mismatch:\n got: %s\nwant: %s", clause, want) + } + if len(args) != 1 { + t.Fatalf("expected 1 arg, got %d", len(args)) + } + gotMax, ok := args[0].(uint64) + if !ok || gotMax != 100 { + t.Fatalf("maxPromoted arg mismatch: %#v", args[0]) + } +} + +func TestIncrementalRawWhereRepairsUnpromotedGap(t *testing.T) { + // Simulates production: tool capped at raw id 18002 while raw has 18003+. + // A since filter alone would miss older created_at rows; the OR id > max + // clause must include them. + since := time.Date(2026, 7, 29, 0, 6, 3, 0, time.UTC) + clause, args := incrementalRawWhere(&since, 18002) + if clause != "(created_at >= ? OR id > ?)" { + t.Fatalf("unexpected clause: %s", clause) + } + maxPromoted := args[1].(uint64) + unpromotedID := uint64(18003) + if !(unpromotedID > maxPromoted) { + t.Fatalf("unpromoted id %d should be greater than maxPromoted %d", unpromotedID, maxPromoted) + } +} + +func TestExtractRepairGap(t *testing.T) { + cases := []struct { + name string + maxRaw uint64 + maxPromoted uint64 + want uint64 + }{ + {name: "equal", maxRaw: 100, maxPromoted: 100, want: 0}, + {name: "raw behind", maxRaw: 50, maxPromoted: 100, want: 0}, + {name: "small gap", maxRaw: 105, maxPromoted: 100, want: 5}, + {name: "large gap", maxRaw: 18102, maxPromoted: 18002, want: 100}, + {name: "empty tool", maxRaw: 500, maxPromoted: 0, want: 500}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := extractRepairGap(tc.maxRaw, tc.maxPromoted) + if got != tc.want { + t.Fatalf("extractRepairGap(%d, %d)=%d want %d", tc.maxRaw, tc.maxPromoted, got, tc.want) + } + }) + } +} + +func TestExtractRepairGapWarnThreshold(t *testing.T) { + cases := []struct { + name string + gap uint64 + warn bool + }{ + {name: "below", gap: 99, warn: false}, + {name: "at", gap: cursorExtractRepairGapWarnThreshold, warn: true}, + {name: "above", gap: 101, warn: true}, + {name: "zero", gap: 0, warn: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + shouldWarn := tc.gap >= cursorExtractRepairGapWarnThreshold + if shouldWarn != tc.warn { + t.Fatalf("gap=%d warn=%v want %v", tc.gap, shouldWarn, tc.warn) + } + }) + } +} diff --git a/backend/plugins/cursor/tasks/extract_utils.go b/backend/plugins/cursor/tasks/extract_utils.go index c755b526960..315ff3f8c8d 100644 --- a/backend/plugins/cursor/tasks/extract_utils.go +++ b/backend/plugins/cursor/tasks/extract_utils.go @@ -24,7 +24,7 @@ import ( // cursorExtractorVersion bumps when extract logic changes in a way that requires // a one-time full re-extract (SubtaskStateManager compares SubtaskConfig). -const cursorExtractorVersion = 2 +const cursorExtractorVersion = 3 func cursorSubtaskCommonArgs(taskCtx plugin.SubTaskContext, data *CursorTaskData, rawTable string) *helper.SubtaskCommonArgs { return &helper.SubtaskCommonArgs{ diff --git a/backend/plugins/cursor/tasks/usage_events_collector.go b/backend/plugins/cursor/tasks/usage_events_collector.go index fe5a04e68b9..dc4b3ff9f9e 100644 --- a/backend/plugins/cursor/tasks/usage_events_collector.go +++ b/backend/plugins/cursor/tasks/usage_events_collector.go @@ -26,8 +26,8 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) -func newUsageEventsDateRangeIterator(since *time.Time) *helper.QueueIterator { - startMs, endMs := computeUsageTimeRangeMs(since, time.Now().UTC()) +func newUsageEventsDateRangeIterator(since *time.Time, isIncremental bool) *helper.QueueIterator { + startMs, endMs := computeUsageTimeRangeMs(since, time.Now().UTC(), isIncremental) iter := helper.NewQueueIterator() for _, chunk := range splitDailyUsageTimeRangeMs(startMs, endMs, cursorDailyUsageMaxDays) { iter.Push(chunk) @@ -92,7 +92,7 @@ func CollectUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { err = collector.InitCollector(helper.ApiCollectorArgs{ ApiClient: apiClient, - Input: newUsageEventsDateRangeIterator(collector.GetSince()), + Input: newUsageEventsDateRangeIterator(collector.GetSince(), collector.IsIncremental()), Method: http.MethodPost, UrlTemplate: "teams/filtered-usage-events", PageSize: cursorApiPageSize, @@ -104,5 +104,6 @@ func CollectUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { return err } + logUsageCollectionWindow(taskCtx.GetLogger(), "teams/filtered-usage-events", collector.GetSince(), collector.IsIncremental()) return collector.Execute() } diff --git a/backend/plugins/cursor/tasks/usage_events_extractor.go b/backend/plugins/cursor/tasks/usage_events_extractor.go index 25ee1071741..f88b0dd76cc 100644 --- a/backend/plugins/cursor/tasks/usage_events_extractor.go +++ b/backend/plugins/cursor/tasks/usage_events_extractor.go @@ -58,10 +58,14 @@ func ExtractUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { if !ok { return errors.Default.New("task data is not CursorTaskData") } + logger := taskCtx.GetLogger() - extractor, err := helper.NewStatefulApiExtractor(&helper.StatefulApiExtractorArgs[usageEventRecord]{ + extractor, err := newCursorStatefulExtractor(&cursorStatefulExtractorArgs[usageEventRecord]{ SubtaskCommonArgs: cursorSubtaskCommonArgs(taskCtx, data, rawUsageEventsTable), - Extract: func(record *usageEventRecord, _ *helper.RawData) ([]any, errors.Error) { + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + ToolTable: models.CursorUsageEvent{}.TableName(), + Extract: func(record *usageEventRecord, row *helper.RawData) ([]any, errors.Error) { eventTime, err := parseEventTimestampMs(record.Timestamp) if err != nil { return nil, err @@ -69,6 +73,11 @@ func ExtractUsageEvents(taskCtx plugin.SubTaskContext) errors.Error { userEmail := strings.TrimSpace(record.UserEmail) if userEmail == "" { + rawID := uint64(0) + if row != nil { + rawID = row.ID + } + logger.Warn(nil, "skipping usage event raw row id=%d: missing userEmail", rawID) return nil, nil } From c54387fad3b2842558f19786b2516106d60d68b4 Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Thu, 6 Aug 2026 10:23:19 -0600 Subject: [PATCH 11/13] style(cursor): fix golangci lint issues * Apply gofmt alignment fixes across cursor models, service, and extractors * Remove unused spend response helpers flagged by the unused linter Signed-off-by: Joshua Smith --- backend/plugins/cursor/e2e/extractors_test.go | 4 +-- backend/plugins/cursor/models/daily_usage.go | 8 +++--- backend/plugins/cursor/models/user_spend.go | 26 ++++++++--------- .../cursor/service/connection_test_helper.go | 4 +-- .../plugins/cursor/tasks/collector_utils.go | 28 ------------------- .../plugins/cursor/tasks/members_extractor.go | 4 +-- .../cursor/tasks/user_spend_extractor.go | 4 +-- 7 files changed, 25 insertions(+), 53 deletions(-) diff --git a/backend/plugins/cursor/e2e/extractors_test.go b/backend/plugins/cursor/e2e/extractors_test.go index 3f08dfca361..aecdcc0a462 100644 --- a/backend/plugins/cursor/e2e/extractors_test.go +++ b/backend/plugins/cursor/e2e/extractors_test.go @@ -103,8 +103,8 @@ func TestCursorExtractorsDataFlow(t *testing.T) { } extraRaw := map[string]interface{}{ - "params": `{"ConnectionId":1,"ScopeId":"team","Endpoint":"https://api.cursor.com"}`, - "data": []byte(`{"day":"2026-07-09","userId":"user_example123","email":"user@example.com","isActive":true,"completions":1,"premiumRequests":0,"agentRequests":1,"chatRequests":0,"composerRequests":0,"totalTabsAccepted":0,"totalTabsShown":0,"usageBasedReqs":0,"subscriptionIncludedReqs":1,"mostUsedModel":"composer-2.5-fast","clientVersion":"0.50.3","totalLinesAdded":0,"totalLinesDeleted":0,"acceptedLinesAdded":0,"acceptedLinesDeleted":0,"totalApplies":0,"totalAccepts":0,"totalRejects":0}`), + "params": `{"ConnectionId":1,"ScopeId":"team","Endpoint":"https://api.cursor.com"}`, + "data": []byte(`{"day":"2026-07-09","userId":"user_example123","email":"user@example.com","isActive":true,"completions":1,"premiumRequests":0,"agentRequests":1,"chatRequests":0,"composerRequests":0,"totalTabsAccepted":0,"totalTabsShown":0,"usageBasedReqs":0,"subscriptionIncludedReqs":1,"mostUsedModel":"composer-2.5-fast","clientVersion":"0.50.3","totalLinesAdded":0,"totalLinesDeleted":0,"acceptedLinesAdded":0,"acceptedLinesDeleted":0,"totalApplies":0,"totalAccepts":0,"totalRejects":0}`), "url": "https://api.cursor.com/teams/daily-usage-data", "input": nil, "created_at": time.Now().UTC(), diff --git a/backend/plugins/cursor/models/daily_usage.go b/backend/plugins/cursor/models/daily_usage.go index d193e374896..6974e9bc876 100644 --- a/backend/plugins/cursor/models/daily_usage.go +++ b/backend/plugins/cursor/models/daily_usage.go @@ -27,10 +27,10 @@ import ( // Line fields map from acceptedLinesAdded/totalLinesAdded (etc.); LinesAdded/LinesDeleted // duplicate accepted_lines_added/accepted_lines_deleted for backward compatibility. type CursorDailyUsage struct { - ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` - ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` - UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` - UsageDate time.Time `gorm:"primaryKey" json:"usageDate"` + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` + UsageDate time.Time `gorm:"primaryKey" json:"usageDate"` Email string `json:"email" gorm:"type:varchar(255);index"` IsActive bool `json:"isActive"` diff --git a/backend/plugins/cursor/models/user_spend.go b/backend/plugins/cursor/models/user_spend.go index c936159482d..e948fda0d48 100644 --- a/backend/plugins/cursor/models/user_spend.go +++ b/backend/plugins/cursor/models/user_spend.go @@ -25,19 +25,19 @@ import ( // CursorUserSpend stores per-user billing cycle spend from /teams/spend. type CursorUserSpend struct { - ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` - ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` - UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` - BillingCycleStart time.Time `gorm:"primaryKey" json:"billingCycleStart"` - CollectedAt time.Time `gorm:"index" json:"collectedAt"` - - Email string `json:"email" gorm:"type:varchar(255);index"` - Name string `json:"name" gorm:"type:varchar(255)"` - Role string `json:"role" gorm:"type:varchar(50)"` - SpendCents float64 `json:"spendCents"` - IncludedSpendCents float64 `json:"includedSpendCents"` - FastPremiumRequests int `json:"fastPremiumRequests"` - MonthlyLimitDollars float64 `json:"monthlyLimitDollars"` + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` + BillingCycleStart time.Time `gorm:"primaryKey" json:"billingCycleStart"` + CollectedAt time.Time `gorm:"index" json:"collectedAt"` + + Email string `json:"email" gorm:"type:varchar(255);index"` + Name string `json:"name" gorm:"type:varchar(255)"` + Role string `json:"role" gorm:"type:varchar(50)"` + SpendCents float64 `json:"spendCents"` + IncludedSpendCents float64 `json:"includedSpendCents"` + FastPremiumRequests int `json:"fastPremiumRequests"` + MonthlyLimitDollars float64 `json:"monthlyLimitDollars"` HardLimitOverrideDollars float64 `json:"hardLimitOverrideDollars"` common.NoPKModel diff --git a/backend/plugins/cursor/service/connection_test_helper.go b/backend/plugins/cursor/service/connection_test_helper.go index 3ae190865b2..b7baa044beb 100644 --- a/backend/plugins/cursor/service/connection_test_helper.go +++ b/backend/plugins/cursor/service/connection_test_helper.go @@ -116,8 +116,8 @@ func TestConnection(ctx stdctx.Context, br corectx.BasicRes, connection *models. hasEnterpriseAnalytics := probeEnterpriseAnalytics(apiClient) return &TestConnectionResult{ - Success: true, - Message: "Team Admin API key validated. Members, spend, and usage events are accessible.", + Success: true, + Message: "Team Admin API key validated. Members, spend, and usage events are accessible.", MemberCount: memberCount, Permissions: permissions, HasEnterpriseAnalytics: hasEnterpriseAnalytics, diff --git a/backend/plugins/cursor/tasks/collector_utils.go b/backend/plugins/cursor/tasks/collector_utils.go index 4247c0466d4..21c4cad06c1 100644 --- a/backend/plugins/cursor/tasks/collector_utils.go +++ b/backend/plugins/cursor/tasks/collector_utils.go @@ -195,20 +195,6 @@ func parseUsageEventsResponse(res *http.Response) ([]json.RawMessage, errors.Err return response.UsageEvents, nil } -func parseSpendMembersResponse(res *http.Response) ([]json.RawMessage, errors.Error) { - body, err := readResponseBody(res) - if err != nil { - return nil, err - } - var response struct { - TeamMemberSpend []json.RawMessage `json:"teamMemberSpend"` - } - if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { - return nil, errors.Default.Wrap(errors.Convert(jsonErr), "failed to decode spend response") - } - return response.TeamMemberSpend, nil -} - func parseDailyUsageResponse(res *http.Response) ([]json.RawMessage, errors.Error) { body, err := readResponseBody(res) if err != nil { @@ -237,20 +223,6 @@ func parseMembersResponse(res *http.Response) ([]json.RawMessage, errors.Error) return response.TeamMembers, nil } -func parseSpendMeta(res *http.Response) (int64, errors.Error) { - body, err := readResponseBody(res) - if err != nil { - return 0, err - } - var response struct { - SubscriptionCycleStart int64 `json:"subscriptionCycleStart"` - } - if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { - return 0, errors.Default.Wrap(errors.Convert(jsonErr), "failed to decode spend metadata") - } - return response.SubscriptionCycleStart, nil -} - func parsePaginationBody(body []byte) (cursorPagination, errors.Error) { var response struct { Pagination cursorPagination `json:"pagination"` diff --git a/backend/plugins/cursor/tasks/members_extractor.go b/backend/plugins/cursor/tasks/members_extractor.go index e541b2f3af1..35558bdd30a 100644 --- a/backend/plugins/cursor/tasks/members_extractor.go +++ b/backend/plugins/cursor/tasks/members_extractor.go @@ -44,8 +44,8 @@ func ExtractMembers(taskCtx plugin.SubTaskContext) errors.Error { extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Table: rawMembersTable, + Ctx: taskCtx, + Table: rawMembersTable, Options: rawParamsFromTaskData(data), }, Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { diff --git a/backend/plugins/cursor/tasks/user_spend_extractor.go b/backend/plugins/cursor/tasks/user_spend_extractor.go index 8f705ada864..2526c4e8b9b 100644 --- a/backend/plugins/cursor/tasks/user_spend_extractor.go +++ b/backend/plugins/cursor/tasks/user_spend_extractor.go @@ -48,8 +48,8 @@ func ExtractUserSpend(taskCtx plugin.SubTaskContext) errors.Error { extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - Table: rawUserSpendTable, + Ctx: taskCtx, + Table: rawUserSpendTable, Options: rawParamsFromTaskData(data), }, Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { From b9c764c188e1c7a42c956dc60de75fecd6cf5c8e Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Mon, 24 Aug 2026 14:23:03 -0600 Subject: [PATCH 12/13] test(e2e): Fix cursor plugin CI failures * Set explicit raw row id in cursor extractor e2e so PostgreSQL does not hit duplicate primary key errors. * Register cursor plugin in schema_e2e allGoPlugins so TestAllGoPluginsListed passes. Signed-off-by: Joshua Smith --- backend/plugins/cursor/e2e/extractors_test.go | 1 + backend/plugins/schema_e2e/migration_schema_test.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/backend/plugins/cursor/e2e/extractors_test.go b/backend/plugins/cursor/e2e/extractors_test.go index aecdcc0a462..8ebe4e8c08d 100644 --- a/backend/plugins/cursor/e2e/extractors_test.go +++ b/backend/plugins/cursor/e2e/extractors_test.go @@ -103,6 +103,7 @@ func TestCursorExtractorsDataFlow(t *testing.T) { } extraRaw := map[string]interface{}{ + "id": uint64(2), "params": `{"ConnectionId":1,"ScopeId":"team","Endpoint":"https://api.cursor.com"}`, "data": []byte(`{"day":"2026-07-09","userId":"user_example123","email":"user@example.com","isActive":true,"completions":1,"premiumRequests":0,"agentRequests":1,"chatRequests":0,"composerRequests":0,"totalTabsAccepted":0,"totalTabsShown":0,"usageBasedReqs":0,"subscriptionIncludedReqs":1,"mostUsedModel":"composer-2.5-fast","clientVersion":"0.50.3","totalLinesAdded":0,"totalLinesDeleted":0,"acceptedLinesAdded":0,"acceptedLinesDeleted":0,"totalApplies":0,"totalAccepts":0,"totalRejects":0}`), "url": "https://api.cursor.com/teams/daily-usage-data", diff --git a/backend/plugins/schema_e2e/migration_schema_test.go b/backend/plugins/schema_e2e/migration_schema_test.go index c51788aaa06..c03d8640043 100644 --- a/backend/plugins/schema_e2e/migration_schema_test.go +++ b/backend/plugins/schema_e2e/migration_schema_test.go @@ -55,6 +55,7 @@ import ( circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" clickup "github.com/apache/incubator-devlake/plugins/clickup/impl" + cursor "github.com/apache/incubator-devlake/plugins/cursor/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" feishu "github.com/apache/incubator-devlake/plugins/feishu/impl" @@ -105,6 +106,7 @@ func allGoPlugins() []plugin.PluginMeta { circleci.Circleci{}, claudeCode.ClaudeCode{}, clickup.ClickUp{}, + cursor.Cursor{}, customize.Customize{}, dora.Dora{}, feishu.Feishu{}, From 85065787a1445f7df08b87e74438b05a4a26aa0b Mon Sep 17 00:00:00 2001 From: Joshua Smith Date: Mon, 31 Aug 2026 18:59:59 +0000 Subject: [PATCH 13/13] fix(cursor): support Postgres migration separate from mysql * Use ALTER COLUMN TYPE on Postgres instead of MySQL-only MODIFY COLUMN syntax. * Unblocks TestMigrationSchemaMatchesModels in the e2e-postgres CI job. Signed-off-by: Joshua Smith --- .../migrationscripts/20260710_requests_costs_float.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/plugins/cursor/models/migrationscripts/20260710_requests_costs_float.go b/backend/plugins/cursor/models/migrationscripts/20260710_requests_costs_float.go index 2b4acf9cb4e..a50bd4acd20 100644 --- a/backend/plugins/cursor/models/migrationscripts/20260710_requests_costs_float.go +++ b/backend/plugins/cursor/models/migrationscripts/20260710_requests_costs_float.go @@ -29,7 +29,14 @@ func (script *changeCursorRequestsCostsToFloat) Up(basicRes context.BasicRes) er if !db.HasTable("_tool_cursor_usage_events") { return nil } - return db.Exec("ALTER TABLE _tool_cursor_usage_events MODIFY COLUMN requests_costs DOUBLE") + switch db.Dialect() { + case "postgres": + return db.Exec("ALTER TABLE _tool_cursor_usage_events ALTER COLUMN requests_costs TYPE double precision") + case "mysql": + return db.Exec("ALTER TABLE _tool_cursor_usage_events MODIFY COLUMN requests_costs DOUBLE") + default: + return nil + } } func (*changeCursorRequestsCostsToFloat) Version() uint64 { return 20260710120000 }