From 4c363c192e5167ba602585b6b3914daee61d8a40 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <192242476+ShivamSharma43@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:03:07 +0530 Subject: [PATCH] fix(asana): page through stories, subtasks and tags and stop on null next_page The story, subtask and tag collectors sent limit=100 without a PageSize or GetNextPageCustomData, so only the first 100 stories, subtasks and tags of each task were collected. The task and user collectors returned nil, nil when next_page was null, which made the collector request page 1 again without an offset whenever the last page was full. Add getNextAsanaPageOffset, which ends the collection when next_page is null, and setAsanaPageQuery, which sets limit and offset, and use them in all five collectors. Closes #9365 Generated-by: Claude Code (Claude Opus 5.5) Co-Authored-By: Claude Opus 5.5 Signed-off-by: Shivam Sharma <192242476+ShivamSharma43@users.noreply.github.com> --- backend/plugins/asana/tasks/shared.go | 54 +++++++++++++++ backend/plugins/asana/tasks/shared_test.go | 65 +++++++++++++++++++ .../plugins/asana/tasks/story_collector.go | 4 +- .../plugins/asana/tasks/subtask_collector.go | 4 +- backend/plugins/asana/tasks/tag_collector.go | 4 +- backend/plugins/asana/tasks/task_collector.go | 21 +----- backend/plugins/asana/tasks/user_collector.go | 21 +----- 7 files changed, 134 insertions(+), 39 deletions(-) create mode 100644 backend/plugins/asana/tasks/shared.go create mode 100644 backend/plugins/asana/tasks/shared_test.go diff --git a/backend/plugins/asana/tasks/shared.go b/backend/plugins/asana/tasks/shared.go new file mode 100644 index 00000000000..3d70c35b16b --- /dev/null +++ b/backend/plugins/asana/tasks/shared.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "net/http" + "net/url" + + "github.com/apache/devlake/core/errors" + "github.com/apache/devlake/helpers/pluginhelper/api" +) + +// asanaPageSize is the largest page Asana list endpoints return. +const asanaPageSize = 100 + +// setAsanaPageQuery sets the page size and, after the first page, the offset +// that the previous page returned in next_page. +func setAsanaPageQuery(query url.Values, reqData *api.RequestData) { + query.Set("limit", fmt.Sprintf("%d", reqData.Pager.Size)) + if offset, ok := reqData.CustomData.(string); ok && offset != "" { + query.Set("offset", offset) + } +} + +// getNextAsanaPageOffset returns the offset of the next page. Asana sets +// next_page to null when there are no more pages, so the collection ends +// there: requesting a page without an offset would start again at page 1. +func getNextAsanaPageOffset(_ *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(prevPageResponse, &resp) + if err != nil { + return nil, err + } + if resp.NextPage == nil || resp.NextPage.Offset == "" { + return nil, api.ErrFinishCollect + } + return resp.NextPage.Offset, nil +} diff --git a/backend/plugins/asana/tasks/shared_test.go b/backend/plugins/asana/tasks/shared_test.go new file mode 100644 index 00000000000..f1a681e74f7 --- /dev/null +++ b/backend/plugins/asana/tasks/shared_test.go @@ -0,0 +1,65 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "io" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/apache/devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func asanaListPage(t *testing.T, body string) *http.Response { + req, err := http.NewRequest(http.MethodGet, "https://app.asana.com/api/1.0/tasks/1/stories", nil) + assert.Nil(t, err) + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(body)), + Request: req, + } +} + +func TestSetAsanaPageQuery(t *testing.T) { + first := url.Values{} + setAsanaPageQuery(first, &api.RequestData{Pager: &api.Pager{Page: 1, Size: asanaPageSize}}) + assert.Equal(t, "100", first.Get("limit")) + assert.False(t, first.Has("offset")) + + next := url.Values{} + setAsanaPageQuery(next, &api.RequestData{Pager: &api.Pager{Page: 2, Size: asanaPageSize}, CustomData: "eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9"}) + assert.Equal(t, "100", next.Get("limit")) + assert.Equal(t, "eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9", next.Get("offset")) +} + +func TestGetNextAsanaPageOffset(t *testing.T) { + offset, err := getNextAsanaPageOffset(nil, asanaListPage(t, `{ + "data": [{"gid": "1"}], + "next_page": {"offset": "eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9", "path": "/tasks/1/stories?limit=100&offset=eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9", "uri": "https://app.asana.com/api/1.0/tasks/1/stories?limit=100&offset=eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9"} + }`)) + assert.Nil(t, err) + assert.Equal(t, "eyJ0eXAiOJiKV1iQLCJhbGciOiJIUzI1NiJ9", offset) + + // A full last page has "next_page": null. Going on without an offset + // would request page 1 again, so the collection must end here. + _, err = getNextAsanaPageOffset(nil, asanaListPage(t, `{"data": [{"gid": "1"}], "next_page": null}`)) + assert.ErrorIs(t, err, api.ErrFinishCollect) +} diff --git a/backend/plugins/asana/tasks/story_collector.go b/backend/plugins/asana/tasks/story_collector.go index f689c9f6908..7801579018b 100644 --- a/backend/plugins/asana/tasks/story_collector.go +++ b/backend/plugins/asana/tasks/story_collector.go @@ -73,13 +73,15 @@ func CollectStory(taskCtx plugin.SubTaskContext) errors.Error { }, ApiClient: data.ApiClient, Input: iterator, + PageSize: asanaPageSize, UrlTemplate: "tasks/{{ .Input.Gid }}/stories", Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} query.Set("opt_fields", "gid,resource_type,resource_subtype,text,html_text,is_pinned,is_edited,sticker_name,created_at,created_by,target") - query.Set("limit", "100") + setAsanaPageQuery(query, reqData) return query, nil }, + GetNextPageCustomData: getNextAsanaPageOffset, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var resp asanaListResponse err := api.UnmarshalResponse(res, &resp) diff --git a/backend/plugins/asana/tasks/subtask_collector.go b/backend/plugins/asana/tasks/subtask_collector.go index 73218dc2d27..871ee6b0531 100644 --- a/backend/plugins/asana/tasks/subtask_collector.go +++ b/backend/plugins/asana/tasks/subtask_collector.go @@ -74,13 +74,15 @@ func CollectSubtask(taskCtx plugin.SubTaskContext) errors.Error { }, ApiClient: data.ApiClient, Input: iterator, + PageSize: asanaPageSize, UrlTemplate: "tasks/{{ .Input.Gid }}/subtasks", Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} query.Set("opt_fields", "gid,name,notes,resource_type,resource_subtype,completed,completed_at,due_on,created_at,modified_at,permalink_url,assignee,created_by,parent,num_subtasks,memberships.section,memberships.project") - query.Set("limit", "100") + setAsanaPageQuery(query, reqData) return query, nil }, + GetNextPageCustomData: getNextAsanaPageOffset, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var resp asanaListResponse err := api.UnmarshalResponse(res, &resp) diff --git a/backend/plugins/asana/tasks/tag_collector.go b/backend/plugins/asana/tasks/tag_collector.go index 01d3b18e423..a038437a9be 100644 --- a/backend/plugins/asana/tasks/tag_collector.go +++ b/backend/plugins/asana/tasks/tag_collector.go @@ -73,13 +73,15 @@ func CollectTag(taskCtx plugin.SubTaskContext) errors.Error { }, ApiClient: data.ApiClient, Input: iterator, + PageSize: asanaPageSize, UrlTemplate: "tasks/{{ .Input.Gid }}/tags", Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} query.Set("opt_fields", "gid,name,resource_type,color,notes,permalink_url") - query.Set("limit", "100") + setAsanaPageQuery(query, reqData) return query, nil }, + GetNextPageCustomData: getNextAsanaPageOffset, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var resp asanaListResponse err := api.UnmarshalResponse(res, &resp) diff --git a/backend/plugins/asana/tasks/task_collector.go b/backend/plugins/asana/tasks/task_collector.go index e21b30c90fa..8287392f2d9 100644 --- a/backend/plugins/asana/tasks/task_collector.go +++ b/backend/plugins/asana/tasks/task_collector.go @@ -61,31 +61,16 @@ func CollectTask(taskCtx plugin.SubTaskContext) errors.Error { Table: rawTaskTable, }, ApiClient: data.ApiClient, - PageSize: 100, + PageSize: asanaPageSize, UrlTemplate: "projects/{{ .Params.ProjectId }}/tasks", Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} - query.Set("limit", "100") // Request all fields needed for transformation including section name query.Set("opt_fields", "gid,name,notes,resource_type,resource_subtype,completed,completed_at,due_on,created_at,modified_at,permalink_url,assignee,assignee.name,created_by,created_by.name,parent,num_subtasks,memberships.section,memberships.section.name,memberships.project") - if reqData.CustomData != nil { - if offset, ok := reqData.CustomData.(string); ok && offset != "" { - query.Set("offset", offset) - } - } + setAsanaPageQuery(query, reqData) return query, nil }, - GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { - var resp asanaTaskListResponse - err := api.UnmarshalResponse(prevPageResponse, &resp) - if err != nil { - return nil, err - } - if resp.NextPage != nil && resp.NextPage.Offset != "" { - return resp.NextPage.Offset, nil - } - return nil, nil - }, + GetNextPageCustomData: getNextAsanaPageOffset, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var w asanaTaskListResponse err := api.UnmarshalResponse(res, &w) diff --git a/backend/plugins/asana/tasks/user_collector.go b/backend/plugins/asana/tasks/user_collector.go index bcfc9859190..65c6670285e 100644 --- a/backend/plugins/asana/tasks/user_collector.go +++ b/backend/plugins/asana/tasks/user_collector.go @@ -52,30 +52,15 @@ func CollectUser(taskCtx plugin.SubTaskContext) errors.Error { Table: rawUserTable, }, ApiClient: data.ApiClient, - PageSize: 100, + PageSize: asanaPageSize, UrlTemplate: "projects/{{ .Params.ProjectId }}/members", Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} query.Set("opt_fields", "gid,name,email,resource_type,photo.image_128x128") - query.Set("limit", "100") - if reqData.CustomData != nil { - if offset, ok := reqData.CustomData.(string); ok && offset != "" { - query.Set("offset", offset) - } - } + setAsanaPageQuery(query, reqData) return query, nil }, - GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { - var resp asanaListResponse - err := api.UnmarshalResponse(prevPageResponse, &resp) - if err != nil { - return nil, err - } - if resp.NextPage != nil && resp.NextPage.Offset != "" { - return resp.NextPage.Offset, nil - } - return nil, nil - }, + GetNextPageCustomData: getNextAsanaPageOffset, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { var resp asanaListResponse err := api.UnmarshalResponse(res, &resp)