Add organization option to back up public repos of an org/group

README.md

@@ -47,6 +47,13 @@ github:
     # (default: [])
     repositories:
       - ChappIO/git-backup
+    # (optional) Backup all public repositories of these
+    # organizations or users, even if you are not a member.
+    # Will be merged with the other discovery mechanisms
+    # and it's result filtered with excluded repos.
+    # (default: [])
+    organization:
+      - golang
     # (optional) Exclude this list of repos
     # or whole organizations/users
     exclude:
@@ -85,6 +92,13 @@ gitlab:
     repositories:
       - gnuwget/wget2
       - gitlab-org/api/client-go
+    # (optional) Backup all public projects of these groups
+    # (including subgroups) or users, even if you are not a member.
+    # Will be merged with the other discovery mechanisms
+    # and it's result filtered with excluded repos.
+    # (default: [])
+    organization:
+      - gnuwget
     # (optional) Exclude this list of repos
     # or whole organizations/users
     exclude:

github.go

@@ -23,6 +23,7 @@ type GithubConfig struct {
 	Owned        *bool    `yaml:"owned,omitempty"`
 	Exclude      []string `yaml:"exclude,omitempty"`
 	Repositories []string `yaml:"repositories,omitempty"`
+	Organization []string `yaml:"organization,omitempty"`
 	client       *github.Client
 }
 
@@ -116,6 +117,21 @@ func (c *GithubConfig) getAllRepos() ([]*github.Repository, error) {
 	// use discovery mechanisms
 	var err error
 
+	for _, org := range c.Organization {
+		for repos, response, apiErr := c.getOrgRepos(org, 1); true; repos, response, apiErr = c.getOrgRepos(org, response.NextPage) {
+			if apiErr != nil {
+				return nil, apiErr
+			}
+			for _, repo := range repos {
+				all[*repo.FullName] = repo
+			}
+
+			if len(repos) == 0 || response.NextPage == 0 {
+				break
+			}
+		}
+	}
+
 	for repos, response, apiErr := c.discoverRepos(1); true; repos, response, apiErr = c.discoverRepos(response.NextPage) {
 		if apiErr != nil {
 			err = apiErr
@@ -175,6 +191,17 @@ func (c *GithubConfig) getRepoList(repos []string) ([]*github.Repository, error)
 	return ghRepos, nil
 }
 
+func (c *GithubConfig) getOrgRepos(org string, page int) ([]*github.Repository, *github.Response, error) {
+	// /users/{name}/repos works for both organizations and user accounts, /orgs/{name}/repos only for organizations
+	return c.client.Repositories.List(context.Background(), org, &github.RepositoryListOptions{
+		Type: "owner",
+		ListOptions: github.ListOptions{
+			Page:    page,
+			PerPage: 100,
+		},
+	})
+}
+
 func (c *GithubConfig) discoverRepos(page int) ([]*github.Repository, *github.Response, error) {
 	affiliations := make([]string, 0)
 

gitlab.go

@@ -1,6 +1,8 @@
 package git_backup
 
 import (
+	"errors"
+	"fmt"
 	"log"
 	"maps"
 	"net/url"
@@ -18,6 +20,7 @@ type GitLabConfig struct {
 	Owned        *bool    `yaml:"owned,omitempty"`
 	Exclude      []string `yaml:"exclude,omitempty"`
 	Repositories []string `yaml:"repositories,omitempty"`
+	Organization []string `yaml:"organization,omitempty"`
 	client       *gitlab.Client
 }
 
@@ -101,6 +104,21 @@ func (g *GitLabConfig) getAllRepos() ([]*gitlab.Project, error) {
 	}
 
 	// use discovery mechanisms
+	for _, group := range g.Organization {
+		repos, err := g.getGroupRepos(group)
+		// groups and users live in separate API namespaces, a user name 404s as a group
+		if errors.Is(err, gitlab.ErrNotFound) {
+			repos, err = g.getUserRepos(group)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("organization %q: %w", group, err)
+		} else {
+			for _, repo := range repos {
+				all[repo.PathWithNamespace] = repo
+			}
+		}
+	}
+
 	if *g.Starred {
 		if repos, err := g.discoverRepos(&gitlab.ListProjectsOptions{Starred: boolPointer(true)}); err != nil {
 			return nil, err
@@ -149,6 +167,57 @@ func (g *GitLabConfig) getRepoList(repos []string) ([]*gitlab.Project, error) {
 
 }
 
+func (g *GitLabConfig) getGroupRepos(group string) ([]*gitlab.Project, error) {
+	opts := &gitlab.ListGroupProjectsOptions{
+		IncludeSubGroups: boolPointer(true),
+		Simple:           boolPointer(true),
+		Visibility:       gitlab.Ptr(gitlab.PublicVisibility),
+	}
+	opts.ListOptions.PerPage = 100
+
+	repos := make([]*gitlab.Project, 0, 100)
+
+	// paginate
+	for i := 1; true; i++ {
+		opts.ListOptions.Page = i
+		projects, _, err := g.client.Groups.ListGroupProjects(group, opts)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(projects) == 0 {
+			break
+		}
+		repos = append(repos, projects...)
+	}
+	return repos, nil
+}
+
+func (g *GitLabConfig) getUserRepos(user string) ([]*gitlab.Project, error) {
+	opts := &gitlab.ListProjectsOptions{
+		Simple:     boolPointer(true),
+		Visibility: gitlab.Ptr(gitlab.PublicVisibility),
+	}
+	opts.ListOptions.PerPage = 100
+
+	repos := make([]*gitlab.Project, 0, 100)
+
+	// paginate
+	for i := 1; true; i++ {
+		opts.ListOptions.Page = i
+		projects, _, err := g.client.Projects.ListUserProjects(user, opts)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(projects) == 0 {
+			break
+		}
+		repos = append(repos, projects...)
+	}
+	return repos, nil
+}
+
 func (g *GitLabConfig) discoverRepos(opts *gitlab.ListProjectsOptions) ([]*gitlab.Project, error) {
 
 	opts.ListOptions.PerPage = 100