apricote/releaser-pleaser
1
0
Fork 0
mirror of https://github.com/apricote/releaser-pleaser.git synced 2026-02-07 10:17:02 +00:00
Code Issues Projects Releases Packages Wiki Activity

refactor(github): add pagination helper (#47)

Browse source
This commit is contained in:
Julian Tölle 2024-09-07 21:36:17 +02:00 • committed by GitHub
parent af505c94c6
commit 2010ac1143
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 131 additions and 205 deletions
Download patch file Download diff file Expand all files Collapse all files

336
internal/forge/github/github.go
View file

@ -64,52 +64,44 @@ func (g *GitHub) GitAuth() transport.AuthMethod {
func (g *GitHub) LatestTags(ctx context.Context) (git.Releases, error) { func (g *GitHub) LatestTags(ctx context.Context) (git.Releases, error) {
g.log.DebugContext(ctx, "listing all tags in github repository") g.log.DebugContext(ctx, "listing all tags in github repository")
page := 1 tags, err := all(func(listOptions github.ListOptions) ([]*github.RepositoryTag, *github.Response, error) {
return g.client.Repositories.ListTags(
ctx, g.options.Owner, g.options.Repo,
&listOptions,
)
})
if err != nil {
return git.Releases{}, err
}
var releases git.Releases var releases git.Releases
for { for _, ghTag := range tags {
tags, resp, err := g.client.Repositories.ListTags( tag := &git.Tag{
ctx, g.options.Owner, g.options.Repo, Hash: ghTag.GetCommit().GetSHA(),
&github.ListOptions{Page: page, PerPage: PerPageMax}, Name: ghTag.GetName(),
) }
version, err := semver.Parse(strings.TrimPrefix(tag.Name, "v"))
if err != nil { if err != nil {
return git.Releases{}, err g.log.WarnContext(
ctx, "unable to parse tag as semver, skipping",
"tag.name", tag.Name,
"tag.hash", tag.Hash,
"error", err,
)
continue
} }
for _, ghTag := range tags { if releases.Latest == nil {
tag := &git.Tag{ releases.Latest = tag
Hash: ghTag.GetCommit().GetSHA(),
Name: ghTag.GetName(),
}
version, err := semver.Parse(strings.TrimPrefix(tag.Name, "v"))
if err != nil {
g.log.WarnContext(
ctx, "unable to parse tag as semver, skipping",
"tag.name", tag.Name,
"tag.hash", tag.Hash,
"error", err,
)
continue
}
if releases.Latest == nil {
releases.Latest = tag
}
if len(version.Pre) == 0 {
// Stable version tag
// We return once we have found the latest stable tag, not needed to look at every single tag.
releases.Stable = tag
break
}
} }
if len(version.Pre) == 0 {
if page == resp.LastPage || resp.LastPage == 0 { // Stable version tag
// We return once we have found the latest stable tag, not needed to look at every single tag.
releases.Stable = tag
break break
} }
page = resp.NextPage
} }
return releases, nil return releases, nil
@ -150,34 +142,19 @@ func (g *GitHub) commitsSinceTag(ctx context.Context, tag *git.Tag) ([]*github.R
log := g.log.With("base", tag.Hash, "head", head) log := g.log.With("base", tag.Hash, "head", head)
log.Debug("comparing commits", "base", tag.Hash, "head", head) log.Debug("comparing commits", "base", tag.Hash, "head", head)
page := 1 repositoryCommits, err := all(
func(listOptions github.ListOptions) ([]*github.RepositoryCommit, *github.Response, error) {
comparison, resp, err := g.client.Repositories.CompareCommits(
ctx, g.options.Owner, g.options.Repo,
tag.Hash, head, &listOptions)
if err != nil {
return nil, nil, err
}
var repositoryCommits []*github.RepositoryCommit return comparison.Commits, resp, err
for { })
log.Debug("fetching page", "page", page) if err != nil {
comparison, resp, err := g.client.Repositories.CompareCommits( return nil, err
ctx, g.options.Owner, g.options.Repo,
tag.Hash, head, &github.ListOptions{
Page: page,
PerPage: PerPageMax,
})
if err != nil {
return nil, err
}
if repositoryCommits == nil {
// Pre-initialize slice on first request
log.Debug("found commits", "length", comparison.GetTotalCommits())
repositoryCommits = make([]*github.RepositoryCommit, 0, comparison.GetTotalCommits())
}
repositoryCommits = append(repositoryCommits, comparison.Commits...)
if page == resp.LastPage || resp.LastPage == 0 {
break
}
page = resp.NextPage
} }
return repositoryCommits, nil return repositoryCommits, nil
@ -188,37 +165,17 @@ func (g *GitHub) commitsSinceInit(ctx context.Context) ([]*github.RepositoryComm
log := g.log.With("head", head) log := g.log.With("head", head)
log.Debug("listing all commits") log.Debug("listing all commits")
page := 1 repositoryCommits, err := all(
func(listOptions github.ListOptions) ([]*github.RepositoryCommit, *github.Response, error) {
var repositoryCommits []*github.RepositoryCommit return g.client.Repositories.ListCommits(
for { ctx, g.options.Owner, g.options.Repo,
log.Debug("fetching page", "page", page) &github.CommitsListOptions{
commits, resp, err := g.client.Repositories.ListCommits( SHA: head,
ctx, g.options.Owner, g.options.Repo, ListOptions: listOptions,
&github.CommitsListOptions{ })
SHA: head, })
ListOptions: github.ListOptions{ if err != nil {
Page: page, return nil, err
PerPage: PerPageMax,
},
})
if err != nil {
return nil, err
}
if repositoryCommits == nil && resp.LastPage > 0 {
// Pre-initialize slice on first request
log.Debug("found commits", "pages", resp.LastPage)
repositoryCommits = make([]*github.RepositoryCommit, 0, resp.LastPage*PerPageMax)
}
repositoryCommits = append(repositoryCommits, commits...)
if page == resp.LastPage || resp.LastPage == 0 {
break
}
page = resp.NextPage
} }
return repositoryCommits, nil return repositoryCommits, nil
@ -230,76 +187,50 @@ func (g *GitHub) prForCommit(ctx context.Context, commit git.Commit) (*git.PullR
// Using the "List pull requests" endpoint might be faster, as it allows us to fetch 100 arbitrary PRs per request, // Using the "List pull requests" endpoint might be faster, as it allows us to fetch 100 arbitrary PRs per request,
// but worst case we need to look up all PRs made in the repository ever. // but worst case we need to look up all PRs made in the repository ever.
log := g.log.With("commit.hash", commit.Hash) g.log.Debug("fetching pull requests associated with commit", "commit.hash", commit.Hash)
page := 1
var associatedPRs []*github.PullRequest
for { associatedPRs, err := all(
log.Debug("fetching pull requests associated with commit", "page", page) func(listOptions github.ListOptions) ([]*github.PullRequest, *github.Response, error) {
prs, resp, err := g.client.PullRequests.ListPullRequestsWithCommit( return g.client.PullRequests.ListPullRequestsWithCommit(
ctx, g.options.Owner, g.options.Repo, ctx, g.options.Owner, g.options.Repo,
commit.Hash, &github.ListOptions{ commit.Hash, &listOptions)
Page: page, })
PerPage: PerPageMax, if err != nil {
}) return nil, err
if err != nil {
return nil, err
}
associatedPRs = append(associatedPRs, prs...)
if page == resp.LastPage || resp.LastPage == 0 {
break
}
page = resp.NextPage
} }
var pullrequest *github.PullRequest var pullRequest *github.PullRequest
for _, pr := range associatedPRs { for _, pr := range associatedPRs {
// We only look for the PR that has this commit set as the "merge commit" => The result of squashing this branch onto main // We only look for the PR that has this commit set as the "merge commit" => The result of squashing this branch onto main
if pr.GetMergeCommitSHA() == commit.Hash { if pr.GetMergeCommitSHA() == commit.Hash {
pullrequest = pr pullRequest = pr
break break
} }
} }
if pullrequest == nil { if pullRequest == nil {
return nil, nil return nil, nil
} }
return gitHubPRToPullRequest(pullrequest), nil return gitHubPRToPullRequest(pullRequest), nil
} }
func (g *GitHub) EnsureLabelsExist(ctx context.Context, labels []releasepr.Label) error { func (g *GitHub) EnsureLabelsExist(ctx context.Context, labels []releasepr.Label) error {
existingLabels := make([]string, 0, len(labels)) g.log.Debug("fetching labels on repo")
ghLabels, err := all(func(listOptions github.ListOptions) ([]*github.Label, *github.Response, error) {
page := 1 return g.client.Issues.ListLabels(
for {
g.log.Debug("fetching labels on repo", "page", page)
ghLabels, resp, err := g.client.Issues.ListLabels(
ctx, g.options.Owner, g.options.Repo, ctx, g.options.Owner, g.options.Repo,
&github.ListOptions{ &listOptions)
Page: page, })
PerPage: PerPageMax, if err != nil {
}) return err
if err != nil {
return err
}
for _, label := range ghLabels {
existingLabels = append(existingLabels, label.GetName())
}
if page == resp.LastPage || resp.LastPage == 0 {
break
}
page = resp.NextPage
} }
for _, label := range labels { for _, label := range labels {
if !slices.Contains(existingLabels, label.Name) { if !slices.ContainsFunc(ghLabels, func(ghLabel *github.Label) bool {
return ghLabel.GetName() == label.Name
}) {
g.log.Info("creating label in repository", "label.name", label.Name) g.log.Info("creating label in repository", "label.name", label.Name)
_, _, err := g.client.Issues.CreateLabel( _, _, err = g.client.Issues.CreateLabel(
ctx, g.options.Owner, g.options.Repo, ctx, g.options.Owner, g.options.Repo,
&github.Label{ &github.Label{
Name: pointer.Pointer(label.Name), Name: pointer.Pointer(label.Name),
@ -317,33 +248,25 @@ func (g *GitHub) EnsureLabelsExist(ctx context.Context, labels []releasepr.Label
} }
func (g *GitHub) PullRequestForBranch(ctx context.Context, branch string) (*releasepr.ReleasePullRequest, error) { func (g *GitHub) PullRequestForBranch(ctx context.Context, branch string) (*releasepr.ReleasePullRequest, error) {
page := 1
for { prs, err := all(
prs, resp, err := g.client.PullRequests.ListPullRequestsWithCommit(ctx, g.options.Owner, g.options.Repo, branch, &github.ListOptions{ func(listOptions github.ListOptions) ([]*github.PullRequest, *github.Response, error) {
Page: page, return g.client.PullRequests.ListPullRequestsWithCommit(ctx, g.options.Owner, g.options.Repo, branch, &listOptions)
PerPage: PerPageMax,
}) })
if err != nil { if err != nil {
var ghErr *github.ErrorResponse var ghErr *github.ErrorResponse
if errors.As(err, &ghErr) { if errors.As(err, &ghErr) {
if ghErr.Message == fmt.Sprintf("No commit found for SHA: %s", branch) { if ghErr.Message == fmt.Sprintf("No commit found for SHA: %s", branch) {
return nil, nil return nil, nil
}
}
return nil, err
}
for _, pr := range prs {
if pr.GetBase().GetRef() == g.options.BaseBranch && pr.GetHead().GetRef() == branch && pr.GetState() == PRStateOpen {
return gitHubPRToReleasePullRequest(pr), nil
} }
} }
return nil, err
}
if page == resp.LastPage || resp.LastPage == 0 { for _, pr := range prs {
break if pr.GetBase().GetRef() == g.options.BaseBranch && pr.GetHead().GetRef() == branch && pr.GetState() == PRStateOpen {
return gitHubPRToReleasePullRequest(pr), nil
} }
page = resp.NextPage
} }
return nil, nil return nil, nil
@ -431,52 +354,36 @@ func (g *GitHub) ClosePullRequest(ctx context.Context, pr *releasepr.ReleasePull
} }
func (g *GitHub) PendingReleases(ctx context.Context, pendingLabel releasepr.Label) ([]*releasepr.ReleasePullRequest, error) { func (g *GitHub) PendingReleases(ctx context.Context, pendingLabel releasepr.Label) ([]*releasepr.ReleasePullRequest, error) {
page := 1 ghPRs, err := all(func(listOptions github.ListOptions) ([]*github.PullRequest, *github.Response, error) {
return g.client.PullRequests.List(
var prs []*releasepr.ReleasePullRequest
for {
ghPRs, resp, err := g.client.PullRequests.List(
ctx, g.options.Owner, g.options.Repo, ctx, g.options.Owner, g.options.Repo,
&github.PullRequestListOptions{ &github.PullRequestListOptions{
State: PRStateClosed, State: PRStateClosed,
Base: g.options.BaseBranch, Base: g.options.BaseBranch,
ListOptions: github.ListOptions{ ListOptions: listOptions,
Page: page,
PerPage: PerPageMax,
},
}) })
if err != nil { })
return nil, err if err != nil {
return nil, err
}
prs := make([]*releasepr.ReleasePullRequest, 0, len(ghPRs))
for _, pr := range ghPRs {
pending := slices.ContainsFunc(pr.Labels, func(l *github.Label) bool {
return l.GetName() == pendingLabel.Name
})
if !pending {
continue
} }
if prs == nil && resp.LastPage > 0 { // pr.Merged is always nil :(
// Pre-initialize slice on first request if pr.MergedAt == nil {
g.log.Debug("found pending releases", "pages", resp.LastPage) // Closed and not merged
prs = make([]*releasepr.ReleasePullRequest, 0, (resp.LastPage-1)*PerPageMax) continue
} }
for _, pr := range ghPRs { prs = append(prs, gitHubPRToReleasePullRequest(pr))
pending := slices.ContainsFunc(pr.Labels, func(l *github.Label) bool {
return l.GetName() == pendingLabel.Name
})
if !pending {
continue
}
// pr.Merged is always nil :(
if pr.MergedAt == nil {
// Closed and not merged
continue
}
prs = append(prs, gitHubPRToReleasePullRequest(pr))
}
if page == resp.LastPage || resp.LastPage == 0 {
break
}
page = resp.NextPage
} }
return prs, nil return prs, nil
@ -507,6 +414,25 @@ func (g *GitHub) CreateRelease(ctx context.Context, commit git.Commit, title, ch
return nil return nil
} }
func all[T any](f func(listOptions github.ListOptions) ([]T, *github.Response, error)) ([]T, error) {
results := make([]T, 0)
page := 1
for {
pageResults, resp, err := f(github.ListOptions{Page: page, PerPage: PerPageMax})
if err != nil {
return nil, err
}
results = append(results, pageResults...)
if page == resp.LastPage || resp.LastPage == 0 {
return results, nil
}
page = resp.NextPage
}
}
func gitHubPRToPullRequest(pr *github.PullRequest) *git.PullRequest { func gitHubPRToPullRequest(pr *github.PullRequest) *git.PullRequest {
return &git.PullRequest{ return &git.PullRequest{
ID: pr.GetNumber(), ID: pr.GetNumber(),