Skip to content

Commit

Permalink
Merge pull request #24 from flant/github-checks
Browse files Browse the repository at this point in the history 
add github client
  • Loading branch information
@marriva
marriva authored Nov 19, 2024
2 parents 018b48b + 4dec096 commit 99c5a47
Show file tree
Hide file tree
Showing 10 changed files with 478 additions and 33 deletions.
6 changes: 6 additions & 0 deletions cmd/projects/branches.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/flant/glaball/pkg/limiter"
"github.com/flant/glaball/pkg/sort/v2"
"github.com/flant/glaball/pkg/util"
"github.com/google/go-github/v56/github"
"github.com/hashicorp/go-hclog"
"github.com/spf13/cobra"
"github.com/xanzy/go-gitlab"
Expand Down Expand Up @@ -83,6 +84,11 @@ type ProjectBranch struct {
Branches []*gitlab.Branch `json:"branches,omitempty"`
}

type RepositoryBranch struct {
Repository *github.Repository `json:"repository,omitempty"`
Branch *github.Branch `json:"branch,omitempty"`
}

func BranchesListCmd() error {
if !sort.ValidOrderBy(branchOrderBy, ProjectBranch{}) {
branchOrderBy = append(branchOrderBy, branchDefaultField)
Expand Down
89 changes: 89 additions & 0 deletions cmd/projects/files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package projects
import (
"bufio"
"bytes"
"context"
"fmt"
"os"
"regexp"
Expand All @@ -12,6 +13,7 @@ import (
"github.com/flant/glaball/pkg/client"
"github.com/flant/glaball/pkg/limiter"
"github.com/flant/glaball/pkg/sort/v2"
"github.com/google/go-github/v58/github"
"gopkg.in/yaml.v3"

"github.com/flant/glaball/cmd/common"
Expand Down Expand Up @@ -230,6 +232,38 @@ func listProjectsFiles(h *client.Host, filepath, ref string, re []*regexp.Regexp
}
}

func listProjectsFilesFromGithub(h *client.Host, filepath, ref string, re []*regexp.Regexp, opt github.RepositoryListByOrgOptions,
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) {

defer wg.Done()

ctx := context.TODO()
wg.Lock()
list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org, &opt)
if err != nil {
if err != nil {
wg.Error(h, err)
wg.Unlock()
return
}
}
wg.Unlock()

for _, v := range list {
wg.Add(1)
// TODO: handle deadlock when no files found
go getRawFileFromGithub(h, v, filepath, ref, re, wg, data)
}

if resp.NextPage > 0 {
wg.Add(1)
opt.Page = resp.NextPage
go listProjectsFilesFromGithub(h, filepath, ref, re, opt, wg, data, options...)
}

return
}

func getRawFile(h *client.Host, project *gitlab.Project, filepath, ref string, re []*regexp.Regexp,
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) {

Expand Down Expand Up @@ -257,6 +291,46 @@ func getRawFile(h *client.Host, project *gitlab.Project, filepath, ref string, r
}
}

// TODO:
func getRawFileFromGithub(h *client.Host, repository *github.Repository, filepath, ref string, re []*regexp.Regexp,
wg *limiter.Limiter, data chan<- interface{}) {

defer wg.Done()

targetRef := ref
if ref == "" {
targetRef = repository.GetDefaultBranch()
}
// TODO:
ctx := context.TODO()
wg.Lock()
fileContent, _, resp, err := h.GithubClient.Repositories.GetContents(ctx,
repository.Owner.GetLogin(),
repository.GetName(),
filepath,
&github.RepositoryContentGetOptions{Ref: targetRef})
wg.Unlock()
if err != nil {
hclog.L().Named("files").Trace("get raw file error", "repository", repository.GetHTMLURL(), "error", err)
return
}

raw, err := fileContent.GetContent()
if err != nil {
hclog.L().Named("files").Trace("get raw file error", "repository", repository.GetHTMLURL(), "error", err)
return
}

for _, r := range re {
if r.MatchString(raw) {
data <- sort.Element{Host: h, Struct: &RepositoryFile{Repository: repository, Raw: raw}, Cached: resp.Header.Get("X-From-Cache") == "1"}
hclog.L().Named("files").Trace("search pattern was found in file", "team", h.Team, "repository", h.Project, "host", h.URL,
"repo", repository.GetHTMLURL(), "file", filepath, "pattern", r.String(), "content", hclog.Fmt("%s", raw))
return
}
}
}

func getGitlabCIFile(h *client.Host, check bool, project *gitlab.Project, re []*regexp.Regexp,
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) {

Expand Down Expand Up @@ -300,6 +374,11 @@ type ProjectFile struct {
Raw []byte
}

type RepositoryFile struct {
Repository *github.Repository `json:"repository,omitempty"`
Raw string
}

type ProjectLintResult struct {
Project *gitlab.Project `json:"project,omitempty"`
MergedYaml map[string]interface{} `json:"merged_yaml,omitempty"`
Expand Down Expand Up @@ -398,7 +477,17 @@ func ListProjectsFiles(h *client.Host, filepath, ref string, re []*regexp.Regexp
listProjectsFiles(h, filepath, ref, re, opt, wg, data, options...)
}

func ListProjectsFilesFromGithub(h *client.Host, filepath, ref string, re []*regexp.Regexp, opt github.RepositoryListByOrgOptions,
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) {
listProjectsFilesFromGithub(h, filepath, ref, re, opt, wg, data, options...)
}

func GetRawFile(h *client.Host, project *gitlab.Project, filepath, ref string, re []*regexp.Regexp,
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) {
getRawFile(h, project, filepath, ref, re, wg, data, options...)
}

func GetRawFileFromGithub(h *client.Host, repository *github.Repository, filepath, ref string, re []*regexp.Regexp,
wg *limiter.Limiter, data chan<- interface{}) {
getRawFileFromGithub(h, repository, filepath, ref, re, wg, data)
}
29 changes: 28 additions & 1 deletion cmd/projects/list.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package projects

import (
"context"
"encoding/csv"
"fmt"
"os"
Expand All @@ -12,6 +13,7 @@ import (
"github.com/flant/glaball/pkg/limiter"
"github.com/flant/glaball/pkg/sort/v2"
"github.com/flant/glaball/pkg/util"
"github.com/google/go-github/v58/github"

"github.com/flant/glaball/cmd/common"

Expand Down Expand Up @@ -345,6 +347,31 @@ func listProjects(h *client.Host, opt gitlab.ListProjectsOptions, wg *limiter.Li

defer wg.Done()

// TODO:
if h.GithubClient != nil {
ctx := context.TODO()
list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org,
&github.RepositoryListByOrgOptions{ListOptions: github.ListOptions{PerPage: 100}},
)
if err != nil {
wg.Error(h, err)
return err
}

for _, v := range list {
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
}

if resp.NextPage > 0 {
wg.Add(1)
opt.Page = resp.NextPage
go listProjects(h, opt, wg, data, options...)
}

return nil

}

wg.Lock()

list, resp, err := h.Client.Projects.ListProjects(&opt, options...)
Expand All @@ -354,7 +381,7 @@ func listProjects(h *client.Host, opt gitlab.ListProjectsOptions, wg *limiter.Li
return err
}

wg.Unlock()
wg.Unlock() // TODO: ratelimiter

for _, v := range list {
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
Expand Down
151 changes: 151 additions & 0 deletions cmd/projects/mr.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package projects

import (
"context"
"encoding/csv"
"fmt"
"os"
Expand All @@ -10,6 +11,7 @@ import (
"github.com/flant/glaball/pkg/limiter"
"github.com/flant/glaball/pkg/sort/v2"
"github.com/flant/glaball/pkg/util"
"github.com/google/go-github/v58/github"

"github.com/flant/glaball/cmd/common"

Expand Down Expand Up @@ -219,6 +221,74 @@ func ListProjectsByNamespace(h *client.Host, namespaces []string, opt gitlab.Lis
return listProjectsByNamespace(h, namespaces, opt, wg, data, options...)
}

func listRepositories(h *client.Host, archived bool, opt github.RepositoryListByOrgOptions,
wg *limiter.Limiter, data chan<- interface{}) error {
defer wg.Done()

ctx := context.TODO()
wg.Lock()
list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org, &opt)
if err != nil {
wg.Error(h, err)
wg.Unlock()
return err
}
wg.Unlock()

for _, v := range list {
if v.GetArchived() == archived {
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
}
}

if resp.NextPage > 0 {
wg.Add(1)
opt.Page = resp.NextPage
go listRepositories(h, archived, opt, wg, data)
}

return nil
}

func ListRepositories(h *client.Host, archived bool, opt github.RepositoryListByOrgOptions,
wg *limiter.Limiter, data chan<- interface{}) error {
return listRepositories(h, archived, opt, wg, data)
}

func listRepositoriesByNamespace(h *client.Host, namespaces []string, archived bool, opt github.RepositoryListByOrgOptions,
wg *limiter.Limiter, data chan<- interface{}) error {
defer wg.Done()

ctx := context.TODO()
wg.Lock()
list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org, &opt)
if err != nil {
wg.Error(h, err)
wg.Unlock()
return err
}
wg.Unlock()

for _, v := range list {
if v.GetArchived() == archived && (len(namespaces) == 0 || util.ContainsString(namespaces, v.GetName())) {
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
}
}

if resp.NextPage > 0 {
wg.Add(1)
opt.Page = resp.NextPage
go listRepositoriesByNamespace(h, namespaces, archived, opt, wg, data)
}

return nil
}

func ListRepositoriesByNamespace(h *client.Host, namespaces []string, archived bool, opt github.RepositoryListByOrgOptions,
wg *limiter.Limiter, data chan<- interface{}) error {
return listRepositoriesByNamespace(h, namespaces, archived, opt, wg, data)
}

func listMergeRequests(h *client.Host, project *gitlab.Project, opt gitlab.ListProjectMergeRequestsOptions,
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error {
defer wg.Done()
Expand Down Expand Up @@ -360,12 +430,93 @@ func listMergeRequestsByAssigneeOrAuthorID(h *client.Host, project *gitlab.Proje
return nil
}

func listPullRequestsByAssigneeOrAuthorID(h *client.Host, repository *github.Repository, IDs []int,
opt github.PullRequestListOptions, wg *limiter.Limiter, data chan<- interface{}) error {
defer wg.Done()

ctx := context.TODO()
wg.Lock()
list, resp, err := h.GithubClient.PullRequests.List(ctx, h.Org, repository.GetName(), &opt)
if err != nil {
wg.Error(h, err)
wg.Unlock()
return err
}
wg.Unlock()

for _, v := range list {
if len(IDs) == 0 {
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
continue
}

// if pr has assignee, then check and continue
if v.Assignee != nil {
if util.ContainsInt(IDs, int(v.Assignee.GetID())) {
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
}
continue
}

// otherwise check the author
if v.User != nil && util.ContainsInt(IDs, int(v.User.GetID())) {
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
}
}

if resp.NextPage > 0 {
wg.Add(1)
opt.Page = resp.NextPage
go listPullRequestsByAssigneeOrAuthorID(h, repository, IDs, opt, wg, data)
}

return nil
}

// authorIDs slice must be sorted in ascending order
func ListMergeRequestsByAuthorOrAssigneeID(h *client.Host, project *gitlab.Project, IDs []int, opt gitlab.ListProjectMergeRequestsOptions,
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error {
return listMergeRequestsByAssigneeOrAuthorID(h, project, IDs, opt, wg, data, options...)
}

// authorIDs slice must be sorted in ascending order
func ListPullRequestsByAuthorOrAssigneeID(h *client.Host, repository *github.Repository, IDs []int,
opt github.PullRequestListOptions, wg *limiter.Limiter, data chan<- interface{}) error {
return listPullRequestsByAssigneeOrAuthorID(h, repository, IDs, opt, wg, data)
}

func listPullRequests(h *client.Host, repository *github.Repository, opt github.PullRequestListOptions,
wg *limiter.Limiter, data chan<- interface{}) error {
defer wg.Done()

ctx := context.TODO()
wg.Lock()
list, resp, err := h.GithubClient.PullRequests.List(ctx, h.Org, repository.GetName(), &opt)
if err != nil {
wg.Error(h, err)
wg.Unlock()
return err
}
wg.Unlock()

for _, v := range list {
data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"}
}

if resp.NextPage > 0 {
wg.Add(1)
opt.Page = resp.NextPage
go listPullRequests(h, repository, opt, wg, data)
}

return nil
}

func ListPullRequests(h *client.Host, repository *github.Repository, opt github.PullRequestListOptions,
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error {
return listPullRequests(h, repository, opt, wg, data)
}

func listMergeRequestsSearch(h *client.Host, project *gitlab.Project, key string, value *regexp.Regexp, opt gitlab.ListProjectMergeRequestsOptions,
wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error {
defer wg.Done()
Expand Down
Loading

0 comments on commit 99c5a47

Please sign in to comment.