diff --git a/cmd/projects/branches.go b/cmd/projects/branches.go index 858a810..c2f6dcf 100644 --- a/cmd/projects/branches.go +++ b/cmd/projects/branches.go @@ -11,6 +11,7 @@ import ( "github.com/flant/glaball/pkg/limiter" "github.com/flant/glaball/pkg/sort/v2" "github.com/flant/glaball/pkg/util" + "github.com/google/go-github/v56/github" "github.com/hashicorp/go-hclog" "github.com/spf13/cobra" "github.com/xanzy/go-gitlab" @@ -83,6 +84,11 @@ type ProjectBranch struct { Branches []*gitlab.Branch `json:"branches,omitempty"` } +type RepositoryBranch struct { + Repository *github.Repository `json:"repository,omitempty"` + Branch *github.Branch `json:"branch,omitempty"` +} + func BranchesListCmd() error { if !sort.ValidOrderBy(branchOrderBy, ProjectBranch{}) { branchOrderBy = append(branchOrderBy, branchDefaultField) diff --git a/cmd/projects/files.go b/cmd/projects/files.go index f5ef284..c632bd2 100644 --- a/cmd/projects/files.go +++ b/cmd/projects/files.go @@ -3,6 +3,7 @@ package projects import ( "bufio" "bytes" + "context" "fmt" "os" "regexp" @@ -12,6 +13,7 @@ import ( "github.com/flant/glaball/pkg/client" "github.com/flant/glaball/pkg/limiter" "github.com/flant/glaball/pkg/sort/v2" + "github.com/google/go-github/v58/github" "gopkg.in/yaml.v3" "github.com/flant/glaball/cmd/common" @@ -230,6 +232,38 @@ func listProjectsFiles(h *client.Host, filepath, ref string, re []*regexp.Regexp } } +func listProjectsFilesFromGithub(h *client.Host, filepath, ref string, re []*regexp.Regexp, opt github.RepositoryListByOrgOptions, + wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) { + + defer wg.Done() + + ctx := context.TODO() + wg.Lock() + list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org, &opt) + if err != nil { + if err != nil { + wg.Error(h, err) + wg.Unlock() + return + } + } + wg.Unlock() + + for _, v := range list { + wg.Add(1) + // TODO: handle deadlock when no files found + go getRawFileFromGithub(h, v, filepath, ref, re, wg, data) + } + + if resp.NextPage > 0 { + wg.Add(1) + opt.Page = resp.NextPage + go listProjectsFilesFromGithub(h, filepath, ref, re, opt, wg, data, options...) + } + + return +} + func getRawFile(h *client.Host, project *gitlab.Project, filepath, ref string, re []*regexp.Regexp, wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) { @@ -257,6 +291,46 @@ func getRawFile(h *client.Host, project *gitlab.Project, filepath, ref string, r } } +// TODO: +func getRawFileFromGithub(h *client.Host, repository *github.Repository, filepath, ref string, re []*regexp.Regexp, + wg *limiter.Limiter, data chan<- interface{}) { + + defer wg.Done() + + targetRef := ref + if ref == "" { + targetRef = repository.GetDefaultBranch() + } + // TODO: + ctx := context.TODO() + wg.Lock() + fileContent, _, resp, err := h.GithubClient.Repositories.GetContents(ctx, + repository.Owner.GetLogin(), + repository.GetName(), + filepath, + &github.RepositoryContentGetOptions{Ref: targetRef}) + wg.Unlock() + if err != nil { + hclog.L().Named("files").Trace("get raw file error", "repository", repository.GetHTMLURL(), "error", err) + return + } + + raw, err := fileContent.GetContent() + if err != nil { + hclog.L().Named("files").Trace("get raw file error", "repository", repository.GetHTMLURL(), "error", err) + return + } + + for _, r := range re { + if r.MatchString(raw) { + data <- sort.Element{Host: h, Struct: &RepositoryFile{Repository: repository, Raw: raw}, Cached: resp.Header.Get("X-From-Cache") == "1"} + hclog.L().Named("files").Trace("search pattern was found in file", "team", h.Team, "repository", h.Project, "host", h.URL, + "repo", repository.GetHTMLURL(), "file", filepath, "pattern", r.String(), "content", hclog.Fmt("%s", raw)) + return + } + } +} + func getGitlabCIFile(h *client.Host, check bool, project *gitlab.Project, re []*regexp.Regexp, wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) { @@ -300,6 +374,11 @@ type ProjectFile struct { Raw []byte } +type RepositoryFile struct { + Repository *github.Repository `json:"repository,omitempty"` + Raw string +} + type ProjectLintResult struct { Project *gitlab.Project `json:"project,omitempty"` MergedYaml map[string]interface{} `json:"merged_yaml,omitempty"` @@ -398,7 +477,17 @@ func ListProjectsFiles(h *client.Host, filepath, ref string, re []*regexp.Regexp listProjectsFiles(h, filepath, ref, re, opt, wg, data, options...) } +func ListProjectsFilesFromGithub(h *client.Host, filepath, ref string, re []*regexp.Regexp, opt github.RepositoryListByOrgOptions, + wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) { + listProjectsFilesFromGithub(h, filepath, ref, re, opt, wg, data, options...) +} + func GetRawFile(h *client.Host, project *gitlab.Project, filepath, ref string, re []*regexp.Regexp, wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) { getRawFile(h, project, filepath, ref, re, wg, data, options...) } + +func GetRawFileFromGithub(h *client.Host, repository *github.Repository, filepath, ref string, re []*regexp.Regexp, + wg *limiter.Limiter, data chan<- interface{}) { + getRawFileFromGithub(h, repository, filepath, ref, re, wg, data) +} diff --git a/cmd/projects/list.go b/cmd/projects/list.go index d798cdc..4a7f7ca 100644 --- a/cmd/projects/list.go +++ b/cmd/projects/list.go @@ -1,6 +1,7 @@ package projects import ( + "context" "encoding/csv" "fmt" "os" @@ -12,6 +13,7 @@ import ( "github.com/flant/glaball/pkg/limiter" "github.com/flant/glaball/pkg/sort/v2" "github.com/flant/glaball/pkg/util" + "github.com/google/go-github/v58/github" "github.com/flant/glaball/cmd/common" @@ -345,6 +347,31 @@ func listProjects(h *client.Host, opt gitlab.ListProjectsOptions, wg *limiter.Li defer wg.Done() + // TODO: + if h.GithubClient != nil { + ctx := context.TODO() + list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org, + &github.RepositoryListByOrgOptions{ListOptions: github.ListOptions{PerPage: 100}}, + ) + if err != nil { + wg.Error(h, err) + return err + } + + for _, v := range list { + data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"} + } + + if resp.NextPage > 0 { + wg.Add(1) + opt.Page = resp.NextPage + go listProjects(h, opt, wg, data, options...) + } + + return nil + + } + wg.Lock() list, resp, err := h.Client.Projects.ListProjects(&opt, options...) @@ -354,7 +381,7 @@ func listProjects(h *client.Host, opt gitlab.ListProjectsOptions, wg *limiter.Li return err } - wg.Unlock() + wg.Unlock() // TODO: ratelimiter for _, v := range list { data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"} diff --git a/cmd/projects/mr.go b/cmd/projects/mr.go index c35b805..f377448 100644 --- a/cmd/projects/mr.go +++ b/cmd/projects/mr.go @@ -1,6 +1,7 @@ package projects import ( + "context" "encoding/csv" "fmt" "os" @@ -10,6 +11,7 @@ import ( "github.com/flant/glaball/pkg/limiter" "github.com/flant/glaball/pkg/sort/v2" "github.com/flant/glaball/pkg/util" + "github.com/google/go-github/v58/github" "github.com/flant/glaball/cmd/common" @@ -219,6 +221,74 @@ func ListProjectsByNamespace(h *client.Host, namespaces []string, opt gitlab.Lis return listProjectsByNamespace(h, namespaces, opt, wg, data, options...) } +func listRepositories(h *client.Host, archived bool, opt github.RepositoryListByOrgOptions, + wg *limiter.Limiter, data chan<- interface{}) error { + defer wg.Done() + + ctx := context.TODO() + wg.Lock() + list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org, &opt) + if err != nil { + wg.Error(h, err) + wg.Unlock() + return err + } + wg.Unlock() + + for _, v := range list { + if v.GetArchived() == archived { + data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"} + } + } + + if resp.NextPage > 0 { + wg.Add(1) + opt.Page = resp.NextPage + go listRepositories(h, archived, opt, wg, data) + } + + return nil +} + +func ListRepositories(h *client.Host, archived bool, opt github.RepositoryListByOrgOptions, + wg *limiter.Limiter, data chan<- interface{}) error { + return listRepositories(h, archived, opt, wg, data) +} + +func listRepositoriesByNamespace(h *client.Host, namespaces []string, archived bool, opt github.RepositoryListByOrgOptions, + wg *limiter.Limiter, data chan<- interface{}) error { + defer wg.Done() + + ctx := context.TODO() + wg.Lock() + list, resp, err := h.GithubClient.Repositories.ListByOrg(ctx, h.Org, &opt) + if err != nil { + wg.Error(h, err) + wg.Unlock() + return err + } + wg.Unlock() + + for _, v := range list { + if v.GetArchived() == archived && (len(namespaces) == 0 || util.ContainsString(namespaces, v.GetName())) { + data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"} + } + } + + if resp.NextPage > 0 { + wg.Add(1) + opt.Page = resp.NextPage + go listRepositoriesByNamespace(h, namespaces, archived, opt, wg, data) + } + + return nil +} + +func ListRepositoriesByNamespace(h *client.Host, namespaces []string, archived bool, opt github.RepositoryListByOrgOptions, + wg *limiter.Limiter, data chan<- interface{}) error { + return listRepositoriesByNamespace(h, namespaces, archived, opt, wg, data) +} + func listMergeRequests(h *client.Host, project *gitlab.Project, opt gitlab.ListProjectMergeRequestsOptions, wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error { defer wg.Done() @@ -360,12 +430,93 @@ func listMergeRequestsByAssigneeOrAuthorID(h *client.Host, project *gitlab.Proje return nil } +func listPullRequestsByAssigneeOrAuthorID(h *client.Host, repository *github.Repository, IDs []int, + opt github.PullRequestListOptions, wg *limiter.Limiter, data chan<- interface{}) error { + defer wg.Done() + + ctx := context.TODO() + wg.Lock() + list, resp, err := h.GithubClient.PullRequests.List(ctx, h.Org, repository.GetName(), &opt) + if err != nil { + wg.Error(h, err) + wg.Unlock() + return err + } + wg.Unlock() + + for _, v := range list { + if len(IDs) == 0 { + data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"} + continue + } + + // if pr has assignee, then check and continue + if v.Assignee != nil { + if util.ContainsInt(IDs, int(v.Assignee.GetID())) { + data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"} + } + continue + } + + // otherwise check the author + if v.User != nil && util.ContainsInt(IDs, int(v.User.GetID())) { + data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"} + } + } + + if resp.NextPage > 0 { + wg.Add(1) + opt.Page = resp.NextPage + go listPullRequestsByAssigneeOrAuthorID(h, repository, IDs, opt, wg, data) + } + + return nil +} + // authorIDs slice must be sorted in ascending order func ListMergeRequestsByAuthorOrAssigneeID(h *client.Host, project *gitlab.Project, IDs []int, opt gitlab.ListProjectMergeRequestsOptions, wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error { return listMergeRequestsByAssigneeOrAuthorID(h, project, IDs, opt, wg, data, options...) } +// authorIDs slice must be sorted in ascending order +func ListPullRequestsByAuthorOrAssigneeID(h *client.Host, repository *github.Repository, IDs []int, + opt github.PullRequestListOptions, wg *limiter.Limiter, data chan<- interface{}) error { + return listPullRequestsByAssigneeOrAuthorID(h, repository, IDs, opt, wg, data) +} + +func listPullRequests(h *client.Host, repository *github.Repository, opt github.PullRequestListOptions, + wg *limiter.Limiter, data chan<- interface{}) error { + defer wg.Done() + + ctx := context.TODO() + wg.Lock() + list, resp, err := h.GithubClient.PullRequests.List(ctx, h.Org, repository.GetName(), &opt) + if err != nil { + wg.Error(h, err) + wg.Unlock() + return err + } + wg.Unlock() + + for _, v := range list { + data <- sort.Element{Host: h, Struct: v, Cached: resp.Header.Get("X-From-Cache") == "1"} + } + + if resp.NextPage > 0 { + wg.Add(1) + opt.Page = resp.NextPage + go listPullRequests(h, repository, opt, wg, data) + } + + return nil +} + +func ListPullRequests(h *client.Host, repository *github.Repository, opt github.PullRequestListOptions, + wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error { + return listPullRequests(h, repository, opt, wg, data) +} + func listMergeRequestsSearch(h *client.Host, project *gitlab.Project, key string, value *regexp.Regexp, opt gitlab.ListProjectMergeRequestsOptions, wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) error { defer wg.Done() diff --git a/cmd/projects/schedules.go b/cmd/projects/schedules.go index 42a3ff8..4123824 100644 --- a/cmd/projects/schedules.go +++ b/cmd/projects/schedules.go @@ -1,6 +1,7 @@ package projects import ( + "context" "fmt" "math" "os" @@ -14,6 +15,7 @@ import ( "github.com/flant/glaball/pkg/limiter" "github.com/flant/glaball/pkg/sort/v2" "github.com/flant/glaball/pkg/util" + "github.com/google/go-github/v58/github" "github.com/flant/glaball/cmd/common" @@ -577,7 +579,7 @@ func listPipelineSchedules(h *client.Host, project *gitlab.Project, opt gitlab.L wg.Unlock() // filter schedules by matching descriptions if any - filteredList := make([]*gitlab.PipelineSchedule, 0, len(list)) + filteredList := make([]*gitlab.PipelineSchedule, 0, len(desc)) filter: for _, v := range list { for _, p := range desc { @@ -684,12 +686,124 @@ filter: } } +func listWorkflowRuns(h *client.Host, repository *github.Repository, opt github.ListOptions, + desc []*regexp.Regexp, withLastWorkflowRuns int, withFileContent bool, wg *limiter.Limiter, data chan<- interface{}) { + + defer wg.Done() + + ctx := context.TODO() + wg.Lock() + list, resp, err := h.GithubClient.Actions.ListWorkflows(ctx, h.Org, repository.GetName(), &opt) + if err != nil { + wg.Error(h, err) + wg.Unlock() + return + } + wg.Unlock() + + // filter schedules by matching descriptions if any + filteredList := make([]*github.Workflow, 0, len(desc)) +filter: + for _, v := range list.Workflows { + for _, p := range desc { + if p.MatchString(v.GetName()) { + filteredList = append(filteredList, v) + continue filter + } + } + } + if len(filteredList) == 0 { + // if no workflows were found and no --active flag value was provided + // return repository with nil workflow + if active == nil && status == nil { + data <- sort.Element{ + Host: h, + Struct: RepositoryWorkflow{repository, nil, nil, nil}, + Cached: resp.Header.Get("X-From-Cache") == "1"} + } + } else { + for _, v := range filteredList { + runs := new(github.WorkflowRuns) + if withLastWorkflowRuns > 0 { + // get last workflow runs + wg.Lock() + runs, _, err = h.GithubClient.Actions.ListWorkflowRunsByID(ctx, + h.Org, + repository.GetName(), + v.GetID(), + &github.ListWorkflowRunsOptions{ + Branch: repository.GetDefaultBranch(), + ExcludePullRequests: true, + ListOptions: github.ListOptions{ + PerPage: withLastWorkflowRuns, + }, + }) + if err != nil { + wg.Error(h, err) + wg.Unlock() + continue + } + wg.Unlock() + } + + // check workflow state + if active != nil && (v.GetState() == "active") != *active { + continue + } + // check last workflow run status + // ignore workflow runs with empty status if defined + if status != nil { + if runs.GetTotalCount() == 0 { + continue + } + if s := runs.WorkflowRuns[0].GetStatus(); s == "" || s != *status { + continue + } + } + + var fileContent *github.RepositoryContent + if withFileContent { + wg.Lock() + fileContent, _, _, err = h.GithubClient.Repositories.GetContents(ctx, + repository.Owner.GetLogin(), + repository.GetName(), + v.GetPath(), + &github.RepositoryContentGetOptions{Ref: repository.GetDefaultBranch()}) + wg.Unlock() + if err != nil { + wg.Error(h, err) + continue + } + } + + // push result to channel + data <- sort.Element{ + Host: h, + Struct: RepositoryWorkflow{repository, v, runs, fileContent}, + Cached: resp.Header.Get("X-From-Cache") == "1"} + } + } + + if resp.NextPage > 0 { + wg.Add(1) + opt.Page = resp.NextPage + go listWorkflowRuns(h, repository, opt, desc, withLastWorkflowRuns, withFileContent, wg, data) + } +} + type ProjectPipelineSchedule struct { Project *gitlab.Project `json:"project,omitempty"` Schedule *gitlab.PipelineSchedule `json:"schedule,omitempty"` Pipelines []*gitlab.Pipeline `json:"pipelines,omitempty"` } +type RepositoryWorkflow struct { + Repository *github.Repository `json:"repository,omitempty"` + Workflow *github.Workflow `json:"workflow,omitempty"` + WorkflowRuns *github.WorkflowRuns `json:"workflow_runs,omitempty"` + WorkflowFileContent *github.RepositoryContent `json:"workflow_file_content,omitempty"` +} + type Schedules []*gitlab.PipelineSchedule func (a Schedules) Descriptions() string { @@ -717,6 +831,10 @@ func ListPipelineSchedules(h *client.Host, project *gitlab.Project, opt gitlab.L desc []*regexp.Regexp, withLastPipelines bool, wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) { listPipelineSchedules(h, project, opt, desc, withLastPipelines, wg, data, options...) } +func ListWorkflowRuns(h *client.Host, repository *github.Repository, opt github.ListOptions, + desc []*regexp.Regexp, withLastPipelines int, withFileContent bool, wg *limiter.Limiter, data chan<- interface{}) { + listWorkflowRuns(h, repository, opt, desc, withLastPipelines, withFileContent, wg, data) +} func takeOwnership(h *client.Host, schedule ProjectPipelineSchedule, wg *limiter.Limiter, data chan<- interface{}, options ...gitlab.RequestOptionFunc) { diff --git a/go.mod b/go.mod index 43c8355..00d0b06 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( dario.cat/mergo v1.0.0 github.com/ahmetb/go-linq v3.0.0+incompatible github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 + github.com/gofri/go-github-ratelimit v1.1.0 + github.com/google/go-github/v58 v58.0.0 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-hclog v1.6.2 @@ -14,7 +16,7 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 - github.com/xanzy/go-gitlab v0.95.2 + github.com/xanzy/go-gitlab v0.96.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -40,7 +42,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e // indirect + golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 6c49a39..b38de6d 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7z github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/gofri/go-github-ratelimit v1.1.0 h1:ijQ2bcv5pjZXNil5FiwglCg8wc9s8EgjTmNkqjw8nuk= +github.com/gofri/go-github-ratelimit v1.1.0/go.mod h1:OnCi5gV+hAG/LMR7llGhU7yHt44se9sYgKPnafoL7RY= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -27,8 +29,10 @@ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v58 v58.0.0 h1:Una7GGERlF/37XfkPwpzYJe0Vp4dt2k1kCjlxwjIvzw= +github.com/google/go-github/v58 v58.0.0/go.mod h1:k4hxDKEfoWpSqFlc8LTpGd9fu2KrV1YAa6Hi6FmDNY4= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= @@ -106,15 +110,15 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/xanzy/go-gitlab v0.95.2 h1:4p0IirHqEp5f0baK/aQqr4TR57IsD+8e4fuyAA1yi88= -github.com/xanzy/go-gitlab v0.95.2/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI= +github.com/xanzy/go-gitlab v0.96.0 h1:LGkZ+wSNMRtHIBaYE4Hq3dZVjprwHv3Y1+rhKU3WETs= +github.com/xanzy/go-gitlab v0.96.0/go.mod h1:ETg8tcj4OhrB84UEgeE8dSuV/0h4BBL1uOV/qK0vlyI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE= -golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA= +golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= diff --git a/pkg/client/client.go b/pkg/client/client.go index ccf53ea..f7e74d1 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -11,6 +11,8 @@ import ( "github.com/flant/glaball/pkg/config" "github.com/flant/glaball/pkg/util" + "github.com/gofri/go-github-ratelimit/github_ratelimit" + "github.com/google/go-github/v58/github" "github.com/gregjones/httpcache" "github.com/ahmetb/go-linq" @@ -20,6 +22,11 @@ import ( "github.com/xanzy/go-gitlab" ) +const ( + Gitlab string = "gitlab" + Github string = "github" +) + type Client struct { Hosts Hosts @@ -60,6 +67,8 @@ func (h Hosts) Less(i, j int) bool { type Host struct { Team, Project, Name, URL string Client *gitlab.Client + GithubClient *github.Client + Org string // TODO: } func (h Host) FullName() string { @@ -137,6 +146,12 @@ func NewClient(cfg *config.Config) (*Client, error) { return nil, err } + // TODO: + ghttpClient, err := github_ratelimit.NewRateLimitWaiterClient(httpClient.Transport) + if err != nil { + return nil, fmt.Errorf("failed to create github http client") + } + options := []gitlab.ClientOptionFunc{ gitlab.WithHTTPClient(httpClient), } @@ -153,32 +168,47 @@ func NewClient(cfg *config.Config) (*Client, error) { if !filter.MatchString(fullName) { continue } - if host.URL == "" { - return nil, fmt.Errorf("missing url for host %q", fullName) - } if host.Token == "" { return nil, fmt.Errorf("missing token for host %q", fullName) } - if cfg.Cache.Enabled && !host.RateLimiter.Enabled { - options = append(options, gitlab.WithCustomLimiter(&FakeLimiter{})) - } - gl, err := gitlab.NewClient(host.Token, - append(options, gitlab.WithBaseURL(host.URL))...) - if err != nil { - return nil, err - } - if host.IP != "" { - customAddresses[gl.BaseURL().Hostname()] = host.IP + // TODO: + switch host.Type { + case Github: + // TODO: add cache + cfg.Cache.Enabled = false + + client.Hosts = append(client.Hosts, &Host{ + Team: team, + Project: project, + Name: name, + URL: fmt.Sprintf("https://github.com/%s", host.Org), // TODO: + Org: host.Org, + GithubClient: github.NewClient(ghttpClient).WithAuthToken(host.Token), + }) + default: + if host.URL == "" { + return nil, fmt.Errorf("missing url for host %q", fullName) + } + if cfg.Cache.Enabled && !host.RateLimiter.Enabled { + options = append(options, gitlab.WithCustomLimiter(&FakeLimiter{})) + } + gl, err := gitlab.NewClient(host.Token, + append(options, gitlab.WithBaseURL(host.URL))...) + if err != nil { + return nil, err + } + if host.IP != "" { + customAddresses[gl.BaseURL().Hostname()] = host.IP + } + client.Hosts = append(client.Hosts, &Host{ + Team: team, + Project: project, + Name: name, + URL: host.URL, + Client: gl, + }) } - - client.Hosts = append(client.Hosts, &Host{ - Team: team, - Project: project, - Name: name, - URL: host.URL, - Client: gl, - }) } } } diff --git a/pkg/config/config.go b/pkg/config/config.go index b4d2e5b..5d2ee01 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -26,9 +26,23 @@ type Host struct { URL string `yaml:"url" mapstructure:"url"` IP string `yaml:"ip" mapstructure:"ip"` Token string `yaml:"token" mapstructure:"token"` + Type string `yaml:"type" mapstructure:"type"` + Org string `yaml:"org" mapstructure:"org"` RateLimiter RateLimiterOptions `yaml:"rate_limiter" mapstructure:"rate_limiter"` } +// TODO: +type GitlabHost struct { + URL string `yaml:"url" mapstructure:"url"` + Token string `yaml:"token" mapstructure:"token"` +} + +// TODO: +type GithubHost struct { + Org string `yaml:"org" mapstructure:"org"` + Token string `yaml:"token" mapstructure:"token"` +} + type RateLimiterOptions struct { Enabled bool `yaml:"enabled" mapstructure:"enabled"` } diff --git a/pkg/sort/v2/util.go b/pkg/sort/v2/util.go index 876d31f..4a273db 100644 --- a/pkg/sort/v2/util.go +++ b/pkg/sort/v2/util.go @@ -12,8 +12,12 @@ func ValidFieldValue(keys []string, v interface{}) (interface{}, error) { rv := reflect.ValueOf(v) for _, k := range keys { if fi := m.GetByPath(k); fi != nil { - if fv := reflectx.FieldByIndexesReadOnly(rv, fi.Index).Interface(); fv != nil && fv != v { - return fv, nil + fv := reflectx.FieldByIndexesReadOnly(rv, fi.Index) + if fi.Field.Type.Kind() == reflect.Ptr { + fv = fv.Elem() + } + if rfv := fv.Interface(); rfv != nil && rfv != v { + return rfv, nil } } }