Skip to content

Commit

Permalink
Merge pull request #31 from src-d/add_logging
Browse files Browse the repository at this point in the history
Add logging
  • Loading branch information
smacker authored Oct 10, 2019
2 parents 504e46a + 694f14e commit 556ea44
Show file tree
Hide file tree
Showing 5 changed files with 233 additions and 28 deletions.
14 changes: 8 additions & 6 deletions examples/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ type Repository struct {
func (c *Repository) Execute(args []string) error {
return c.ExecuteBody(
log.New(log.Fields{"owner": c.Owner, "repo": c.Name}),
func(httpClient *http.Client, downloader *github.Downloader) error {
func(logger log.Logger, httpClient *http.Client, downloader *github.Downloader) error {
return downloader.DownloadRepository(context.TODO(), c.Owner, c.Name, c.Version)
})
}
Expand All @@ -65,7 +65,7 @@ type Organization struct {
func (c *Organization) Execute(args []string) error {
return c.ExecuteBody(
log.New(log.Fields{"org": c.Name}),
func(httpClient *http.Client, downloader *github.Downloader) error {
func(logger log.Logger, httpClient *http.Client, downloader *github.Downloader) error {
return downloader.DownloadOrganization(context.TODO(), c.Name, c.Version)
})
}
Expand All @@ -81,7 +81,7 @@ type Ghsync struct {
func (c *Ghsync) Execute(args []string) error {
return c.ExecuteBody(
log.New(log.Fields{"org": c.Name}),
func(httpClient *http.Client, downloader *github.Downloader) error {
func(logger log.Logger, httpClient *http.Client, downloader *github.Downloader) error {
repos, err := listRepositories(context.TODO(), httpClient, c.Name, c.NoForks)
if err != nil {
return err
Expand All @@ -92,19 +92,21 @@ func (c *Ghsync) Execute(args []string) error {
return fmt.Errorf("failed to download organization %v: %v", c.Name, err)
}

for _, repo := range repos {
for i, repo := range repos {
logger.Infof("start downloading '%s'", repo)
err = downloader.DownloadRepository(context.TODO(), c.Name, repo, c.Version)
if err != nil {
return fmt.Errorf("failed to download repository %v/%v: %v", c.Name, repo, err)
}
logger.Infof("finished downloading '%s' (%d/%d)", repo, i+1, len(repos))
}

return nil

})
}

type bodyFunc = func(httpClient *http.Client, downloader *github.Downloader) error
type bodyFunc = func(logger log.Logger, httpClient *http.Client, downloader *github.Downloader) error

func (c *DownloaderCmd) ExecuteBody(logger log.Logger, fn bodyFunc) error {
client := oauth2.NewClient(context.TODO(), oauth2.StaticTokenSource(
Expand Down Expand Up @@ -153,7 +155,7 @@ func (c *DownloaderCmd) ExecuteBody(logger log.Logger, fn bodyFunc) error {
}
t0 := time.Now()

err = fn(client, downloader)
err = fn(logger, client, downloader)
if err != nil {
return err
}
Expand Down
38 changes: 38 additions & 0 deletions github/downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ import (

"github.com/src-d/metadata-retrieval/github/graphql"
"github.com/src-d/metadata-retrieval/github/store"
"github.com/src-d/metadata-retrieval/utils/ctxlog"

"github.com/shurcooL/githubv4"
"gopkg.in/src-d/go-log.v1"
)

const (
Expand All @@ -22,6 +24,9 @@ const (
pullRequestReviewsPage = 5
pullRequestsPage = 50
repositoryTopicsPage = 50

// to track progress of sub-resources only each N page to avoid log flooding
logEachPageN = 3
)

type storer interface {
Expand Down Expand Up @@ -82,6 +87,8 @@ func NewStdoutDownloader(httpClient *http.Client) (*Downloader, error) {
// DownloadRepository downloads the metadata for the given repository and all
// its resources (issues, PRs, comments, reviews)
func (d Downloader) DownloadRepository(ctx context.Context, owner string, name string, version int) error {
ctx, _ = ctxlog.WithLogFields(ctx, log.Fields{"owner": owner, "repo": name})

d.storer.Version(version)

var err error
Expand Down Expand Up @@ -178,6 +185,10 @@ func (d Downloader) RateRemaining(ctx context.Context) (int, error) {
}

func (d Downloader) downloadTopics(ctx context.Context, repository *graphql.Repository) ([]string, error) {
logger := ctxlog.Get(ctx)
logger.Infof("start downloading topics")
defer logger.Infof("finished downloading topics")

topics := []string{}

// Topics included in the first page
Expand Down Expand Up @@ -225,6 +236,10 @@ func (d Downloader) downloadTopics(ctx context.Context, repository *graphql.Repo
}

func (d Downloader) downloadIssues(ctx context.Context, owner string, name string, repository *graphql.Repository) error {
logger := ctxlog.Get(ctx)
logger.Infof("start downloading issues")
defer logger.Infof("finished downloading issues")

process := func(issue *graphql.Issue) error {
assignees, err := d.downloadIssueAssignees(ctx, issue)
if err != nil {
Expand All @@ -243,6 +258,8 @@ func (d Downloader) downloadIssues(ctx context.Context, owner string, name strin
return d.downloadIssueComments(ctx, owner, name, issue)
}

count := len(repository.Issues.Nodes)

// Save issues included in the first page
for _, issue := range repository.Issues.Nodes {
err := process(&issue)
Expand Down Expand Up @@ -270,6 +287,10 @@ func (d Downloader) downloadIssues(ctx context.Context, owner string, name strin
endCursor := repository.Issues.PageInfo.EndCursor

for hasNextPage {
if count%(issuesPage*logEachPageN) == 0 {
logger.Infof("%d/%d issues downloaded", count, repository.Issues.TotalCount)
}

// get only issues
var q struct {
Node struct {
Expand All @@ -293,6 +314,7 @@ func (d Downloader) downloadIssues(ctx context.Context, owner string, name strin
}
}

count += len(q.Node.Repository.Issues.Nodes)
hasNextPage = q.Node.Repository.Issues.PageInfo.HasNextPage
endCursor = q.Node.Repository.Issues.PageInfo.EndCursor
}
Expand Down Expand Up @@ -446,6 +468,10 @@ func (d Downloader) downloadIssueComments(ctx context.Context, owner string, nam
}

func (d Downloader) downloadPullRequests(ctx context.Context, owner string, name string, repository *graphql.Repository) error {
logger := ctxlog.Get(ctx)
logger.Infof("start downloading pull requests")
defer logger.Infof("finished downloading pull requests")

process := func(pr *graphql.PullRequest) error {
assignees, err := d.downloadPullRequestAssignees(ctx, pr)
if err != nil {
Expand Down Expand Up @@ -473,6 +499,8 @@ func (d Downloader) downloadPullRequests(ctx context.Context, owner string, name
return nil
}

count := len(repository.PullRequests.Nodes)

// Save PRs included in the first page
for _, pr := range repository.PullRequests.Nodes {
err := process(&pr)
Expand Down Expand Up @@ -504,6 +532,10 @@ func (d Downloader) downloadPullRequests(ctx context.Context, owner string, name
endCursor := repository.PullRequests.PageInfo.EndCursor

for hasNextPage {
if count%(pullRequestsPage*logEachPageN) == 0 {
logger.Infof("%d/%d pull requests downloaded", count, repository.PullRequests.TotalCount)
}

// get only PRs
var q struct {
Node struct {
Expand All @@ -527,6 +559,7 @@ func (d Downloader) downloadPullRequests(ctx context.Context, owner string, name
}
}

count += len(q.Node.Repository.PullRequests.Nodes)
hasNextPage = q.Node.Repository.PullRequests.PageInfo.HasNextPage
endCursor = q.Node.Repository.PullRequests.PageInfo.EndCursor
}
Expand Down Expand Up @@ -859,6 +892,11 @@ func (d Downloader) DownloadOrganization(ctx context.Context, name string, versi
}

func (d Downloader) downloadUsers(ctx context.Context, name string, organization *graphql.Organization) error {
var logger log.Logger
ctx, logger = ctxlog.WithLogFields(ctx, log.Fields{"owner": name})
logger.Infof("start downloading users")
defer logger.Infof("finished downloading users")

process := func(user *graphql.UserExtended) error {
err := d.storer.SaveUser(user)
if err != nil {
Expand Down
46 changes: 24 additions & 22 deletions github/graphql/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,12 @@ package graphql

import "time"

// Connection represents common fields for paginated the connections
type Connection struct {
PageInfo PageInfo
TotalCount int
}

// PageInfo represents https://developer.github.com/v4/object/pageinfo/
type PageInfo struct {
HasNextPage bool
Expand Down Expand Up @@ -45,9 +51,8 @@ type OrganizationFields struct {

// OrganizationMemberConnection represents https://developer.github.com/v4/object/organizationmemberconnection/
type OrganizationMemberConnection struct {
TotalCount int
PageInfo PageInfo
Nodes []UserExtended
Connection
Nodes []UserExtended
} // `graphql:"membersWithRole(first: $membersWithRolePage, after: $membersWithRoleCursor)"`

// UserExtended is the same type as User, but requesting more fields.
Expand Down Expand Up @@ -158,8 +163,8 @@ type RepositoryFields struct {

// RepositoryTopicsConnection represents https://developer.github.com/v4/object/repositorytopicconnection/
type RepositoryTopicsConnection struct {
PageInfo PageInfo
Nodes []struct {
Connection
Nodes []struct {
Topic struct {
Name string
}
Expand All @@ -168,14 +173,13 @@ type RepositoryTopicsConnection struct {

// IssueConnection represents https://developer.github.com/v4/object/issueconnection/
type IssueConnection struct {
PageInfo PageInfo
Nodes []Issue
Connection
Nodes []Issue
} //`graphql:"issues(first: $issuesPage, after: $issuesCursor)"`

type IssueCommentsConnection struct {
TotalCount int
PageInfo PageInfo
Nodes []IssueComment
Connection
Nodes []IssueComment
} // `graphql:"comments(first: $issueCommentsPage, after: $issueCommentsCursor)"`

// Issue represents https://developer.github.com/v4/object/issue/
Expand Down Expand Up @@ -229,8 +233,8 @@ type ClosedByConnection struct {

// UserConnection represents https://developer.github.com/v4/object/userconnection/
type UserConnection struct {
PageInfo PageInfo
Nodes []User
Connection
Nodes []User
} //`graphql:"assignees(first: $assigneesPage, after: $assigneesCursor)"`

// Label represents https://developer.github.com/v4/object/label/
Expand All @@ -240,8 +244,8 @@ type Label struct {

// LabelConnection represents https://developer.github.com/v4/object/labelconnection/
type LabelConnection struct {
PageInfo PageInfo
Nodes []Label
Connection
Nodes []Label
} //`graphql:"labels(first: $labelsPage, after: $labelsCursor)"`

type IssueComment struct {
Expand All @@ -256,8 +260,8 @@ type IssueComment struct {
}

type PullRequestConnection struct {
PageInfo PageInfo
Nodes []PullRequest
Connection
Nodes []PullRequest
} //`graphql:"pullRequests(first: $pullRequestsPage, after: $pullRequestsCursor)"`

type PullRequest struct {
Expand Down Expand Up @@ -327,9 +331,8 @@ type PullRequestFields struct {
}

type PullRequestReviewConnection struct {
//TotalCount int
PageInfo PageInfo
Nodes []PullRequestReview
Connection
Nodes []PullRequestReview
} // `graphql:"reviews(first: $pullRequestReviewsPage, after: $pullRequestReviewsCursor)"`

type PullRequestReview struct {
Expand All @@ -353,9 +356,8 @@ type PullRequestReviewFields struct {
}

type PullRequestReviewCommentConnection struct {
//TotalCount int
PageInfo PageInfo
Nodes []PullRequestReviewComment
Connection
Nodes []PullRequestReviewComment
}

type PullRequestReviewComment struct {
Expand Down
70 changes: 70 additions & 0 deletions utils/ctxlog/context.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package ctxlog

import (
"context"

log "gopkg.in/src-d/go-log.v1"
)

// NewLogger function to create new log.Logger, needed for mocking in tests
var NewLogger = log.New

type ctxKey int

// logFieldsKey is the key that holds log Fields in a context.
const logFieldsKey ctxKey = 0

// Get returns a logger configured with the context log Fields or the default fields
func Get(ctx context.Context) log.Logger {
var fields log.Fields

if v := ctx.Value(logFieldsKey); v != nil {
fields = v.(log.Fields)
}

return NewLogger(fields)
}

// Fields returns a copy of the context log fields. It can be nil
func Fields(ctx context.Context) log.Fields {
if v := ctx.Value(logFieldsKey); v != nil {
f := v.(log.Fields)
copy := make(map[string]interface{}, len(f))

for key, val := range f {
copy[key] = val
}

return copy
}

return nil
}

// WithLogFields returns a context with new logger Fields added to the current
// ones, and a logger
func WithLogFields(ctx context.Context, fields log.Fields) (context.Context, log.Logger) {
if fields == nil {
return ctx, Get(ctx)
}

newFields := make(map[string]interface{}, len(fields))

if v := ctx.Value(logFieldsKey); v != nil {
for key, val := range v.(log.Fields) {
newFields[key] = val
}
}

for key, val := range fields {
newFields[key] = val
}

ctx = set(ctx, newFields)
return ctx, Get(ctx)
}

// set returns a copy of ctx with the log Fields saves as context Value
func set(ctx context.Context, fields log.Fields) context.Context {
return context.WithValue(ctx, logFieldsKey, fields)
}
Loading

0 comments on commit 556ea44

Please sign in to comment.