Skip to content

Commit

Permalink
Merge branch 'master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@kyrcha
kyrcha authored Oct 14, 2019
2 parents 93433d7 + 0a799e2 commit 179269a
Show file tree
Hide file tree
Showing 8 changed files with 187 additions and 202 deletions.
5 changes: 0 additions & 5 deletions database/migrations/000001_init.up.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,18 @@ CREATE TABLE IF NOT EXISTS organizations_versioned (
versions integer ARRAY,

avatar_url text,
billing_email text,
collaborators bigint,
created_at timestamptz,
description text,
email text,
htmlurl text,
id bigint,
location text,
login text,
name text,
node_id text,
owned_private_repos bigint,
public_repos bigint,
total_private_repos bigint,
two_factor_requirement_enabled boolean,
updated_at timestamptz
);

Expand Down Expand Up @@ -47,7 +44,6 @@ CREATE TABLE IF NOT EXISTS users_versioned (
private_gists bigint,
public_gists bigint,
public_repos bigint,
site_admin boolean,
total_private_repos bigint,
updated_at timestamptz
);
Expand Down Expand Up @@ -76,7 +72,6 @@ CREATE TABLE IF NOT EXISTS repositories_versioned (
htmlurl text,
id bigint,
language text,
mirror_url text,
name text,
node_id text,
open_issues_count bigint,
Expand Down
12 changes: 7 additions & 5 deletions examples/cmd/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,9 @@ func (c *Ghsync) Execute(args []string) error {
type bodyFunc = func(logger log.Logger, httpClient *http.Client, downloader *github.Downloader) error

func (c *DownloaderCmd) ExecuteBody(logger log.Logger, fn bodyFunc) error {
client := oauth2.NewClient(context.TODO(), oauth2.StaticTokenSource(
ctx := context.Background()

client := oauth2.NewClient(ctx, oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: c.Token},
))

Expand Down Expand Up @@ -149,7 +151,7 @@ func (c *DownloaderCmd) ExecuteBody(logger log.Logger, fn bodyFunc) error {
downloader, err = github.NewDownloader(client, db)
}

rate0, err := downloader.RateRemaining(context.TODO())
rate0, err := downloader.RateRemaining(ctx)
if err != nil {
return err
}
Expand All @@ -160,18 +162,18 @@ func (c *DownloaderCmd) ExecuteBody(logger log.Logger, fn bodyFunc) error {
return err
}

err = downloader.SetCurrent(c.Version)
err = downloader.SetCurrent(ctx, c.Version)
if err != nil {
return err
}

if c.Cleanup {
return downloader.Cleanup(c.Version)
return downloader.Cleanup(ctx, c.Version)
}

elapsed := time.Since(t0)

rate1, err := downloader.RateRemaining(context.TODO())
rate1, err := downloader.RateRemaining(ctx)
if err != nil {
return err
}
Expand Down
78 changes: 39 additions & 39 deletions github/downloader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,22 +30,22 @@ const (
)

type storer interface {
SaveOrganization(organization *graphql.Organization) error
SaveUser(user *graphql.UserExtended) error
SaveRepository(repository *graphql.RepositoryFields, topics []string) error
SaveIssue(repositoryOwner, repositoryName string, issue *graphql.Issue, assignees []string, labels []string) error
SaveIssueComment(repositoryOwner, repositoryName string, issueNumber int, comment *graphql.IssueComment) error
SavePullRequest(repositoryOwner, repositoryName string, pr *graphql.PullRequest, assignees []string, labels []string) error
SavePullRequestComment(repositoryOwner, repositoryName string, pullRequestNumber int, comment *graphql.IssueComment) error
SavePullRequestReview(repositoryOwner, repositoryName string, pullRequestNumber int, review *graphql.PullRequestReview) error
SavePullRequestReviewComment(repositoryOwner, repositoryName string, pullRequestNumber int, pullRequestReviewId int, comment *graphql.PullRequestReviewComment) error
SaveOrganization(ctx context.Context, organization *graphql.Organization) error
SaveUser(ctx context.Context, user *graphql.UserExtended) error
SaveRepository(ctx context.Context, repository *graphql.RepositoryFields, topics []string) error
SaveIssue(ctx context.Context, repositoryOwner, repositoryName string, issue *graphql.Issue, assignees []string, labels []string) error
SaveIssueComment(ctx context.Context, repositoryOwner, repositoryName string, issueNumber int, comment *graphql.IssueComment) error
SavePullRequest(ctx context.Context, repositoryOwner, repositoryName string, pr *graphql.PullRequest, assignees []string, labels []string) error
SavePullRequestComment(ctx context.Context, repositoryOwner, repositoryName string, pullRequestNumber int, comment *graphql.IssueComment) error
SavePullRequestReview(ctx context.Context, repositoryOwner, repositoryName string, pullRequestNumber int, review *graphql.PullRequestReview) error
SavePullRequestReviewComment(ctx context.Context, repositoryOwner, repositoryName string, pullRequestNumber int, pullRequestReviewID int, comment *graphql.PullRequestReviewComment) error

Begin() error
Commit() error
Rollback() error
Version(v int)
SetActiveVersion(v int) error
Cleanup(currentVersion int) error
SetActiveVersion(ctx context.Context, v int) error
Cleanup(ctx context.Context, currentVersion int) error
}

// Downloader fetches GitHub data using the v4 API
Expand Down Expand Up @@ -148,7 +148,7 @@ func (d Downloader) DownloadRepository(ctx context.Context, owner string, name s
return err
}

err = d.storer.SaveRepository(&q.Repository.RepositoryFields, topics)
err = d.storer.SaveRepository(ctx, &q.Repository.RepositoryFields, topics)
if err != nil {
return fmt.Errorf("failed to save repository %v: %v", q.Repository.NameWithOwner, err)
}
Expand Down Expand Up @@ -197,7 +197,7 @@ func (d Downloader) downloadTopics(ctx context.Context, repository *graphql.Repo
}

variables := map[string]interface{}{
"id": githubv4.ID(repository.Id),
"id": githubv4.ID(repository.ID),

"repositoryTopicsPage": githubv4.Int(repositoryTopicsPage),
"repositoryTopicsCursor": (*githubv4.String)(nil),
Expand Down Expand Up @@ -251,7 +251,7 @@ func (d Downloader) downloadIssues(ctx context.Context, owner string, name strin
return err
}

err = d.storer.SaveIssue(owner, name, issue, assignees, labels)
err = d.storer.SaveIssue(ctx, owner, name, issue, assignees, labels)
if err != nil {
return err
}
Expand All @@ -269,7 +269,7 @@ func (d Downloader) downloadIssues(ctx context.Context, owner string, name strin
}

variables := map[string]interface{}{
"id": githubv4.ID(repository.Id),
"id": githubv4.ID(repository.ID),

"assigneesPage": githubv4.Int(assigneesPage),
"issueCommentsPage": githubv4.Int(issueCommentsPage),
Expand Down Expand Up @@ -331,7 +331,7 @@ func (d Downloader) downloadIssueAssignees(ctx context.Context, issue *graphql.I
}

variables := map[string]interface{}{
"id": githubv4.ID(issue.Id),
"id": githubv4.ID(issue.ID),

"assigneesPage": githubv4.Int(assigneesPage),
"assigneesCursor": (*githubv4.String)(nil),
Expand Down Expand Up @@ -378,7 +378,7 @@ func (d Downloader) downloadIssueLabels(ctx context.Context, issue *graphql.Issu
}

variables := map[string]interface{}{
"id": githubv4.ID(issue.Id),
"id": githubv4.ID(issue.ID),

"labelsPage": githubv4.Int(labelsPage),
"labelsCursor": (*githubv4.String)(nil),
Expand Down Expand Up @@ -419,14 +419,14 @@ func (d Downloader) downloadIssueLabels(ctx context.Context, issue *graphql.Issu
func (d Downloader) downloadIssueComments(ctx context.Context, owner string, name string, issue *graphql.Issue) error {
// save first page of comments
for _, comment := range issue.Comments.Nodes {
err := d.storer.SaveIssueComment(owner, name, issue.Number, &comment)
err := d.storer.SaveIssueComment(ctx, owner, name, issue.Number, &comment)
if err != nil {
return err
}
}

variables := map[string]interface{}{
"id": githubv4.ID(issue.Id),
"id": githubv4.ID(issue.ID),

"issueCommentsPage": githubv4.Int(issueCommentsPage),
"issueCommentsCursor": (*githubv4.String)(nil),
Expand Down Expand Up @@ -454,7 +454,7 @@ func (d Downloader) downloadIssueComments(ctx context.Context, owner string, nam
}

for _, comment := range q.Node.Issue.Comments.Nodes {
err := d.storer.SaveIssueComment(owner, name, issue.Number, &comment)
err := d.storer.SaveIssueComment(ctx, owner, name, issue.Number, &comment)
if err != nil {
return fmt.Errorf("failed to save issue comments for issue #%v: %v", issue.Number, err)
}
Expand Down Expand Up @@ -483,7 +483,7 @@ func (d Downloader) downloadPullRequests(ctx context.Context, owner string, name
return err
}

err = d.storer.SavePullRequest(owner, name, pr, assignees, labels)
err = d.storer.SavePullRequest(ctx, owner, name, pr, assignees, labels)
if err != nil {
return err
}
Expand All @@ -510,7 +510,7 @@ func (d Downloader) downloadPullRequests(ctx context.Context, owner string, name
}

variables := map[string]interface{}{
"id": githubv4.ID(repository.Id),
"id": githubv4.ID(repository.ID),

"assigneesPage": githubv4.Int(assigneesPage),
"issueCommentsPage": githubv4.Int(issueCommentsPage),
Expand Down Expand Up @@ -576,7 +576,7 @@ func (d Downloader) downloadPullRequestAssignees(ctx context.Context, pr *graphq
}

variables := map[string]interface{}{
"id": githubv4.ID(pr.Id),
"id": githubv4.ID(pr.ID),

"assigneesPage": githubv4.Int(assigneesPage),
"assigneesCursor": (*githubv4.String)(nil),
Expand Down Expand Up @@ -623,7 +623,7 @@ func (d Downloader) downloadPullRequestLabels(ctx context.Context, pr *graphql.P
}

variables := map[string]interface{}{
"id": githubv4.ID(pr.Id),
"id": githubv4.ID(pr.ID),

"labelsPage": githubv4.Int(assigneesPage),
"labelsCursor": (*githubv4.String)(nil),
Expand Down Expand Up @@ -664,14 +664,14 @@ func (d Downloader) downloadPullRequestLabels(ctx context.Context, pr *graphql.P
func (d Downloader) downloadPullRequestComments(ctx context.Context, owner string, name string, pr *graphql.PullRequest) error {
// save first page of comments
for _, comment := range pr.Comments.Nodes {
err := d.storer.SavePullRequestComment(owner, name, pr.Number, &comment)
err := d.storer.SavePullRequestComment(ctx, owner, name, pr.Number, &comment)
if err != nil {
return fmt.Errorf("failed to save PR comments for PR #%v: %v", pr.Number, err)
}
}

variables := map[string]interface{}{
"id": githubv4.ID(pr.Id),
"id": githubv4.ID(pr.ID),

"issueCommentsPage": githubv4.Int(issueCommentsPage),
"issueCommentsCursor": (*githubv4.String)(nil),
Expand Down Expand Up @@ -699,7 +699,7 @@ func (d Downloader) downloadPullRequestComments(ctx context.Context, owner strin
}

for _, comment := range q.Node.PullRequest.Comments.Nodes {
err := d.storer.SavePullRequestComment(owner, name, pr.Number, &comment)
err := d.storer.SavePullRequestComment(ctx, owner, name, pr.Number, &comment)
if err != nil {
return fmt.Errorf("failed to save PR comments for PR #%v: %v", pr.Number, err)
}
Expand All @@ -714,7 +714,7 @@ func (d Downloader) downloadPullRequestComments(ctx context.Context, owner strin

func (d Downloader) downloadPullRequestReviews(ctx context.Context, owner string, name string, pr *graphql.PullRequest) error {
process := func(review *graphql.PullRequestReview) error {
err := d.storer.SavePullRequestReview(owner, name, pr.Number, review)
err := d.storer.SavePullRequestReview(ctx, owner, name, pr.Number, review)
if err != nil {
return fmt.Errorf("failed to save PR review for PR #%v: %v", pr.Number, err)
}
Expand All @@ -730,7 +730,7 @@ func (d Downloader) downloadPullRequestReviews(ctx context.Context, owner string
}

variables := map[string]interface{}{
"id": githubv4.ID(pr.Id),
"id": githubv4.ID(pr.ID),

"pullRequestReviewCommentsPage": githubv4.Int(pullRequestReviewCommentsPage),
"pullRequestReviewsPage": githubv4.Int(pullRequestReviewsPage),
Expand Down Expand Up @@ -776,11 +776,11 @@ func (d Downloader) downloadPullRequestReviews(ctx context.Context, owner string

func (d Downloader) downloadReviewComments(ctx context.Context, repositoryOwner, repositoryName string, pullRequestNumber int, review *graphql.PullRequestReview) error {
process := func(comment *graphql.PullRequestReviewComment) error {
err := d.storer.SavePullRequestReviewComment(repositoryOwner, repositoryName, pullRequestNumber, review.DatabaseId, comment)
err := d.storer.SavePullRequestReviewComment(ctx, repositoryOwner, repositoryName, pullRequestNumber, review.DatabaseID, comment)
if err != nil {
return fmt.Errorf(
"failed to save PullRequestReviewComment for PR #%v, review ID %v: %v",
pullRequestNumber, review.Id, err)
pullRequestNumber, review.ID, err)
}

return nil
Expand All @@ -795,7 +795,7 @@ func (d Downloader) downloadReviewComments(ctx context.Context, repositoryOwner,
}

variables := map[string]interface{}{
"id": githubv4.ID(review.Id),
"id": githubv4.ID(review.ID),

"pullRequestReviewCommentsPage": githubv4.Int(pullRequestReviewCommentsPage),
"pullRequestReviewCommentsCursor": (*githubv4.String)(nil),
Expand All @@ -820,7 +820,7 @@ func (d Downloader) downloadReviewComments(ctx context.Context, repositoryOwner,
if err != nil {
return fmt.Errorf(
"failed to query PR review comments for PR #%v, review ID %v: %v",
pullRequestNumber, review.Id, err)
pullRequestNumber, review.ID, err)
}

for _, comment := range q.Node.PullRequestReview.Comments.Nodes {
Expand Down Expand Up @@ -877,7 +877,7 @@ func (d Downloader) DownloadOrganization(ctx context.Context, name string, versi
return fmt.Errorf("organization query failed: %v", err)
}

err = d.storer.SaveOrganization(&q.Organization)
err = d.storer.SaveOrganization(ctx, &q.Organization)
if err != nil {
return fmt.Errorf("failed to save organization %v: %v", name, err)
}
Expand All @@ -898,7 +898,7 @@ func (d Downloader) downloadUsers(ctx context.Context, name string, organization
defer logger.Infof("finished downloading users")

process := func(user *graphql.UserExtended) error {
err := d.storer.SaveUser(user)
err := d.storer.SaveUser(ctx, user)
if err != nil {
return fmt.Errorf("failed to save UserExtended: %v", err)
}
Expand Down Expand Up @@ -955,17 +955,17 @@ func (d Downloader) downloadUsers(ctx context.Context, name string, organization
}

// SetCurrent enables the given version as the current one accessible in the DB
func (d Downloader) SetCurrent(version int) error {
err := d.storer.SetActiveVersion(version)
func (d Downloader) SetCurrent(ctx context.Context, version int) error {
err := d.storer.SetActiveVersion(ctx, version)
if err != nil {
return fmt.Errorf("failed to set current DB version to %v: %v", version, err)
}
return nil
}

// Cleanup deletes from the DB all records that do not belong to the currentVersion
func (d Downloader) Cleanup(currentVersion int) error {
err := d.storer.Cleanup(currentVersion)
func (d Downloader) Cleanup(ctx context.Context, currentVersion int) error {
err := d.storer.Cleanup(ctx, currentVersion)
if err != nil {
return fmt.Errorf("failed to do cleanup for DB version %v: %v", currentVersion, err)
}
Expand Down
6 changes: 4 additions & 2 deletions github/downloader_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,8 @@ func testRepo(t *testing.T, oracle testutils.RepositoryTest, d *Downloader, stor
err := d.DownloadRepository(context.TODO(), oracle.Owner, oracle.Repository, oracle.Version)
require := require.New(t) // Make a new require object for the specified test, so no need to pass it around
require.Nil(err)
require.Equal(oracle.URL, storer.Repository.Url)
// Sample some properties that will not change, no topics available in git-fixtures
require.Equal(oracle.URL, storer.Repository.URL)
require.Equal(oracle.CreatedAt, storer.Repository.CreatedAt.String())
require.Equal(oracle.IsPrivate, storer.Repository.IsPrivate)
require.Equal(oracle.IsArchived, storer.Repository.IsArchived)
Expand Down Expand Up @@ -221,8 +222,9 @@ func testOrg(t *testing.T, oracle testutils.OrganizationTest, d *Downloader, sto
err := d.DownloadOrganization(context.TODO(), oracle.Org, oracle.Version)
require := require.New(t)
require.Nil(err, "DownloadOrganization(%s) failed", oracle.Org)
// Sample some properties that will not change, no topics available in git-fixtures
require.Equal(oracle.Org, storer.Organization.Login)
require.Equal(oracle.URL, storer.Organization.Url)
require.Equal(oracle.URL, storer.Organization.URL)
require.Equal(oracle.CreatedAt, storer.Organization.CreatedAt.String())
require.Equal(oracle.PublicRepos, storer.Organization.PublicRepos.TotalCount)
require.Equal(oracle.TotalPrivateRepos, storer.Organization.TotalPrivateRepos.TotalCount)
Expand Down
Loading

0 comments on commit 179269a

Please sign in to comment.