diff --git a/.travis.yml b/.travis.yml index 8506943..2fa82f0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,21 +3,29 @@ go_import_path: github.com/src-d/metdata-retrieval go: - 1.13.x +env: + global: + - PSQL_USER=user + - PSQL_PWD=password + - PSQL_DB=ghsync + branches: only: - master - /^v\d+\.\d+(\.\d+)?(-\S*)?$/ -stages: - - name: tests - -os: - - linux - - osx - jobs: include: - - stage: tests - name: 'Unit Tests' + - stage: test + name: "Integration tests" + os: linux + services: docker script: + # only linux supports docker https://docs.travis-ci.com/user/docker/ => tests with docker skipped programmatically ~ OS + - docker-compose up -d - make test-coverage codecov + - stage: test + name: "Integration tests" + os: osx + script: + - make test-coverage codecov \ No newline at end of file diff --git a/README.md b/README.md index 6ee2937..d53244d 100644 --- a/README.md +++ b/README.md @@ -371,7 +371,7 @@ make migration ### Testing -To test run: +To test, run: ``` GITHUB_TOKEN= go test ./... @@ -380,9 +380,13 @@ GITHUB_TOKEN= go test ./... and ``` -GITHUB_TOKEN= go test -cover ./... +GITHUB_TOKEN= go test -coverpkg=./... -coverprofile=coverage.out ./... ``` -for coverage information. +for coverage information on all the packages which can be seen with: + +``` +go tool cover -html=coverage.out +``` Where `GITHUB_TOKEN` is a personal access token (scopes **read:org**, **repo**). diff --git a/examples/cmd/testing/recordingscript.go b/examples/cmd/testing/recordingscript.go new file mode 100644 index 0000000..e3ef9ad --- /dev/null +++ b/examples/cmd/testing/recordingscript.go @@ -0,0 +1,130 @@ +package main + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/gob" + "flag" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/src-d/metadata-retrieval/github" + + "github.com/motemen/go-loghttp" + "golang.org/x/oauth2" + "gopkg.in/src-d/go-log.v1" +) + +// This script will use the downloader to crawl using the graphql API a repository (default: src-d/gitbase) +// and an organization (default: src-d) and will record requests (graphql queries) and responses (json body data) +// into a map stored in a gob file. +// NB: script is assumed to run from the project root (for correct file storage) +func main() { + var ( + org string + repo string + ) + flag.StringVar(&org, "org", "src-d", "a GitHub organization") + flag.StringVar(&repo, "repo", "gitbase", "a GitHub repository") + flag.Parse() + + // Variables to hold graphql queries (keys) and responses (values) + var ( + reqResp map[string]string = make(map[string]string) + query string + ctx context.Context + ) + ctx = context.Background() + // Create a with authentication and add logging (recording) + client := oauth2.NewClient( + ctx, + oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + )) + + client.Transport = &loghttp.Transport{ + Transport: client.Transport, + LogRequest: func(req *http.Request) { + query = cloneRequest(req) + }, + LogResponse: func(resp *http.Response) { + // TODO(@kyrcha): also record other types of responses + if resp.StatusCode == http.StatusOK { + reqResp[query] = cloneResponse(resp) + } + }, + } + + downloader, err := github.NewStdoutDownloader(client) + if err != nil { + panic(err) + } + + // record a repo crawl + log.Infof("Start recording a repo") + downloader.DownloadRepository(ctx, org, repo, 0) + log.Infof("End recording a repo") + + // store the results + dt := time.Now() + filename := fmt.Sprintf("repository_%s_%s_%s.gob.gz", org, repo, dt.Format("2006-01-02")) + err = encodeAndStore(filename, reqResp) + if err != nil { + panic(err) + } + + // reset map + reqResp = make(map[string]string) + + // record an org crawl + log.Infof("Start recording an org") + downloader.DownloadOrganization(ctx, org, 0) + log.Infof("End recording an org") + + // store the results + filename = fmt.Sprintf("organization_%s_%s.gob.gz", org, dt.Format("2006-01-02")) + err = encodeAndStore(filename, reqResp) + if err != nil { + panic(err) + } + +} + +func cloneRequest(req *http.Request) string { + savecl := req.ContentLength + bodyBytes, _ := ioutil.ReadAll(req.Body) + defer req.Body.Close() + // recreate request body + req.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + req.ContentLength = savecl + return string(bodyBytes) +} + +func cloneResponse(resp *http.Response) string { + // consume response body + savecl := resp.ContentLength + bodyBytes, _ := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + // recreate response body + resp.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + resp.ContentLength = savecl + // save response body + return string(bodyBytes) +} + +func encodeAndStore(filename string, reqResp map[string]string) error { + filepath := filepath.Join("testdata", filename) + encodeFile, err := os.Create(filepath) + if err != nil { + return err + } + defer encodeFile.Close() + zw := gzip.NewWriter(encodeFile) + defer zw.Close() + return gob.NewEncoder(zw).Encode(reqResp) +} diff --git a/github/downloader_test.go b/github/downloader_test.go index 594c70a..efade56 100644 --- a/github/downloader_test.go +++ b/github/downloader_test.go @@ -1,61 +1,68 @@ +// Integration tests of the metadata graphql crawler: +// - with and without the DB +// - online and offline tests + package github import ( + "bytes" + "compress/gzip" "context" + "database/sql" + "encoding/gob" "encoding/json" "fmt" "io/ioutil" + "net/http" "os" + "runtime" + "strings" "testing" + "time" + "github.com/src-d/metadata-retrieval/database" + "github.com/src-d/metadata-retrieval/github/store" "github.com/src-d/metadata-retrieval/testutils" + "github.com/cenkalti/backoff" + "github.com/golang-migrate/migrate/v4" + "github.com/lib/pq" + "github.com/shurcooL/githubv4" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" "golang.org/x/oauth2" ) -// RepositoryTest struct to hold a test oracle for a repository -type RepositoryTest struct { - Owner string `json:"owner"` - Repository string `json:"repository"` - Version int `json:"version"` - URL string `json:"url"` - CreatedAt string `json:"createdAt"` - IsPrivate bool `json:"isPrivate"` - IsArchived bool `json:"isArchived"` - HasWiki bool `json:"hasWiki"` - NumOfPRs int `json:"numOfPrs"` - NumOfPRComments int `json:"numOfPrComments"` -} - -// OrganizationTest struct to hold a test oracle for an organization -type OrganizationTest struct { - Org string `json:"org"` - Version int `json:"version"` - URL string `json:"url"` - CreatedAt string `json:"createdAt"` - PublicRepos int `json:"publicRepos"` - TotalPrivateRepos int `json:"totalPrivateRepos"` - NumOfUsers int `json:"numOfUsers"` -} - -// OnlineTests struct to hold the online tests -type OnlineTests struct { - RepositoryTests []RepositoryTest `json:"repositoryTests"` - OrganizationsTests []OrganizationTest `json:"organizationTests"` -} +const ( + orgRecFile = "../testdata/organization_src-d_2019-10-14.gob.gz" + repoRecFile = "../testdata/repository_src-d_gitbase_2019-10-14.gob.gz" + onlineRepoTests = "../testdata/online-repository-tests.json" + onlineOrgTests = "../testdata/online-organization-tests.json" + offlineRepoTests = "../testdata/offline-repository-tests.json" + offlineOrgTests = "../testdata/offline-organization-tests.json" +) -func checkToken(t *testing.T) { - if os.Getenv("GITHUB_TOKEN") == "" { - t.Skip("GITHUB_TOKEN is not set") - return +// loads requests-response data from a gob file +func loadReqResp(filepath string, reqResp map[string]string) error { + // Open a file + decodeFile, err := os.Open(filepath) + if err != nil { + return err } + defer decodeFile.Close() + reader, err := gzip.NewReader(decodeFile) + if err != nil { + return err + } + // Create a decoder and decode + return gob.NewDecoder(reader).Decode(&reqResp) } -func loadOnlineTests(filepath string) (OnlineTests, error) { +// loads tests from a json file +func loadTests(filepath string) (testutils.Tests, error) { var ( err error - tests OnlineTests + tests testutils.Tests ) jsonFile, err := os.Open(filepath) if err != nil { @@ -70,6 +77,48 @@ func loadOnlineTests(filepath string) (OnlineTests, error) { return tests, nil } +// checks whether a token exists as an env var, if not it skips the test +func checkToken(t *testing.T) { + if os.Getenv("GITHUB_TOKEN") == "" { + t.Skip("GITHUB_TOKEN is not set") + } +} + +func isOSXOnTravis() bool { + // docker service is not supported on osx in Travis: https://docs.travis-ci.com/user/docker/ + // but maybe in a local osx dev env so we will skip only in travis + if runtime.GOOS == "darwin" && os.Getenv("TRAVIS") == "true" { + return true + } + return false +} + +// Testing connection documentation, docker-compose and Migrate method +func getDB(t *testing.T) (db *sql.DB) { + DBURL := fmt.Sprintf("postgres://%s:%s@localhost:5432/%s?sslmode=disable", os.Getenv("PSQL_USER"), os.Getenv("PSQL_PWD"), os.Getenv("PSQL_DB")) + err := backoff.Retry(func() error { + var err error + db, err = sql.Open("postgres", DBURL) + return err + }, backoff.NewExponentialBackOff()) + require.NoError(t, err, "DB URL is not working") + if err = db.Ping(); err != nil { + require.Nil(t, err, "DB connection is not working") + } + if err = database.Migrate(DBURL); err != nil && err != migrate.ErrNoChange { + require.Nil(t, err, "Cannot migrate the DB") + } + return db +} + +// RoundTripFunc a function type that gets a request and returns a response +type RoundTripFunc func(req *http.Request) *http.Response + +// RoundTrip function to implement the interface of a RoundTripper Transport +func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + func getDownloader() (*Downloader, *testutils.Memory, error) { downloader, err := NewStdoutDownloader( oauth2.NewClient( @@ -80,13 +129,51 @@ func getDownloader() (*Downloader, *testutils.Memory, error) { if err != nil { return nil, nil, err } - storer := new(testutils.Memory) downloader.storer = storer return downloader, storer, nil } -func testOnlineRepo(t *testing.T, oracle RepositoryTest, d *Downloader, storer *testutils.Memory) { +// Tests: +// 1. Online (live) download with token for git-fixtures org and memory cache +// 2. Online (live) download with token for git-fixtures repos and memory cache +// 4. Offline (recorded) download without token for src-d/gitbase repo and memory cache +// 3. Offline (recorded) download without token for src-d org and memory cache +// 5. Online (live) download with token for git-fixtures org and DB +// 6. Online (live) download with token for git-fixtures repos and DB +// 7. Offline (recorded) download without token for src-d org and DB +// 8. Offline (recorded) download without token for src-d/gitbase repo and DB + +type DownloaderTestSuite struct { + suite.Suite + db *sql.DB + downloader *Downloader +} + +func (suite *DownloaderTestSuite) SetupSuite() { + if !isOSXOnTravis() { + suite.db = getDB(suite.T()) + suite.NotNil(suite.db) + } +} + +// TestOnlineRepositoryDownload Tests the download of known and fixed GitHub repositories +func (suite *DownloaderTestSuite) TestOnlineRepositoryDownload() { + t := suite.T() + checkToken(t) + tests, err := loadTests(onlineRepoTests) + suite.NoError(err, "Failed to read the testcases") + downloader, storer, err := getDownloader() + suite.NoError(err, "Failed to instantiate downloader") + for _, test := range tests.RepositoryTests { + test := test // pinned, see scopelint for more info + t.Run(fmt.Sprintf("Repo: %s/%s", test.Owner, test.Repository), func(t *testing.T) { + testRepo(t, test, downloader, storer, false) + }) + } +} + +func testRepo(t *testing.T, oracle testutils.RepositoryTest, d *Downloader, storer *testutils.Memory, strict bool) { err := d.DownloadRepository(context.TODO(), oracle.Owner, oracle.Repository, oracle.Version) require := require.New(t) // Make a new require object for the specified test, so no need to pass it around require.Nil(err) @@ -96,35 +183,47 @@ func testOnlineRepo(t *testing.T, oracle RepositoryTest, d *Downloader, storer * require.Equal(oracle.IsPrivate, storer.Repository.IsPrivate) require.Equal(oracle.IsArchived, storer.Repository.IsArchived) require.Equal(oracle.HasWiki, storer.Repository.HasWikiEnabled) - require.Len(storer.PRs, oracle.NumOfPRs) - require.Len(storer.PRComments, oracle.NumOfPRComments) + require.ElementsMatch(oracle.Topics, storer.Topics) + numOfPRReviews, numOfPRReviewComments := storer.CountPRReviewsAndReviewComments() + if strict { + require.Len(storer.PRs, oracle.NumOfPRs) + require.Len(storer.PRComments, oracle.NumOfPRComments) + require.Len(storer.Issues, oracle.NumOfIssues) + require.Len(storer.IssueComments, oracle.NumOfIssueComments) + require.Equal(oracle.NumOfPRReviews, numOfPRReviews) + require.Equal(oracle.NumOfPRReviewComments, numOfPRReviewComments) + } else { + require.GreaterOrEqual(oracle.NumOfPRs, len(storer.PRs)) + require.GreaterOrEqual(oracle.NumOfPRComments, len(storer.PRComments)) + require.GreaterOrEqual(oracle.NumOfIssues, len(storer.Issues)) + require.GreaterOrEqual(oracle.NumOfIssueComments, len(storer.IssueComments)) + require.GreaterOrEqual(oracle.NumOfPRReviews, numOfPRReviews) + require.GreaterOrEqual(oracle.NumOfPRReviewComments, numOfPRReviewComments) + } } -// TestOnlineRepositoryDownload Tests the download of known and fixed GitHub repositories -func TestOnlineRepositoryDownload(t *testing.T) { +// TestOnlineOrganizationDownload Tests the download of known and fixed GitHub organization +func (suite *DownloaderTestSuite) TestOnlineOrganizationDownload() { + t := suite.T() checkToken(t) - var err error - tests, err := loadOnlineTests("../testdata/online-repository-tests.json") - if err != nil { - t.Errorf("Failed to read the testcases:%s", err) - } - + tests, err := loadTests(onlineOrgTests) + suite.NoError(err, "Failed to read the testcases") downloader, storer, err := getDownloader() - require.NoError(t, err) - - for _, test := range tests.RepositoryTests { - t.Run(fmt.Sprintf("%s/%s", test.Owner, test.Repository), func(t *testing.T) { - testOnlineRepo(t, test, downloader, storer) + suite.NoError(err, "Failed to instantiate downloader") + for _, test := range tests.OrganizationsTests { + test := test + t.Run(fmt.Sprintf("Org: %s", test.Org), func(t *testing.T) { + testOrg(t, test, downloader, storer) }) } } -func testOnlineOrg(t *testing.T, oracle OrganizationTest, d *Downloader, storer *testutils.Memory) { +func testOrg(t *testing.T, oracle testutils.OrganizationTest, d *Downloader, storer *testutils.Memory) { err := d.DownloadOrganization(context.TODO(), oracle.Org, oracle.Version) require := require.New(t) require.Nil(err, "DownloadOrganization(%s) failed", oracle.Org) // Sample some properties that will not change, no topics available in git-fixtures - require.Equal(oracle.Org, storer.Organization.Name) + require.Equal(oracle.Org, storer.Organization.Login) require.Equal(oracle.URL, storer.Organization.URL) require.Equal(oracle.CreatedAt, storer.Organization.CreatedAt.String()) require.Equal(oracle.PublicRepos, storer.Organization.PublicRepos.TotalCount) @@ -132,22 +231,280 @@ func testOnlineOrg(t *testing.T, oracle OrganizationTest, d *Downloader, storer require.Len(storer.Users, oracle.NumOfUsers) } -// TestOnlineOrganizationDownload Tests the download of known and fixed GitHub organization -func TestOnlineOrganizationDownload(t *testing.T) { +func getRoundTripDownloader(reqResp map[string]string, storer storer) *Downloader { + return &Downloader{ + storer: storer, + client: githubv4.NewClient(&http.Client{ + Transport: RoundTripFunc(func(req *http.Request) *http.Response { + // consume request body + savecl := req.ContentLength + bodyBytes, _ := ioutil.ReadAll(req.Body) + defer req.Body.Close() + // recreate request body + req.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + req.ContentLength = savecl + data := reqResp[string(bodyBytes)] + return &http.Response{ + StatusCode: 200, + Body: ioutil.NopCloser(bytes.NewBufferString(data)), + Header: make(http.Header), + } + })}), + } +} + +// TestOfflineOrganizationDownload Tests a large organization by replaying recorded responses +func (suite *DownloaderTestSuite) TestOfflineOrganizationDownload() { + t := suite.T() + reqResp := make(map[string]string) + // Load the recording + suite.NoError(loadReqResp(orgRecFile, reqResp), "Failed to read the offline recordings") + // Setup the downloader with RoundTrip functionality. + // Not using the NewStdoutDownloader initialization because it overides the transport + storer := &testutils.Memory{} + downloader := getRoundTripDownloader(reqResp, storer) + tests, err := loadTests(offlineOrgTests) + suite.NoError(err, "Failed to read the offline tests") + for _, test := range tests.OrganizationsTests { + test := test + t.Run(fmt.Sprintf("Org: %s", test.Org), func(t *testing.T) { + testOrg(t, test, downloader, storer) + }) + } +} + +// TestOfflineRepositoryDownload Tests a large repository by replaying recorded responses +func (suite *DownloaderTestSuite) TestOfflineRepositoryDownload() { + t := suite.T() + reqResp := make(map[string]string) + suite.NoError(loadReqResp(repoRecFile, reqResp), "Failed to read the offline recordings") + storer := &testutils.Memory{} + downloader := getRoundTripDownloader(reqResp, storer) + tests, err := loadTests(offlineRepoTests) + suite.NoError(err, "Failed to read the offline tests") + for _, test := range tests.RepositoryTests { + test := test + t.Run(fmt.Sprintf("Repo: %s/%s", test.Owner, test.Repository), func(t *testing.T) { + testRepo(t, test, downloader, storer, true) + }) + } +} + +func testOrgWithDB(t *testing.T, oracle testutils.OrganizationTest, d *Downloader, db *sql.DB) { + err := d.DownloadOrganization(context.TODO(), oracle.Org, oracle.Version) + require := require.New(t) // Make a new require object for the specified test, so no need to pass it around + require.NoError(err, "Error in downloading") + var ( + htmlurl string + createdAt time.Time + numOfPublicRepos int + numOfPrivateRepos int + numOfUsers int + ) + // Retrieve data + err = db.QueryRow("select htmlurl, created_at, public_repos, total_private_repos from organizations where login = $1", oracle.Org).Scan(&htmlurl, &createdAt, &numOfPublicRepos, &numOfPrivateRepos) + require.NoError(err, "Error in retrieving orgs") + // TODO(@kyrcha): when schema is updated add query: select count(*) from users where owner = "src-d" for example + err = db.QueryRow("select count(*) from users").Scan(&numOfUsers) + require.NoError(err, "Error in retrieving users") + // Checks + require.Equal(oracle.URL, htmlurl) + require.Equal(oracle.CreatedAt, createdAt.String()) + require.Equal(oracle.PublicRepos, numOfPublicRepos) + require.Equal(oracle.TotalPrivateRepos, numOfPrivateRepos) + require.Equal(oracle.NumOfUsers, numOfUsers) +} + +// TestOnlineOrganizationDownloadWithDB Tests the download of known and fixed GitHub repositories and stores them in a Postgresql DB +func (suite *DownloaderTestSuite) TestOnlineOrganizationDownloadWithDB() { + t := suite.T() checkToken(t) - var err error - tests, err := loadOnlineTests("../testdata/online-organization-tests.json") - if err != nil { - t.Errorf("Failed to read the testcases:%s", err) + tests, err := loadTests(onlineOrgTests) + suite.NoError(err, "Failed to read the online tests") + downloader, err := NewDownloader(oauth2.NewClient( + context.TODO(), + oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + )), suite.db) + suite.NoError(err, "Failed to init the downloader") + downloader.SetActiveVersion(context.TODO(), 0) + suite.downloader = downloader + for _, test := range tests.OrganizationsTests { + test := test + t.Run(fmt.Sprintf("Org %s with DB", test.Org), func(t *testing.T) { + testOrgWithDB(t, test, downloader, suite.db) + }) } +} - downloader, storer, err := getDownloader() - require.NoError(t, err) +func testRepoWithDB(t *testing.T, oracle testutils.RepositoryTest, d *Downloader, db *sql.DB, strict bool) { + err := d.DownloadRepository(context.TODO(), oracle.Owner, oracle.Repository, oracle.Version) + require := require.New(t) // Make a new require object for the specified test, so no need to pass it around + require.Nil(err) + checkRepo(require, db, oracle, strict) + checkIssues(require, db, oracle, strict) + checkIssuePRComments(require, db, oracle, strict) + checkPRs(require, db, oracle, strict) + checkPRReviewComments(require, db, oracle, strict) +} + +func checkIssues(require *require.Assertions, db *sql.DB, oracle testutils.RepositoryTest, strict bool) { + var numOfIssues int + err := db.QueryRow("select count(*) from issues where repository_owner = $1 and repository_name = $2", oracle.Owner, oracle.Repository).Scan(&numOfIssues) + require.NoError(err, "Error in retrieving issues") + if strict { + require.Equal(oracle.NumOfIssues, numOfIssues, "Issues") + } else { + require.GreaterOrEqual(oracle.NumOfIssues, numOfIssues, "Issues") + } + +} +func checkIssuePRComments(require *require.Assertions, db *sql.DB, oracle testutils.RepositoryTest, strict bool) { + var numOfComments int + err := db.QueryRow("select count(*) from issue_comments where repository_owner = $1 and repository_name = $2", oracle.Owner, oracle.Repository).Scan(&numOfComments) + require.NoError(err, "Error in retrieving issue comments") + // NB: ghsync saves both Issue and PRs comments in the same table, issue_comments => See store/db.go comment + if strict { + require.Equal(oracle.NumOfPRComments+oracle.NumOfIssueComments, numOfComments, "Issue and PR Comments") + } else { + require.GreaterOrEqual(oracle.NumOfPRComments+oracle.NumOfIssueComments, numOfComments, "Issue and PR Comments") + } +} + +func checkPRs(require *require.Assertions, db *sql.DB, oracle testutils.RepositoryTest, strict bool) { + var numOfPRs int + err := db.QueryRow("select count(*) from pull_requests where repository_owner = $1 and repository_name = $2", oracle.Owner, oracle.Repository).Scan(&numOfPRs) + require.NoError(err, "Error in retrieving pull requests") + if strict { + require.Equal(oracle.NumOfPRs, numOfPRs, "PRs") + } else { + require.GreaterOrEqual(oracle.NumOfPRs, numOfPRs, "PRs") + } +} + +func checkPRReviewComments(require *require.Assertions, db *sql.DB, oracle testutils.RepositoryTest, strict bool) { + var numOfPRReviewComments int + err := db.QueryRow("select count(*) from pull_request_comments where repository_owner = $1 and repository_name = $2", oracle.Owner, oracle.Repository).Scan(&numOfPRReviewComments) + require.NoError(err, "Error in retrieving pull request comments") + if strict { + require.Equal(oracle.NumOfPRReviewComments, numOfPRReviewComments, "PR Review Comments") + } else { + require.GreaterOrEqual(oracle.NumOfPRReviewComments, numOfPRReviewComments, "PR Review Comments") + } +} + +func checkRepo(require *require.Assertions, db *sql.DB, oracle testutils.RepositoryTest, strict bool) { + var ( + htmlurl string + createdAt time.Time + private bool + archived bool + hasWiki bool + topics []string + ) + err := db.QueryRow("select htmlurl, created_at, private, archived, has_wiki, topics from repositories where owner_login = $1 and name = $2", oracle.Owner, oracle.Repository).Scan(&htmlurl, &createdAt, &private, &archived, &hasWiki, pq.Array(&topics)) + require.NoError(err, "Error in retrieving repo") + require.Equal(oracle.URL, htmlurl) + require.Equal(oracle.CreatedAt, createdAt.String()) + require.Equal(oracle.IsPrivate, private) + require.Equal(oracle.IsArchived, archived) + require.Equal(oracle.HasWiki, hasWiki) + require.ElementsMatch(oracle.Topics, topics) +} + +// TestOnlineRepositoryDownloadWithDB Tests the download of known and fixed GitHub organization and stores it in a Postgresql DB +func (suite *DownloaderTestSuite) TestOnlineRepositoryDownloadWithDB() { + t := suite.T() + checkToken(t) + tests, err := loadTests(onlineRepoTests) + suite.NoError(err, "Failed to read the online tests") + downloader, err := NewDownloader(oauth2.NewClient( + context.TODO(), + oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + )), suite.db) + suite.NoError(err, "Failed to init the downloader") + downloader.SetActiveVersion(context.TODO(), 0) + suite.downloader = downloader + for _, test := range tests.RepositoryTests { + test := test + t.Run(fmt.Sprintf("Repo %s/%s with DB", test.Owner, test.Repository), func(t *testing.T) { + testRepoWithDB(t, test, downloader, suite.db, false) + }) + } +} + +// TestOfflineOrganizationDownloadWithDB Tests a large organization by replaying recorded responses and storing the results in Postgresql +func (suite *DownloaderTestSuite) TestOfflineOrganizationDownloadWithDB() { + t := suite.T() + reqResp := make(map[string]string) + // Load the recording + suite.NoError(loadReqResp(orgRecFile, reqResp), "Failed to read the recordings") + // Setup the downloader with RoundTrip functionality. + // Not using the NewStdoutDownloader initialization because it overides the transport + storer := &store.DB{DB: suite.db} + downloader := getRoundTripDownloader(reqResp, storer) + downloader.SetActiveVersion(context.TODO(), 0) // Will create the views + suite.downloader = downloader + tests, err := loadTests(offlineOrgTests) + suite.NoError(err, "Failed to read the offline tests") for _, test := range tests.OrganizationsTests { + test := test t.Run(fmt.Sprintf("%s", test.Org), func(t *testing.T) { - testOnlineOrg(t, test, downloader, storer) + testOrgWithDB(t, test, downloader, suite.db) + }) + } +} + +// TestOfflineRepositoryDownload Tests a large repository by replaying recorded responses and stores the results in postgresql +func (suite *DownloaderTestSuite) TestOfflineRepositoryDownloadWithDB() { + t := suite.T() + reqResp := make(map[string]string) + // Load the recording + suite.NoError(loadReqResp(repoRecFile, reqResp), "Failed to read the recordings") + storer := &store.DB{DB: suite.db} + downloader := getRoundTripDownloader(reqResp, storer) + downloader.SetActiveVersion(context.TODO(), 0) + suite.downloader = downloader + tests, err := loadTests(offlineRepoTests) + suite.NoError(err, "Failed to read the offline tests") + for _, test := range tests.RepositoryTests { + test := test + t.Run(fmt.Sprintf("%s/%s", test.Owner, test.Repository), func(t *testing.T) { + testRepoWithDB(t, test, downloader, suite.db, true) }) } +} + +func (suite *DownloaderTestSuite) BeforeTest(suiteName, testName string) { + if strings.HasSuffix(testName, "WithDB") && isOSXOnTravis() { + suite.T().Skip("Don't test OSX with docker psql") + } +} + +// AfterTest after specific tests that use the DB, cleans up the DB for later tests +func (suite *DownloaderTestSuite) AfterTest(suiteName, testName string) { + if testName == "TestOnlineOrganizationDownloadWithDB" || testName == "TestOfflineOrganizationDownloadWithDB" { + // I cleanup with a different version (1 vs. 0), so to clean all the data from the DB + suite.downloader.Cleanup(context.TODO(), 1) + // Check + var countOrgs int + err := suite.db.QueryRow("select count(*) from organizations").Scan(&countOrgs) + suite.NoError(err, "Failed to count the orgs") + suite.Equal(0, countOrgs) + } else if testName == "TestOnlineRepositoryDownloadWithDB" || testName == "TestOfflineRepositoryDownloadWithDB" { + // I cleanup with a different version (1 vs. 0), so to clean all the data from the DB + suite.downloader.Cleanup(context.TODO(), 1) + // Check + var countRepos int + err := suite.db.QueryRow("select count(*) from repositories").Scan(&countRepos) + suite.NoError(err, "Failed to count the repos") + suite.Equal(0, countRepos) + } +} +// Run the suite +func TestDownloaderTestSuite(t *testing.T) { + suite.Run(t, new(DownloaderTestSuite)) } diff --git a/go.mod b/go.mod index a6157f1..18ed56b 100644 --- a/go.mod +++ b/go.mod @@ -3,29 +3,38 @@ module github.com/src-d/metadata-retrieval go 1.12 require ( + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect + github.com/cenkalti/backoff v2.2.1+incompatible github.com/golang-migrate/migrate/v4 v4.4.0 github.com/golang/protobuf v1.3.2 // indirect github.com/google/go-cmp v0.3.1 // indirect github.com/jessevdk/go-flags v1.4.0 // indirect github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d // indirect + github.com/kyoh86/scopelint v0.1.2 // indirect github.com/lib/pq v1.1.1 github.com/mattn/go-colorable v0.1.2 // indirect - github.com/mattn/go-isatty v0.0.9 // indirect + github.com/mattn/go-isatty v0.0.10 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect + github.com/motemen/go-loghttp v0.0.0-20170804080138-974ac5ceac27 + github.com/motemen/go-nuts v0.0.0-20190725124253-1d2432db96b0 // indirect github.com/onsi/ginkgo v1.10.0 // indirect github.com/onsi/gomega v1.7.0 // indirect github.com/shurcooL/githubv4 v0.0.0-20190718010115-4ba037080260 github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect github.com/sirupsen/logrus v1.4.2 // indirect github.com/src-d/envconfig v1.0.0 // indirect + github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.4.0 github.com/x-cray/logrus-prefixed-formatter v0.5.2 // indirect golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 // indirect golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 - golang.org/x/sys v0.0.0-20190830080133-08d80c9d36de // indirect + golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f // indirect golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3 // indirect google.golang.org/appengine v1.6.2 // indirect + gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/src-d/go-cli.v0 v0.0.0-20190821111025-f9dec40d74d8 gopkg.in/src-d/go-log.v1 v1.0.2 + gopkg.in/yaml.v2 v2.2.4 // indirect ) diff --git a/go.sum b/go.sum index 4d2c324..a16ce5d 100644 --- a/go.sum +++ b/go.sum @@ -10,13 +10,22 @@ github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEV github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= +github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= +github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= @@ -120,19 +129,28 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kshvakov/clickhouse v1.3.5/go.mod h1:DMzX7FxRymoNkVgizH0DWAL8Cur7wHLgx3MUnGwJqpE= +github.com/kyoh86/scopelint v0.1.2 h1:FmRDsHLdY/Y2A+4rdvB3FTdVYIkPRzb5CLmTIYmEW7c= +github.com/kyoh86/scopelint v0.1.2/go.mod h1:veFgnmDG8sPR5nFaXGX2mEIOXKHjWpGo79v/NaiTcRE= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU= +github.com/motemen/go-loghttp v0.0.0-20170804080138-974ac5ceac27 h1:uAI3rnOT1OSSY4PUtI/M1orb3q0ewkovwd3wr8xSno4= +github.com/motemen/go-loghttp v0.0.0-20170804080138-974ac5ceac27/go.mod h1:6eu9CfGt5kfrMVgeu9MfB9PRUnpc47I+udLswiTszI8= +github.com/motemen/go-nuts v0.0.0-20190725124253-1d2432db96b0 h1:CnSVrlMNAZMWI1+uH6ldpXRv2pe7t50IQX448EJrJhw= +github.com/motemen/go-nuts v0.0.0-20190725124253-1d2432db96b0/go.mod h1:vfh/NPxHgDwggXit20W1llPsXcz39xJ7I8vo7kVrOCk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -162,6 +180,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shurcooL/githubv4 v0.0.0-20190718010115-4ba037080260 h1:xKXiRdBUtMVp64NaxACcyX4kvfmHJ9KrLU+JvyB1mdM= github.com/shurcooL/githubv4 v0.0.0-20190718010115-4ba037080260/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= @@ -175,6 +194,7 @@ github.com/src-d/envconfig v1.0.0 h1:/AJi6DtjFhZKNx3OB2qMsq7y4yT5//AeSZIe7rk+PX8 github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -242,6 +262,9 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190830080133-08d80c9d36de h1:MtIqW4Vp7DcnuoJTsTOgsa2R3jBQnCU0bjwXo7DcNT8= golang.org/x/sys v0.0.0-20190830080133-08d80c9d36de/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f h1:hjzMYz/7Ea1mNKfOnFOfktR0mlA5jqhvywClCMHM/qw= +golang.org/x/sys v0.0.0-20191009170203-06d7bd2c5f4f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -279,10 +302,13 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +gopkg.in/alecthomas/kingpin.v2 v2.2.5/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -295,6 +321,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/testdata/offline-organization-tests.json b/testdata/offline-organization-tests.json new file mode 100644 index 0000000..ef68432 --- /dev/null +++ b/testdata/offline-organization-tests.json @@ -0,0 +1,14 @@ +{ + "organizationTests" : + [ + { + "org": "src-d", + "version": 0, + "url": "https://github.com/src-d", + "createdAt": "2015-10-14 17:13:24 +0000 UTC", + "publicRepos": 145, + "totalPrivateRepos": 41, + "numOfUsers": 54 + } + ] +} diff --git a/testdata/offline-repository-tests.json b/testdata/offline-repository-tests.json new file mode 100644 index 0000000..3d3db05 --- /dev/null +++ b/testdata/offline-repository-tests.json @@ -0,0 +1,22 @@ +{ + "repositoryTests" : + [ + { + "owner": "src-d", + "repository": "gitbase", + "version": 0, + "url": "https://github.com/src-d/gitbase", + "createdAt": "2016-09-19 10:14:13 +0000 UTC", + "topics": ["sql", "git", "sql-interface", "golang", "ast"], + "isPrivate": false, + "isArchived": false, + "hasWiki": false, + "numOfPrs": 548, + "numOfPrComments": 746, + "numOfIssues": 398, + "numOfIssueComments": 965, + "numOfPRReviews": 1512, + "numOfPRReviewComments": 668 + } + ] +} diff --git a/testdata/organization_src-d_2019-10-14.gob.gz b/testdata/organization_src-d_2019-10-14.gob.gz new file mode 100644 index 0000000..b152f49 Binary files /dev/null and b/testdata/organization_src-d_2019-10-14.gob.gz differ diff --git a/testdata/repository_src-d_gitbase_2019-10-14.gob.gz b/testdata/repository_src-d_gitbase_2019-10-14.gob.gz new file mode 100644 index 0000000..80a0a17 Binary files /dev/null and b/testdata/repository_src-d_gitbase_2019-10-14.gob.gz differ diff --git a/testutils/memory.go b/testutils/memory.go index e90d735..a8c13db 100644 --- a/testutils/memory.go +++ b/testutils/memory.go @@ -10,12 +10,28 @@ import ( // Memory implements the storer interface type Memory struct { - Organization *graphql.Organization - Repository *graphql.RepositoryFields - Topics []string - Users []*graphql.UserExtended - PRs []*graphql.PullRequest - PRComments []*graphql.IssueComment + Organization *graphql.Organization + Repository *graphql.RepositoryFields + Topics []string + Users []*graphql.UserExtended + Issues []*graphql.Issue + IssueComments []*graphql.IssueComment + PRs []*graphql.PullRequest + PRComments []*graphql.IssueComment + PRReviews map[int][]*graphql.PullRequestReview + PRReviewComments map[int]map[int][]*graphql.PullRequestReviewComment +} + +// CountPRReviewsAndReviewComments returns the number of PR reviews and PR review comments +func (s *Memory) CountPRReviewsAndReviewComments() (int, int) { + var reviewsCounter, commentsCounter int + for k, v := range s.PRReviews { + reviewsCounter += len(v) + for _, vv := range s.PRReviewComments[k] { + commentsCounter += len(vv) + } + } + return reviewsCounter, commentsCounter } // SaveOrganization stores an organization in memory, @@ -42,29 +58,34 @@ func (s *Memory) SaveRepository(ctx context.Context, repository *graphql.Reposit s.Repository = repository s.Topics = topics // Initialize prs and comments to 0 for each repo + s.Issues = make([]*graphql.Issue, 0) + s.IssueComments = make([]*graphql.IssueComment, 0) s.PRs = make([]*graphql.PullRequest, 0) s.PRComments = make([]*graphql.IssueComment, 0) + s.PRReviews = make(map[int][]*graphql.PullRequestReview) + s.PRReviewComments = make(map[int]map[int][]*graphql.PullRequestReviewComment) return nil } -// TODO(kyrcha): add memory in noop methods as the tests expand - -// SaveIssue noop +// SaveIssue appends an issue to the issue list in memory func (s *Memory) SaveIssue(ctx context.Context, repositoryOwner, repositoryName string, issue *graphql.Issue, assignees []string, labels []string) error { log.Infof("issue data fetched for #%v %s\n", issue.Number, issue.Title) + s.Issues = append(s.Issues, issue) return nil } -// SaveIssueComment noop +// SaveIssueComment appends an issue comment to the issue comments list in memory func (s *Memory) SaveIssueComment(ctx context.Context, repositoryOwner, repositoryName string, issueNumber int, comment *graphql.IssueComment) error { - log.Infof(" \tissue comment data fetched by %s at %v: %q\n", comment.Author.Login, comment.CreatedAt, trim(comment.Body)) + log.Infof("\tissue comment data fetched by %s at %v: %q\n", comment.Author.Login, comment.CreatedAt, trim(comment.Body)) + s.IssueComments = append(s.IssueComments, comment) return nil } -// SavePullRequest appends an PR to the PR list in memory +// SavePullRequest appends an PR to the PR list in memory, also initializes the map for the review commnets for that particular PR func (s *Memory) SavePullRequest(ctx context.Context, repositoryOwner, repositoryName string, pr *graphql.PullRequest, assignees []string, labels []string) error { log.Infof("PR data fetched for #%v %s\n", pr.Number, pr.Title) s.PRs = append(s.PRs, pr) + s.PRReviewComments[pr.Number] = make(map[int][]*graphql.PullRequestReviewComment) return nil } @@ -75,15 +96,17 @@ func (s *Memory) SavePullRequestComment(ctx context.Context, repositoryOwner, re return nil } -// SavePullRequestReview noop +// SavePullRequestReview appends a PR review to the PR review list in memory func (s *Memory) SavePullRequestReview(ctx context.Context, repositoryOwner, repositoryName string, pullRequestNumber int, review *graphql.PullRequestReview) error { - log.Infof(" \tPR Review data fetched by %s at %v: %q\n", review.Author.Login, review.SubmittedAt, trim(review.Body)) + log.Infof("\tPR Review data fetched by %s at %v: %q\n", review.Author.Login, review.SubmittedAt, trim(review.Body)) + s.PRReviews[pullRequestNumber] = append(s.PRReviews[pullRequestNumber], review) return nil } -// SavePullRequestReviewComment noop +// SavePullRequestReviewComment appends a PR review comment to the PR review comments list in memory func (s *Memory) SavePullRequestReviewComment(ctx context.Context, repositoryOwner, repositoryName string, pullRequestNumber int, pullRequestReviewID int, comment *graphql.PullRequestReviewComment) error { log.Infof("\t\tPR review comment data fetched by %s at %v: %q\n", comment.Author.Login, comment.CreatedAt, trim(comment.Body)) + s.PRReviewComments[pullRequestNumber][pullRequestReviewID] = append(s.PRReviewComments[pullRequestNumber][pullRequestReviewID], comment) return nil } @@ -120,6 +143,5 @@ func trim(s string) string { if len(s) > 40 { return s[0:39] + "..." } - return s } diff --git a/testutils/teststructs.go b/testutils/teststructs.go new file mode 100644 index 0000000..dd3f466 --- /dev/null +++ b/testutils/teststructs.go @@ -0,0 +1,37 @@ +package testutils + +// RepositoryTest struct to hold a test oracle for a repository +type RepositoryTest struct { + Owner string `json:"owner"` + Repository string `json:"repository"` + Version int `json:"version"` + URL string `json:"url"` + Topics []string `json:"topics"` + CreatedAt string `json:"createdAt"` + IsPrivate bool `json:"isPrivate"` + IsArchived bool `json:"isArchived"` + HasWiki bool `json:"hasWiki"` + NumOfPRs int `json:"numOfPrs"` + NumOfPRComments int `json:"numOfPrComments"` + NumOfIssues int `json:"numOfIssues"` + NumOfIssueComments int `json:"numOfIssueComments"` + NumOfPRReviews int `json:"numOfPRReviews"` + NumOfPRReviewComments int `json:"numOfPRReviewComments"` +} + +// OrganizationTest struct to hold a test oracle for an organization +type OrganizationTest struct { + Org string `json:"org"` + Version int `json:"version"` + URL string `json:"url"` + CreatedAt string `json:"createdAt"` + PublicRepos int `json:"publicRepos"` + TotalPrivateRepos int `json:"totalPrivateRepos"` + NumOfUsers int `json:"numOfUsers"` +} + +// Tests struct to hold the tests from json files +type Tests struct { + RepositoryTests []RepositoryTest `json:"repositoryTests"` + OrganizationsTests []OrganizationTest `json:"organizationTests"` +}