Skip to content

Commit

Permalink
Resolve Start Date test case failure (singer-io#144)
Browse files Browse the repository at this point in the history 
* resolve start date test case failure

* added comment

* implement crud for start date test

* updated the start date for events stream

* updated the code according to the comments
  • Loading branch information
@hpatel41
hpatel41 authored Nov 12, 2021
1 parent aaad1ff commit 87caf2b
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 19 deletions.
51 changes: 37 additions & 14 deletions tests/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ class TestGithubBase(unittest.TestCase):
}
START_DATE = ""
FULL_TABLE_SUB_STREAMS = ['reviews', 'review_comments', 'pr_commits', 'team_members', 'team_memberships']
OBEYS_START_DATE = "obey-start-date"

def setUp(self):
missing_envs = [x for x in [
Expand Down Expand Up @@ -94,104 +95,126 @@ def expected_metadata(self):
"assignees": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.FULL,
self.OBEYS_START_DATE: False
},
"collaborators": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.FULL,
self.OBEYS_START_DATE: False
},
"comments": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"commit_comments": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"commits": {
self.PRIMARY_KEYS: {"sha"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"events": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"created_at"}
self.BOOKMARK: {"created_at"},
self.OBEYS_START_DATE: True
},
"issue_labels": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.FULL,
self.OBEYS_START_DATE: False
},
"issue_milestones": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"due_on"}
self.BOOKMARK: {"due_on"},
self.OBEYS_START_DATE: True
},
"issue_events": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"created_at"}
self.BOOKMARK: {"created_at"},
self.OBEYS_START_DATE: True
},
"issues": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"pr_commits": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"project_cards": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"project_columns": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"projects": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"pull_requests": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"releases": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.FULL,
self.OBEYS_START_DATE: False
},
"review_comments": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"reviews": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.INCREMENTAL,
self.BOOKMARK: {"updated_at"}
self.BOOKMARK: {"updated_at"},
self.OBEYS_START_DATE: True
},
"stargazers": {
self.PRIMARY_KEYS: {"user_id"},
self.REPLICATION_METHOD: self.FULL,
self.OBEYS_START_DATE: False
},
"team_members": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.FULL,
self.OBEYS_START_DATE: False
},
"team_memberships": {
self.PRIMARY_KEYS: {"url"},
self.REPLICATION_METHOD: self.FULL,
self.OBEYS_START_DATE: False
},
"teams": {
self.PRIMARY_KEYS: {"id"},
self.REPLICATION_METHOD: self.FULL,
self.OBEYS_START_DATE: False
}
}

Expand Down
41 changes: 36 additions & 5 deletions tests/test_github_start_date.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import os

import requests
from tap_tester import connections, runner

from base import TestGithubBase
from datetime import datetime, timedelta


class GithubStartDateTest(TestGithubBase):
Expand All @@ -14,18 +15,47 @@ class GithubStartDateTest(TestGithubBase):
def name():
return "tap_tester_github_start_date_test"

def generate_data(self):
# get the token
token = os.getenv("TAP_GITHUB_TOKEN")
url = "https://api.github.com/user/starred/singer-io/tap-github"
headers = {"Authorization": "Bearer {}".format(token)}

# generate a data for 'events' stream: 'watchEvent' ie. star the repo
requests.put(url=url, headers=headers)
# as per the Documentation: https://docs.github.com/en/developers/webhooks-and-events/events/github-event-types#watchevent
# the event is generated when we 'star' a repo, hence 'unstar' it as we can 'star' it next time
requests.delete(url=url, headers=headers)

def test_run(self):
# generate data for 'events' stream
self.generate_data()

# run the test for all the streams excluding 'events' stream
# as for 'events' stream we have to use dynamic dates
self.run_test('2020-04-01T00:00:00Z', '2021-06-10T00:00:00Z', self.expected_streams() - {'events'})

# As per the Documentation: https://docs.github.com/en/rest/reference/activity#events
# the 'events' of past 90 days will only be returned
# if there are no events in past 90 days, then there will be '304 Not Modified' error
today = datetime.today()
date_1 = datetime.strftime(today - timedelta(days=4), "%Y-%m-%dT00:00:00Z")
date_2 = datetime.strftime(today - timedelta(days=1), "%Y-%m-%dT00:00:00Z")
# run the test for 'events' stream
self.run_test(date_1, date_2, {'events'})

def run_test(self, date_1, date_2, streams):
"""Instantiate start date according to the desired data set and run the test"""

self.start_date_1 = '2020-04-01T00:00:00Z'
self.start_date_2 = '2021-06-10T00:00:00Z'
self.start_date_1 = date_1
self.start_date_2 = date_2

start_date_1_epoch = self.dt_to_ts(self.start_date_1)
start_date_2_epoch = self.dt_to_ts(self.start_date_2)

self.START_DATE = self.start_date_1

expected_streams = self.expected_streams()
expected_streams = streams

##########################################################################
### First Sync
Expand Down Expand Up @@ -91,6 +121,7 @@ def test_run(self):
# expected values
expected_primary_keys = self.expected_primary_keys()[stream]
expected_bookmark_keys = self.expected_bookmark_keys()[stream]
expected_metadata = self.expected_metadata()[stream]

# collect information for assertions from syncs 1 & 2 base on expected values
record_count_sync_1 = record_count_by_stream_1.get(stream, 0)
Expand All @@ -105,7 +136,7 @@ def test_run(self):
primary_keys_sync_1 = set(primary_keys_list_1)
primary_keys_sync_2 = set(primary_keys_list_2)

if self.is_incremental(stream):
if expected_metadata.get(self.OBEYS_START_DATE):

# Sub stream fetch all data for records of related incremental super stream.
# Data of commit doesn't contain created_at or updated_at field.
Expand Down

0 comments on commit 87caf2b

Please sign in to comment.