Skip to content

Commit

Permalink
Merge pull request #105 from AutoIDM/snowflake-issues
Browse files Browse the repository at this point in the history 
Added a ref Resolver
  • Loading branch information
@visch
visch authored Mar 3, 2022
2 parents 79153c6 + e606837 commit 6452c25
Show file tree
Hide file tree
Showing 13 changed files with 327 additions and 157 deletions.
7 changes: 7 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,13 @@ tap-clickup --config CONFIG --discover > ./catalog.json

## Developer Resources

### Schema Debugging
We are waiting on https://gitlab.com/meltano/sdk/-/issues/299 to get fixed as we make usage of refs in our json schema. Until then we parse the schemas in client.py

Sometimes it's useful to debug how the refs are being resolved. To do that there's a script inside of ./schema-parser , README in that directory describes how to run the parser

Not worry about making this super fool proof as we expect the SDK to release some kind of fix for this

### Initialize your Development Environment

```bash
Expand Down
23 changes: 19 additions & 4 deletions meltano.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ plugins:
- name: api_token
kind: password
select:
- '!shared_hierarchy.*'
- '*.*'
loaders:
- name: target-jsonl
Expand All @@ -33,11 +34,25 @@ plugins:
pip_url: target-stitch
executable: target-stitch
settings:
- name: token
kind: password
- name: token
kind: password
config:
client_id: 191229
small_batch_url: https://api.stitchdata.com/v2/import/batch
big_batch_url: https://api.stitchdata.com/v2/import/batch
batch_size_preferences:
a : "a"
batch_size_preferences:
a: a
- name: target-snowflake
variant: transferwise
pip_url: pipelinewise-target-snowflake
config:
account: yja98422
database: test
user: tap_clickup
warehouse: COMPUTE_WH
s3_bucket: snowflake-clickup-testing
stage: tap_clickup.tap_clickup
file_format: tap_clickup.tap_clickup
s3_region_name: us-west-2
default_target_schema: tap_clickup
primary_key_required: False
7 changes: 7 additions & 0 deletions schema-parser/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
To run the schema parser
1. python -m venv .venv
1. source .venv/bin/activate
1. pip install -r requirements.txt
1. python resolver.py

Data will land in ./parsed_schemas
11 changes: 11 additions & 0 deletions schema-parser/requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
attrs==21.4.0
backoff==1.11.1
ciso8601==2.2.0
jsonschema==3.2.0
orjson==3.6.1
pipelinewise-singer-python==2.0.1
pkg_resources==0.0.0
pyrsistent==0.18.0
python-dateutil==2.8.2
pytz==2021.3
six==1.16.0
18 changes: 18 additions & 0 deletions schema-parser/resolver.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
import singer
import json
import os

#list of files to transform
file_names = []
for file in os.listdir("."):
if file.endswith(".json"):
file_names.append(file)

for schema_file in file_names:
with open(schema_file) as f:
old_schema = json.load(f)
new_schema = singer.resolve_schema_references(old_schema)
with open(f"./parsed_schemas/{schema_file}", "w") as w:
w.write(json.dumps(new_schema, indent=4))


12 changes: 12 additions & 0 deletions tap_clickup/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import time
import requests
import backoff
import singer
from requests.exceptions import RequestException
from singer_sdk.helpers.jsonpath import extract_jsonpath
from singer_sdk.streams import RESTStream
Expand All @@ -20,6 +21,17 @@ class ClickUpStream(RESTStream):
records_jsonpath = "$[*]" # Or override `parse_response`.
next_page_token_jsonpath = "$.next_page" # Or override `get_next_page_token`.

@property
def schema(self) -> dict:
"""Get schema.
We are waiting on https://gitlab.com/meltano/sdk/-/issues/299 this works
well until then
Returns:
JSON Schema dictionary for this stream.
"""
return singer.resolve_schema_references(self._schema)

def get_url_params(
self, context: Optional[dict], next_page_token: Optional[Any]
) -> Dict[str, Any]:
Expand Down
2 changes: 1 addition & 1 deletion tap_clickup/schemas/custom_field.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@
"type": ["null", "string"]
},
"orderindex": {
"type": ["null", "integer"]
"type": ["null", "integer", "string"]
},
"label": {
"type": ["null", "string"]
Expand Down
35 changes: 20 additions & 15 deletions tap_clickup/schemas/folder.json
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
{
"definitions": {
"status": {
"type": "object",
"type": ["object", "null"],
"properties": {
"id": {
"type": "string"
"type": ["null", "string"]
},
"status": {
"type": "string"
"type": ["null", "string"]
},
"orderindex": {
"type": ["null", "string"]
"type": ["null", "integer", "string"]
},
"color": {
"type": ["null", "string"]
Expand All @@ -21,16 +21,16 @@
}
},
"list": {
"type": "object",
"type": ["object", "null"],
"properties": {
"id": {
"type": ["string"]
"type": ["string", "null"]
},
"name": {
"type": ["null", "string"]
},
"orderindex": {
"type": ["null", "integer"]
"type": ["null", "integer", "string"]
},
"status": {
"type": ["null", "string"]
Expand All @@ -53,19 +53,24 @@
"space": {
"type": ["null", "object"],
"properties": {
"$ref": "#definitions/feature"
"id": {
"type": ["null", "string"]
},
"name": {
"type": ["null", "string"]
}
}
},
"archived": {
"type": ["null", "boolean"]
},
"override_statuses": {
"type": ["null", "string"]
"type": ["null", "boolean", "string"]
},
"statuses": {
"type": ["array","null"],
"items": {
"$ref": "#definitions/status"
"$ref": "#/definitions/status"
}
},
"permission_level": {
Expand All @@ -83,16 +88,16 @@
"type": ["null", "string"]
},
"orderindex": {
"type": ["null", "integer"]
"type": ["null", "string", "integer"]
},
"override_statuses": {
"type": ["null", "boolean"]
"type": ["null", "string", "boolean"]
},
"hidden": {
"type": ["null", "boolean"]
},
"space": {
"type": "object",
"type": ["object", "null"],
"properties": {
"id": {
"type": ["null", "string"]
Expand All @@ -108,13 +113,13 @@
"statuses": {
"type": ["array", "null"],
"items": {
"$ref": "#definitions/status"
"$ref": "#/definitions/status"
}
},
"lists": {
"type": ["array", "null"],
"items": {
"$ref": "#definitions/list"
"$ref": "#/definitions/list"
}
},
"archived": {
Expand Down
22 changes: 11 additions & 11 deletions tap_clickup/schemas/list.json
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"definitions": {
"folder": {
"type": "object",
"type": ["object", "null"],
"properties": {
"id": {
"type": "string"
"type": ["null", "string"]
},
"name": {
"type": ["null", "string"]
Expand All @@ -18,16 +18,16 @@
}
},
"status": {
"type": "object",
"type": ["object", "null"],
"properties": {
"id": {
"type": "string"
"type": ["null", "string"]
},
"status": {
"type": "string"
"type": ["null", "string"]
},
"orderindex": {
"type": ["null", "string"]
"type": ["null", "string", "integer"]
},
"color": {
"type": ["null", "string"]
Expand All @@ -47,13 +47,13 @@
"type": ["null", "string"]
},
"orderindex": {
"type": ["null", "integer"]
"type": ["null", "integer", "string"]
},
"content": {
"type": ["null", "string"]
},
"status": {
"$ref": "#definitions/status"
"$ref": "#/definitions/status"
},
"priority": {
"type": ["string", "object", "null"],
Expand All @@ -65,7 +65,7 @@
"type": ["string"]
},
"orderindex":{
"type": ["string"]
"type": ["string", "null", "integer"]
},
"priority":{
"type": ["string"]
Expand Down Expand Up @@ -105,7 +105,7 @@
"type": ["null", "string"]
},
"folder": {
"$ref": "#definitions/folder"
"$ref": "#/definitions/folder"
},
"space": {
"type": ["null", "object"],
Expand All @@ -125,7 +125,7 @@
"type": ["null", "boolean"]
},
"override_statuses": {
"type": ["null", "boolean"]
"type": ["null", "string", "boolean"]
},
"permission_level": {
"type": ["null", "string"]
Expand Down
20 changes: 10 additions & 10 deletions tap_clickup/schemas/shared.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@
"type": ["object", "null"],
"properties": {
"id": {
"type": "string"
"type": ["null", "string"]
},
"status": {
"type": "string"
"type": ["null", "string"]
},
"orderindex": {
"type": ["null", "string"]
"type": ["null", "string", "integer"]
},
"color": {
"type": ["null", "string"]
Expand All @@ -35,18 +35,18 @@
"type": ["string", "null"]
},
"orderindex": {
"type": ["integer", "null"]
"type": ["integer", "string", "null"]
},
"content": {
"type": ["string", "null"]
},
"status": {
"$ref": "#definitions/status"
"$ref": "#/definitions/status"
},
"statuses": {
"type": ["array", "null"],
"items": {
"$ref": "#definitions/status"
"$ref": "#/definitions/status"
}
},
"priority": {
Expand Down Expand Up @@ -82,18 +82,18 @@
"type": ["string", "null"]
},
"orderindex": {
"type": ["integer", "null"]
"type": ["integer", "string", "null"]
},
"content": {
"type": ["string", "null"]
},
"status": {
"$ref": "#definitions/status"
"$ref": "#/definitions/status"
},
"statuses": {
"type": ["array", "null"],
"items": {
"$ref": "#definitions/status"
"$ref": "#/definitions/status"
}
},
"priority": {
Expand Down Expand Up @@ -129,7 +129,7 @@
"type": ["string", "null"]
},
"orderindex": {
"type": ["integer", "null"]
"type": ["integer", "string", "null"]
},
"content": {
"type": ["string", "null"]
Expand Down
Loading

0 comments on commit 6452c25

Please sign in to comment.