diff --git a/dbt/adapters/upsolver/__version__.py b/dbt/adapters/upsolver/__version__.py index 162e338..f84daae 100644 --- a/dbt/adapters/upsolver/__version__.py +++ b/dbt/adapters/upsolver/__version__.py @@ -1 +1 @@ -version = "1.5.25" +version = "1.5.26" diff --git a/dbt/adapters/upsolver/impl.py b/dbt/adapters/upsolver/impl.py index ef665c1..3944a16 100644 --- a/dbt/adapters/upsolver/impl.py +++ b/dbt/adapters/upsolver/impl.py @@ -88,12 +88,36 @@ def render_option_from_dict(self, option_value): item.append(' ,'.join(value)) item.append(')') else: + item.append("'") if key.lower() == 'column' else False item.append(value) + item.append("'") if key.lower() == 'column' else False res.append(''.join(item)) return f"({' ,'.join(res)})" except Exception: raise dbt.exceptions.ParsingError(f"Error while parsing value: {value}. Expected type: dictionary") + def render_option_from_dict_str(self, option_value): + try: + res = [] + for key, value in option_value.items(): + item = [f'{key}='] + item.append("'") + item.append(value) + item.append("'") + res.append(''.join(item)) + return f"({' ,'.join(res)})" + except Exception: + raise dbt.exceptions.ParsingError(f"Error while parsing value: {value}. Expected type: dictionary") + + def render_option_from_list_dict(self, option_value): + try: + res = [] + for value in option_value: + res.append(self.render_option_from_dict(value)) + return f"({' ,'.join(res)})" + except Exception: + raise dbt.exceptions.ParsingError(f"Error while parsing value: {value}. Expected type: dictionary") + def render_option_from_list(self, option_value): try: if isinstance(option_value, list) and len(option_value) > 1: @@ -114,6 +138,10 @@ def enrich_options(self, config_options, source, options_type): value = self.render_option_from_list(value) elif options[option.lower()]['type'] == 'dict': value = self.render_option_from_dict(value) + elif options[option.lower()]['type'] == 'dict_str': + value = self.render_option_from_dict_str(value) + elif options[option.lower()]['type'] == 'list_dict': + value = self.render_option_from_list_dict(value) enriched_options[option] = find_value enriched_options[option]['value'] = value else: diff --git a/dbt/adapters/upsolver/options/connection_properties.md b/dbt/adapters/upsolver/options/connection_properties.md deleted file mode 100644 index 1bbd144..0000000 --- a/dbt/adapters/upsolver/options/connection_properties.md +++ /dev/null @@ -1,221 +0,0 @@ - - -## Connection options - -| Option | Storage | Type | Editable | Optional | Config Syntax | -| --------| --------- | ---- | -------- | -------- | ------------- | -| aws_role | s3 | text | True | True | 'aws_role': '' | -| external_id | s3 | text | True | True | 'external_id': '' | -| aws_access_key_id | s3 | text | True | True | 'aws_access_key_id': '' | -| aws_secret_access_key_id | s3 | text | True | True | 'aws_secret_access_key_id': '' | -| path_display_filter | s3 | text | True | True | 'path_display_filter': '' | -| path_display_filters | s3 | list | True | True | 'path_display_filters': ('', ...) | -| read_only | s3 | boolean | True | True | 'read_only': True/False | -| encryption_kms_key | s3 | text | True | True | 'encryption_kms_key': '' | -| encryption_customer_kms_key | s3 | text | True | True | 'encryption_customer_kms_key': '' | -| comment | s3 | text | True | True | 'comment': '' | -| host | kafka | text | False | False | 'host': '' | -| hosts | kafka | list | False | False | 'hosts': ('', ...) | -| consumer_properties | kafka | text | True | True | 'consumer_properties': '' | -| version | kafka | value | False | True | 'version': '' | -| require_static_ip | kafka | boolean | True | True | 'require_static_ip': True/False | -| ssl | kafka | boolean | True | True | 'ssl': True/False | -| topic_display_filter | kafka | text | True | True | 'topic_display_filter': '' | -| topic_display_filters | kafka | list | True | True | 'topic_display_filter': ('', ...) | -| comment | kafka | text | True | True | 'comment': '' | -| aws_role | glue_catalog | text | True | True | 'aws_role': '' | -| external_id | glue_catalog | text | True | True | 'external_id': '' | -| aws_access_key_id | glue_catalog | text | True | True | 'aws_access_key_id': '' | -| aws_secret_access_key | glue_catalog | text | True | True | 'aws_secret_access_key': '' | -| default_storage_connection | glue_catalog | identifier | False | False | 'default_storage_connection': '' | -| default_storage_location | glue_catalog | text | False | False | 'default_storage_location': '' | -| region | glue_catalog | text | False | True | 'region': '' | -| database_display_filter | glue_catalog | text | True | True | 'database_display_filter': '' | -| database_display_filters | glue_catalog | list | True | True | 'database_display_filters': ('', ...) | -| comment | glue_catalog | text | True | True | 'comment': '' | -| aws_role | kinesis | text | True | True | 'aws_role': '' | -| external_id | kinesis | text | True | True | 'external_id': '' | -| aws_access_key_id | kinesis | text | True | True | 'aws_access_key_id': '' | -| aws_secret_access_key | kinesis | text | True | True | 'aws_secret_access_key': '' | -| region | kinesis | text | False | False | 'region': '' | -| read_only | kinesis | boolean | False | True | 'read_only': True/False | -| max_writers | kinesis | integer | True | True | 'max_writers': | -| stream_display_filter | kinesis | text | True | True | 'stream_display_filter': '' | -| stream_display_filters | kinesis | list | True | True | 'stream_display_filters': ('', ...) | -| comment | kinesis | text | True | True | 'comment': '' | -| connection_string | snowflake | text | True | False | 'connection_string': '' | -| user_name | snowflake | text | True | False | 'user_name': '' | -| password | snowflake | text | True | False | 'password': '' | -| max_concurrent_connections | snowflake | integer | True | True | 'max_concurrent_connections': | -| comment | snowflake | text | True | True | 'comment': '' | -| connection_string | redshift | text | True | False | 'connection_string': '' | -| user_name | redshift | text | True | False | 'user_name': '' | -| password | redshift | text | True | False | 'password': '' | -| max_concurrent_connections | redshift | integer | True | True | 'max_concurrent_connections': | -| comment | redshift | text | True | True | 'comment': '' | -| connection_string | mysql | text | True | False | 'connection_string': '' | -| user_name | mysql | text | True | False | 'user_name': '' | -| password | mysql | text | True | False | 'password': '' | -| comment | mysql | text | True | True | 'comment': '' | -| connection_string | postgres | text | True | False | 'connection_string': '' | -| user_name | postgres | text | True | False | 'user_name': '' | -| password | postgres | text | True | False | 'password': '' | -| comment | postgres | text | True | True | 'comment': '' | -| connection_string | elasticsearch | text | True | False | 'connection_string': '' | -| user_name | elasticsearch | text | True | False | 'user_name': '' | -| password | elasticsearch | text | True | False | 'password': '' | -| comment | elasticsearch | text | True | True | 'comment': '' | - - -## Target options - -| Option | Storage | Type | Editable | Optional | Config Syntax | -| --------| --------- | ---- | -------- | -------- | ------------- | -| globally_unique_keys | datalake | boolean | False | True | 'globally_unique_keys': True/False | -| storage_connection | datalake | identifier | False | True | 'storage_connection': '' | -| storage_location | datalake | text | False | True | 'storage_location': '' | -| compute_cluster | datalake | identifier | True | True | 'compute_cluster': '' | -| compression | datalake | value | True | True | 'compression': 'SNAPPY/GZIP' | -| compaction_processes | datalake | integer | True | True | 'compaction_processes': | -| disable_compaction | datalake | boolean | True | True | 'disable_compaction': True/False | -| retention_date_partition | datalake | identifier | False | True | 'retention_date_partition': '' | -| table_data_retention | datalake | identifier | True | True | 'table_data_retention': '' | -| column_data_retention | datalake | dict | True | True | 'column_data_retention': {'COLUMN' : '','DURATION': N DAYS} | -| comment | datalake | text | True | True | 'comment': '' | -| storage_connection | materialized_view | identifier | False | True | 'storage_connection': '' | -| storage_location | materialized_view | text | False | True | 'storage_location': '' | -| max_time_travel_duration | materialized_view | identifier | True | True | 'max_time_travel_duration': '' | -| compute_cluster | materialized_view | identifier | True | True | 'compute_cluster': '' | -| column_transformations | snowflake | dict | False | True | 'column_transformations': {'' : '' , ...} | -| deduplicate_with | snowflake | dict | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | -| exclude_columns | snowflake | list | False | True | 'exclude_columns': ('', ...) | -| create_table_if_missing | snowflake | boolean | False | True | 'create_table_if_missing': True/False} | -| run_interval | snowflake | integer | False | True | 'run_interval': '' | - - -## Transformation options - -| Option | Storage | Type | Editable | Optional | Config Syntax | -| --------| --------- | ---- | -------- | -------- | ------------- | -| run_interval | s3 | ineger | False | True | 'run_interval': '' | -| start_from | s3 | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | s3 | value | True | True | 'end_at': '/NOW' | -| compute_cluster | s3 | identifier | True | True | 'compute_cluster': '' | -| comment | s3 | text | True | True | 'comment': '' | -| allow_cartesian_products | s3 | boolean | False | True | 'allow_cartesian_products': True/False | -| aggregation_parallelism | s3 | integer | True | True | 'aggregation_parallelism': | -| run_parallelism | s3 | integer | True | True | 'run_parallelism': | -| file_format | s3 | value | False | False | 'file_format': 'CSV/TSV ...' | -| compression | s3 | value | False | True | 'compression': 'SNAPPY/GZIP ...' | -| date_pattern | s3 | text | False | True | 'date_pattern': '' | -| output_offset | s3 | identifier | False | True | 'output_offset': '' | -| location | s3 | text | False | False | 'location': '' | -| run_interval | elasticsearch | identifier | False | True | 'run_interval': '' | -| start_from | elasticsearch | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | elasticsearch | value | True | True | 'end_at': '/NOW' | -| compute_cluster | elasticsearch | identifier | True | True | 'compute_cluster': '' | -| allow_cartesian_products | elasticsearch | boolean | False | True | 'allow_cartesian_products': True/False | -| aggregation_parallelism | elasticsearch | integer | True | True | 'aggregation_parallelism': | -| run_parallelism | elasticsearch | integer | True | True | 'run_parallelism': | -| bulk_max_size_bytes | elasticsearch | integer | True | True | 'bulk_max_size_bytes': | -| index_partition_size | elasticsearch | value | True | True | 'index_partition_size': 'HOURLY/DAILY ...' | -| comment | elasticsearch | text | True | True | 'comment': '' | -| custom_insert_expressions | snowflake | dict | True | True | 'custom_insert_expressions': {'INSERT_TIME' : 'CURRENT_TIMESTAMP()','MY_VALUE': ''} | -| custom_update_expressions | snowflake | dict | True | True | 'custom_update_expressions': {'UPDATE_TIME' : 'CURRENT_TIMESTAMP()','MY_VALUE': ''} | -| keep_existing_values_when_null | snowflake | boolean | True | True | 'keep_existing_values_when_null': True/False | -| add_missing_columns | snowflake | boolean | False | True | 'add_missing_columns': True/False | -| run_interval | snowflake | identifier | False | True | 'run_interval': '' | -| start_from | snowflake | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | snowflake | value | True | True | 'end_at': '/NOW' | -| compute_cluster | snowflake | identifier | True | True | 'compute_cluster': '' | -| allow_cartesian_products | snowflake | boolean | False | True | 'allow_cartesian_products': True/False | -| aggregation_parallelism | snowflake | integer | True | True | 'aggregation_parallelism': | -| run_parallelism | snowflake | integer | True | True | 'run_parallelism': | -| comment | snowflake | text | True | True | 'comment': '' | -| add_missing_columns | datalake | boolean | False | True | 'add_missing_columns': True/False | -| run_interval | datalake | identifier | False | True | 'run_interval': '' | -| start_from | datalake | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | datalake | value | True | True | 'end_at': '/NOW' | -| compute_cluster | datalake | identifier | True | True | 'compute_cluster': '' | -| allow_cartesian_products | datalake | boolean | False | True | 'allow_cartesian_products': True/False | -| aggregation_parallelism | datalake | integer | True | True | 'aggregation_parallelism': | -| run_parallelism | datalake | integer | True | True | 'run_parallelism': | -| comment | datalake | text | True | True | 'comment': '' | -| run_interval | redshift | identifier | False | True | 'run_interval': '' | -| start_from | redshift | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | redshift | value | True | True | 'end_at': '/NOW' | -| compute_cluster | redshift | identifier | True | True | 'compute_cluster': '' | -| allow_cartesian_products | redshift | boolean | False | True | 'allow_cartesian_products': True/False | -| aggregation_parallelism | redshift | integer | True | True | 'aggregation_parallelism': | -| run_parallelism | redshift | integer | True | True | 'run_parallelism': | -| skip_failed_files | redshift | boolean | False | True | 'skip_failed_files': True/False | -| fail_on_write_error | redshift | boolean | False | True | 'fail_on_write_error': True/False | -| comment | redshift | text | True | True | 'comment': '' | - - -## Copy options - -| Option | Storage | Category | Type | Editable | Optional | Config Syntax | -| -------| ---------- | -------- | -----| -------- | -------- | ------------- | -| topic | kafka | source_options | text | False | False | 'comment': '' | -| exclude_columns | kafka | job_options | list | False | True | 'exclude_columns': ('', ...) | -| deduplicate_with | kafka | job_options | dict | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | -| consumer_properties | kafka | job_options | text | True | True | 'comment': '' | -| reader_shards | kafka | job_options | integer | True | True | 'reader_shards': | -| store_raw_data | kafka | job_options | boolean | False | True | 'store_raw_data': True/False | -| start_from | kafka | job_options | value | False | True | 'start_from': 'BEGINNING/NOW' | -| end_at | kafka | job_options | value | True | True | 'end_at': '/NOW' | -| compute_cluster | kafka | job_options | identifier | True | True | 'compute_cluster': '' | -| run_parallelism | kafka | job_options | integer | True | True | 'run_parallelism': | -| content_type | kafka | job_options | value | True | True | 'content_type': 'AUTO/CSV/...' | -| compression | kafka | job_options | value | False | True | 'compression': 'AUTO/GZIP/...' | -| comment | kafka | job_options | text | True | True | 'comment': '' | -| table_include_list | mysql | source_options | list | True | True | 'table_include_list': ('', ...) | -| column_exclude_list | mysql | source_options | list | True | True | 'column_exclude_list': ('', ...) | -| exclude_columns | mysql | job_options | list | False | True | 'exclude_columns': ('', ...) | -| column_transformations | mysql | job_options | dict | False | True | 'column_transformations': {'' : '' , ...} | -| skip_snapshots | mysql | job_options | boolean | True | True | 'skip_snapshots': True/False | -| end_at | mysql | job_options | value | True | True | 'end_at': '/NOW' | -| compute_cluster | mysql | job_options | identifier | True | True | 'compute_cluster': '' | -| comment | mysql | job_options | text | True | True | 'comment': '' | -| table_include_list | postgres | source_options | list | False | False | 'table_include_list': ('', ...) | -| column_exclude_list | postgres | source_options | list | False | True | 'column_exclude_list': ('', ...) | -| heartbeat_table | postgres | job_options | text | False | True | 'heartbeat_table': '' | -| skip_snapshots | postgres | job_options | boolean | False | True | 'skip_snapshots': True/False | -| publication_name | postgres | job_options | text | False | False | 'publication_name': '' | -| end_at | postgres | job_options | value | True | True | 'end_at': '/NOW' | -| start_from | postgres | job_options | value | False | True | 'start_from': '/NOW/BEGINNING' | -| compute_cluster | postgres | job_options | identifier | True | True | 'compute_cluster': '' | -| comment | postgres | job_options | text | True | True | 'comment': '' | -| parse_json_columns | postgres | job_options | boolean | False | False | 'parse_json_columns': True/False | -| column_transformations | postgres | job_options | dict | False | True | 'column_transformations': {'' : '' , ...} | -| exclude_columns | postgres | job_options | list | False | True | 'exclude_columns': ('', ...) | -| location | s3 | source_options | text | False | False | 'location': '' | -| date_pattern | s3 | job_options | text | False | True | 'date_pattern': '' | -| file_pattern | s3 | job_options | text | False | True | 'file_pattern': '' | -| initial_load_pattern | s3 | job_options | text | False | True | 'initial_load_pattern': '' | -| initial_load_prefix | s3 | job_options | text | False | True | 'initial_load_prefix': '' | -| delete_files_after_load | s3 | job_options | boolean | False | True | 'delete_files_after_load': True/False | -| deduplicate_with | s3 | job_options | dict | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | -| end_at | s3 | job_options | value | True | True | 'end_at': '/NOW' | -| start_from | s3 | job_options | value | False | True | 'start_from': '/NOW/BEGINNING' | -| compute_cluster | s3 | job_options | identifier | True | True | 'compute_cluster': '' | -| run_parallelism | s3 | job_options | integer | True | True | 'run_parallelism': | -| content_type | s3 | job_options | value | True | True | 'content_type': 'AUTO/CSV...' | -| compression | s3 | job_options | value | False | True | 'compression': 'AUTO/GZIP...' | -| comment | s3 | job_options | text | True | True | 'comment': '' | -| column_transformations | s3 | job_options | dict | False | True | 'column_transformations': {'' : '' , ...} | -| exclude_columns | s3 | job_options | list | False | True | 'exclude_columns': ('', ...) | -| stream | kinesis | source_options | text | False | False | 'stream': '' | -| reader_shards | kinesis | job_options | integer | True | True | 'reader_shards': | -| store_raw_data | kinesis | job_options | boolean | False | True | 'store_raw_data': True/False | -| start_from | kinesis | job_options | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | kinesis | job_options | value | False | True | 'end_at': '/NOW' | -| compute_cluster | kinesis | job_options | identifier | True | True | 'compute_cluster': '' | -| run_parallelism | kinesis | job_options | integer | False | True | 'run_parallelism': | -| content_type | kinesis | job_options | value | True | True | 'content_type': 'AUTO/CSV...' | -| compression | kinesis | job_options | value | False | True | 'compression': 'AUTO/GZIP...' | -| comment | kinesis | job_options | text | True | True | 'comment': '' | -| column_transformations | kinesis | job_options | text | True | True | 'column_transformations': {'' : '' , ...} | -| deduplicate_with | kinesis | job_options | dict | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | -| exclude_columns | kinesis | job_options | list | False | True | 'exclude_columns': ('', ...) | diff --git a/dbt/adapters/upsolver/options/dbt_docs/connection_properties.md b/dbt/adapters/upsolver/options/dbt_docs/connection_properties.md new file mode 100644 index 0000000..04de0e2 --- /dev/null +++ b/dbt/adapters/upsolver/options/dbt_docs/connection_properties.md @@ -0,0 +1,221 @@ + + +## Connection options + +| Option | Storage | Editable | Optional | Config Syntax | +| -------| --------- | -------- | -------- | ------------- | +| aws_role | s3 | True | True | 'aws_role': '' | +| external_id | s3 | True | True | 'external_id': '' | +| aws_access_key_id | s3 | True | True | 'aws_access_key_id': '' | +| aws_secret_access_key_id | s3 | True | True | 'aws_secret_access_key_id': '' | +| path_display_filter | s3 | True | True | 'path_display_filter': '' | +| path_display_filters | s3 | True | True | 'path_display_filters': ('', ...) | +| read_only | s3 | True | True | 'read_only': True/False | +| encryption_kms_key | s3 | True | True | 'encryption_kms_key': '' | +| encryption_customer_kms_key | s3 | True | True | 'encryption_customer_kms_key': '' | +| comment | s3 | True | True | 'comment': '' | +| host | kafka | False | False | 'host': '' | +| hosts | kafka | False | False | 'hosts': ('', ...) | +| consumer_properties | kafka | True | True | 'consumer_properties': '' | +| version | kafka | False | True | 'version': '' | +| require_static_ip | kafka | True | True | 'require_static_ip': True/False | +| ssl | kafka | True | True | 'ssl': True/False | +| topic_display_filter | kafka | True | True | 'topic_display_filter': '' | +| topic_display_filters | kafka | True | True | 'topic_display_filter': ('', ...) | +| comment | kafka | True | True | 'comment': '' | +| aws_role | glue_catalog | True | True | 'aws_role': '' | +| external_id | glue_catalog | True | True | 'external_id': '' | +| aws_access_key_id | glue_catalog | True | True | 'aws_access_key_id': '' | +| aws_secret_access_key | glue_catalog | True | True | 'aws_secret_access_key': '' | +| default_storage_connection | glue_catalog | False | False | 'default_storage_connection': '' | +| default_storage_location | glue_catalog | False | False | 'default_storage_location': '' | +| region | glue_catalog | False | True | 'region': '' | +| database_display_filter | glue_catalog | True | True | 'database_display_filter': '' | +| database_display_filters | glue_catalog | True | True | 'database_display_filters': ('', ...) | +| comment | glue_catalog | True | True | 'comment': '' | +| aws_role | kinesis | True | True | 'aws_role': '' | +| external_id | kinesis | True | True | 'external_id': '' | +| aws_access_key_id | kinesis | True | True | 'aws_access_key_id': '' | +| aws_secret_access_key | kinesis | True | True | 'aws_secret_access_key': '' | +| region | kinesis | False | False | 'region': '' | +| read_only | kinesis | False | True | 'read_only': True/False | +| max_writers | kinesis | True | True | 'max_writers': | +| stream_display_filter | kinesis | True | True | 'stream_display_filter': '' | +| stream_display_filters | kinesis | True | True | 'stream_display_filters': ('', ...) | +| comment | kinesis | True | True | 'comment': '' | +| connection_string | snowflake | True | False | 'connection_string': '' | +| user_name | snowflake | True | False | 'user_name': '' | +| password | snowflake | True | False | 'password': '' | +| max_concurrent_connections | snowflake | True | True | 'max_concurrent_connections': | +| comment | snowflake | True | True | 'comment': '' | +| connection_string | redshift | True | False | 'connection_string': '' | +| user_name | redshift | True | False | 'user_name': '' | +| password | redshift | True | False | 'password': '' | +| max_concurrent_connections | redshift | True | True | 'max_concurrent_connections': | +| comment | redshift | True | True | 'comment': '' | +| connection_string | mysql | True | False | 'connection_string': '' | +| user_name | mysql | True | False | 'user_name': '' | +| password | mysql | True | False | 'password': '' | +| comment | mysql | True | True | 'comment': '' | +| connection_string | postgres | True | False | 'connection_string': '' | +| user_name | postgres | True | False | 'user_name': '' | +| password | postgres | True | False | 'password': '' | +| comment | postgres | True | True | 'comment': '' | +| connection_string | elasticsearch | True | False | 'connection_string': '' | +| user_name | elasticsearch | True | False | 'user_name': '' | +| password | elasticsearch | True | False | 'password': '' | +| comment | elasticsearch | True | True | 'comment': '' | + + +## Target options + +| Option | Storage | Editable | Optional | Config Syntax | +| -------| --------- | -------- | -------- | ------------- | +| globally_unique_keys | datalake | False | True | 'globally_unique_keys': True/False | +| storage_connection | datalake | False | True | 'storage_connection': '' | +| storage_location | datalake | False | True | 'storage_location': '' | +| compute_cluster | datalake | True | True | 'compute_cluster': '' | +| compression | datalake | True | True | 'compression': 'SNAPPY/GZIP' | +| compaction_processes | datalake | True | True | 'compaction_processes': | +| disable_compaction | datalake | True | True | 'disable_compaction': True/False | +| retention_date_partition | datalake | False | True | 'retention_date_partition': '' | +| table_data_retention | datalake | True | True | 'table_data_retention': '' | +| column_data_retention | datalake | True | True | 'column_data_retention': ({'COLUMN' : '','DURATION': ''}) | +| comment | datalake | True | True | 'comment': '' | +| storage_connection | materialized_view | False | True | 'storage_connection': '' | +| storage_location | materialized_view | False | True | 'storage_location': '' | +| max_time_travel_duration | materialized_view | True | True | 'max_time_travel_duration': '' | +| compute_cluster | materialized_view | True | True | 'compute_cluster': '' | +| column_transformations | snowflake | False | True | 'column_transformations': {'' : '' , ...} | +| deduplicate_with | snowflake | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | +| exclude_columns | snowflake | False | True | 'exclude_columns': ('', ...) | +| create_table_if_missing | snowflake | False | True | 'create_table_if_missing': True/False} | +| run_interval | snowflake | False | True | 'run_interval': '' | + + +## Transformation options + +| Option | Storage | Editable | Optional | Config Syntax | +| -------| --------- | -------- | -------- | ------------- | +| run_interval | s3 | False | True | 'run_interval': '' | +| start_from | s3 | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | s3 | True | True | 'end_at': '/NOW' | +| compute_cluster | s3 | True | True | 'compute_cluster': '' | +| comment | s3 | True | True | 'comment': '' | +| allow_cartesian_products | s3 | False | True | 'allow_cartesian_products': True/False | +| aggregation_parallelism | s3 | True | True | 'aggregation_parallelism': | +| run_parallelism | s3 | True | True | 'run_parallelism': | +| file_format | s3 | False | False | 'file_format': 'CSV/TSV ...' | +| compression | s3 | False | True | 'compression': 'SNAPPY/GZIP ...' | +| date_pattern | s3 | False | True | 'date_pattern': '' | +| output_offset | s3 | False | True | 'output_offset': '' | +| location | s3 | False | False | 'location': '' | +| run_interval | elasticsearch | False | True | 'run_interval': '' | +| start_from | elasticsearch | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | elasticsearch | True | True | 'end_at': '/NOW' | +| compute_cluster | elasticsearch | True | True | 'compute_cluster': '' | +| allow_cartesian_products | elasticsearch | False | True | 'allow_cartesian_products': True/False | +| aggregation_parallelism | elasticsearch | True | True | 'aggregation_parallelism': | +| run_parallelism | elasticsearch | True | True | 'run_parallelism': | +| bulk_max_size_bytes | elasticsearch | True | True | 'bulk_max_size_bytes': | +| index_partition_size | elasticsearch | True | True | 'index_partition_size': 'HOURLY/DAILY ...' | +| comment | elasticsearch | True | True | 'comment': '' | +| custom_insert_expressions | snowflake | True | True | 'custom_insert_expressions': {'INSERT_TIME' : 'CURRENT_TIMESTAMP()','MY_VALUE': ''} | +| custom_update_expressions | snowflake | True | True | 'custom_update_expressions': {'UPDATE_TIME' : 'CURRENT_TIMESTAMP()','MY_VALUE': ''} | +| keep_existing_values_when_null | snowflake | True | True | 'keep_existing_values_when_null': True/False | +| add_missing_columns | snowflake | False | True | 'add_missing_columns': True/False | +| run_interval | snowflake | False | True | 'run_interval': '' | +| start_from | snowflake | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | snowflake | True | True | 'end_at': '/NOW' | +| compute_cluster | snowflake | True | True | 'compute_cluster': '' | +| allow_cartesian_products | snowflake | False | True | 'allow_cartesian_products': True/False | +| aggregation_parallelism | snowflake | True | True | 'aggregation_parallelism': | +| run_parallelism | snowflake | True | True | 'run_parallelism': | +| comment | snowflake | True | True | 'comment': '' | +| add_missing_columns | datalake | False | True | 'add_missing_columns': True/False | +| run_interval | datalake | False | True | 'run_interval': '' | +| start_from | datalake | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | datalake | True | True | 'end_at': '/NOW' | +| compute_cluster | datalake | True | True | 'compute_cluster': '' | +| allow_cartesian_products | datalake | False | True | 'allow_cartesian_products': True/False | +| aggregation_parallelism | datalake | True | True | 'aggregation_parallelism': | +| run_parallelism | datalake | True | True | 'run_parallelism': | +| comment | datalake | True | True | 'comment': '' | +| run_interval | redshift | False | True | 'run_interval': '' | +| start_from | redshift | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | redshift | True | True | 'end_at': '/NOW' | +| compute_cluster | redshift | True | True | 'compute_cluster': '' | +| allow_cartesian_products | redshift | False | True | 'allow_cartesian_products': True/False | +| aggregation_parallelism | redshift | True | True | 'aggregation_parallelism': | +| run_parallelism | redshift | True | True | 'run_parallelism': | +| skip_failed_files | redshift | False | True | 'skip_failed_files': True/False | +| fail_on_write_error | redshift | False | True | 'fail_on_write_error': True/False | +| comment | redshift | True | True | 'comment': '' | + + +## Copy options + +| Option | Storage | Category | Editable | Optional | Config Syntax | +| -------| ---------- | -------- | -------- | -------- | ------------- | +| topic | kafka | source_options | False | False | 'comment': '' | +| exclude_columns | kafka | job_options | False | True | 'exclude_columns': ('', ...) | +| deduplicate_with | kafka | job_options | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | +| consumer_properties | kafka | job_options | True | True | 'comment': '' | +| reader_shards | kafka | job_options | True | True | 'reader_shards': | +| store_raw_data | kafka | job_options | False | True | 'store_raw_data': True/False | +| start_from | kafka | job_options | False | True | 'start_from': 'BEGINNING/NOW' | +| end_at | kafka | job_options | True | True | 'end_at': '/NOW' | +| compute_cluster | kafka | job_options | True | True | 'compute_cluster': '' | +| run_parallelism | kafka | job_options | True | True | 'run_parallelism': | +| content_type | kafka | job_options | True | True | 'content_type': 'AUTO/CSV/...' | +| compression | kafka | job_options | False | True | 'compression': 'AUTO/GZIP/...' | +| comment | kafka | job_options | True | True | 'comment': '' | +| table_include_list | mysql | source_options | True | True | 'table_include_list': ('', ...) | +| column_exclude_list | mysql | source_options | True | True | 'column_exclude_list': ('', ...) | +| exclude_columns | mysql | job_options | False | True | 'exclude_columns': ('', ...) | +| column_transformations | mysql | job_options | False | True | 'column_transformations': {'' : '' , ...} | +| skip_snapshots | mysql | job_options | True | True | 'skip_snapshots': True/False | +| end_at | mysql | job_options | True | True | 'end_at': '/NOW' | +| compute_cluster | mysql | job_options | True | True | 'compute_cluster': '' | +| comment | mysql | job_options | True | True | 'comment': '' | +| table_include_list | postgres | source_options | False | False | 'table_include_list': ('', ...) | +| column_exclude_list | postgres | source_options | False | True | 'column_exclude_list': ('', ...) | +| heartbeat_table | postgres | job_options | False | True | 'heartbeat_table': '' | +| skip_snapshots | postgres | job_options | False | True | 'skip_snapshots': True/False | +| publication_name | postgres | job_options | False | False | 'publication_name': '' | +| end_at | postgres | job_options | True | True | 'end_at': '/NOW' | +| start_from | postgres | job_options | False | True | 'start_from': '/NOW/BEGINNING' | +| compute_cluster | postgres | job_options | True | True | 'compute_cluster': '' | +| comment | postgres | job_options | True | True | 'comment': '' | +| parse_json_columns | postgres | job_options | False | False | 'parse_json_columns': True/False | +| column_transformations | postgres | job_options | False | True | 'column_transformations': {'' : '' , ...} | +| exclude_columns | postgres | job_options | False | True | 'exclude_columns': ('', ...) | +| location | s3 | source_options | False | False | 'location': '' | +| date_pattern | s3 | job_options | False | True | 'date_pattern': '' | +| file_pattern | s3 | job_options | False | True | 'file_pattern': '' | +| initial_load_pattern | s3 | job_options | False | True | 'initial_load_pattern': '' | +| initial_load_prefix | s3 | job_options | False | True | 'initial_load_prefix': '' | +| delete_files_after_load | s3 | job_options | False | True | 'delete_files_after_load': True/False | +| deduplicate_with | s3 | job_options | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | +| end_at | s3 | job_options | True | True | 'end_at': '/NOW' | +| start_from | s3 | job_options | False | True | 'start_from': '/NOW/BEGINNING' | +| compute_cluster | s3 | job_options | True | True | 'compute_cluster': '' | +| run_parallelism | s3 | job_options | True | True | 'run_parallelism': | +| content_type | s3 | job_options | True | True | 'content_type': 'AUTO/CSV...' | +| compression | s3 | job_options | False | True | 'compression': 'AUTO/GZIP...' | +| comment | s3 | job_options | True | True | 'comment': '' | +| column_transformations | s3 | job_options | False | True | 'column_transformations': {'' : '' , ...} | +| exclude_columns | s3 | job_options | False | True | 'exclude_columns': ('', ...) | +| stream | kinesis | source_options | False | False | 'stream': '' | +| reader_shards | kinesis | job_options | True | True | 'reader_shards': | +| store_raw_data | kinesis | job_options | False | True | 'store_raw_data': True/False | +| start_from | kinesis | job_options | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | kinesis | job_options | False | True | 'end_at': '/NOW' | +| compute_cluster | kinesis | job_options | True | True | 'compute_cluster': '' | +| run_parallelism | kinesis | job_options | False | True | 'run_parallelism': | +| content_type | kinesis | job_options | True | True | 'content_type': 'AUTO/CSV...' | +| compression | kinesis | job_options | False | True | 'compression': 'AUTO/GZIP...' | +| comment | kinesis | job_options | True | True | 'comment': '' | +| column_transformations | kinesis | job_options | True | True | 'column_transformations': {'' : '' , ...} | +| deduplicate_with | kinesis | job_options | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | +| exclude_columns | kinesis | job_options | False | True | 'exclude_columns': ('', ...) | diff --git a/dbt/adapters/upsolver/options/geterate_options_docs.py b/dbt/adapters/upsolver/options/dbt_docs/geterate_options_docs.py similarity index 67% rename from dbt/adapters/upsolver/options/geterate_options_docs.py rename to dbt/adapters/upsolver/options/dbt_docs/geterate_options_docs.py index d5f074a..764eb3b 100644 --- a/dbt/adapters/upsolver/options/geterate_options_docs.py +++ b/dbt/adapters/upsolver/options/dbt_docs/geterate_options_docs.py @@ -6,25 +6,25 @@ def write_header(options_header, file): file.write(f"\n\n## {options_header}\n\n") - file.write("| Option | Storage | Type | Editable | Optional | Config Syntax |\n") - file.write("| --------| --------- | ---- | -------- | -------- | ------------- |\n") + file.write("| Option | Storage | Editable | Optional | Config Syntax |\n") + file.write("| -------| --------- | -------- | -------- | ------------- |\n") def write_options_to_md(options_category, options_header, file): write_header(options_header, file) for key_con, value in options_category.items(): for key, value in value.items(): #formated_description = (' '.join(value.get('description', '').split())).replace('[\\t\\n\\r]+',' ') - md_file.write(f"| {key} | {key_con} | {value['type']} | {value['editable']} | {value['optional']} | {value.get('syntax', '')} |\n") + md_file.write(f"| {key} | {key_con} | {value['editable']} | {value['optional']} | {value.get('syntax', '')} |\n") def write_copy_options_to_md(options_category, options_header, file): file.write(f"\n\n## {options_header}\n\n") - file.write("| Option | Storage | Category | Type | Editable | Optional | Config Syntax |\n") - file.write("| -------| ---------- | -------- | -----| -------- | -------- | ------------- |\n") + file.write("| Option | Storage | Category | Editable | Optional | Config Syntax |\n") + file.write("| -------| ---------- | -------- | -------- | -------- | ------------- |\n") count = 0 for key_con, value_con in options_category.items(): for key_job, value_job in value_con.items(): for key, value in value_job.items(): - md_file.write(f"| {key} | {key_con} | {key_job} | {value['type']} | {value['editable']} | {value['optional']} | {value.get('syntax', '')} |\n") + md_file.write(f"| {key} | {key_con} | {key_job} | {value['editable']} | {value['optional']} | {value.get('syntax', '')} |\n") with open('connection_properties.md', 'w') as md_file: write_options_to_md(Connection_options, 'Connection options', md_file) diff --git a/dbt/adapters/upsolver/options/dbt_docs/upsolver-configs.md b/dbt/adapters/upsolver/options/dbt_docs/upsolver-configs.md index 9343048..7c81135 100644 --- a/dbt/adapters/upsolver/options/dbt_docs/upsolver-configs.md +++ b/dbt/adapters/upsolver/options/dbt_docs/upsolver-configs.md @@ -146,223 +146,267 @@ GROUP BY ... Running this model will compile CREATE MATERIALIZED VIEW SQL(or ALTER MATERIALIZED VIEW if exists) and send it to Upsolver engine. Name of the materializedview will be name of the model. +## Expectations/constraints + +Data quality conditions can be added to your job to drop a row or trigger a warning when a column violates a predefined condition. + +```sql +WITH EXPECTATION EXPECT +ON VIOLATION WARN +``` + +Expectations can be implemented with dbt constraints +Supported constraints: check and not_null + +```yaml +models: + - name: + # required + config: + contract: + enforced: true + # model-level constraints + constraints: + - type: check + columns: ['', ''] + expression: "column1 <= column2" + name: + - type: not_null + columns: ['column1', 'column2'] + name: + + columns: + - name: + data_type: string + + # column-level constraints + constraints: + - type: not_null + - type: check + expression: "REGEXP_LIKE(, '^[0-9]{4}[a-z]{5}$')" + name: +``` + ## Projects examples > projects examples link: [github.com/dbt-upsolver/examples/](https://github.com/Upsolver/dbt-upsolver/tree/main/examples) ## Connection options -| Option | Storage | Type | Editable | Optional | Config Syntax | -| --------| --------- | ---- | -------- | -------- | ------------- | -| aws_role | s3 | text | True | True | 'aws_role': '' | -| external_id | s3 | text | True | True | 'external_id': '' | -| aws_access_key_id | s3 | text | True | True | 'aws_access_key_id': '' | -| aws_secret_access_key_id | s3 | text | True | True | 'aws_secret_access_key_id': '' | -| path_display_filter | s3 | text | True | True | 'path_display_filter': '' | -| path_display_filters | s3 | list | True | True | 'path_display_filters': ('', ...) | -| read_only | s3 | boolean | True | True | 'read_only': True/False | -| encryption_kms_key | s3 | text | True | True | 'encryption_kms_key': '' | -| encryption_customer_kms_key | s3 | text | True | True | 'encryption_customer_kms_key': '' | -| comment | s3 | text | True | True | 'comment': '' | -| host | kafka | text | False | False | 'host': '' | -| hosts | kafka | list | False | False | 'hosts': ('', ...) | -| consumer_properties | kafka | text | True | True | 'consumer_properties': '' | -| version | kafka | value | False | True | 'version': '' | -| require_static_ip | kafka | boolean | True | True | 'require_static_ip': True/False | -| ssl | kafka | boolean | True | True | 'ssl': True/False | -| topic_display_filter | kafka | text | True | True | 'topic_display_filter': '' | -| topic_display_filters | kafka | list | True | True | 'topic_display_filter': ('', ...) | -| comment | kafka | text | True | True | 'comment': '' | -| aws_role | glue_catalog | text | True | True | 'aws_role': '' | -| external_id | glue_catalog | text | True | True | 'external_id': '' | -| aws_access_key_id | glue_catalog | text | True | True | 'aws_access_key_id': '' | -| aws_secret_access_key | glue_catalog | text | True | True | 'aws_secret_access_key': '' | -| default_storage_connection | glue_catalog | identifier | False | False | 'default_storage_connection': '' | -| default_storage_location | glue_catalog | text | False | False | 'default_storage_location': '' | -| region | glue_catalog | text | False | True | 'region': '' | -| database_display_filter | glue_catalog | text | True | True | 'database_display_filter': '' | -| database_display_filters | glue_catalog | list | True | True | 'database_display_filters': ('', ...) | -| comment | glue_catalog | text | True | True | 'comment': '' | -| aws_role | kinesis | text | True | True | 'aws_role': '' | -| external_id | kinesis | text | True | True | 'external_id': '' | -| aws_access_key_id | kinesis | text | True | True | 'aws_access_key_id': '' | -| aws_secret_access_key | kinesis | text | True | True | 'aws_secret_access_key': '' | -| region | kinesis | text | False | False | 'region': '' | -| read_only | kinesis | boolean | False | True | 'read_only': True/False | -| max_writers | kinesis | integer | True | True | 'max_writers': | -| stream_display_filter | kinesis | text | True | True | 'stream_display_filter': '' | -| stream_display_filters | kinesis | list | True | True | 'stream_display_filters': ('', ...) | -| comment | kinesis | text | True | True | 'comment': '' | -| connection_string | snowflake | text | True | False | 'connection_string': '' | -| user_name | snowflake | text | True | False | 'user_name': '' | -| password | snowflake | text | True | False | 'password': '' | -| max_concurrent_connections | snowflake | integer | True | True | 'max_concurrent_connections': | -| comment | snowflake | text | True | True | 'comment': '' | -| connection_string | redshift | text | True | False | 'connection_string': '' | -| user_name | redshift | text | True | False | 'user_name': '' | -| password | redshift | text | True | False | 'password': '' | -| max_concurrent_connections | redshift | integer | True | True | 'max_concurrent_connections': | -| comment | redshift | text | True | True | 'comment': '' | -| connection_string | mysql | text | True | False | 'connection_string': '' | -| user_name | mysql | text | True | False | 'user_name': '' | -| password | mysql | text | True | False | 'password': '' | -| comment | mysql | text | True | True | 'comment': '' | -| connection_string | postgres | text | True | False | 'connection_string': '' | -| user_name | postgres | text | True | False | 'user_name': '' | -| password | postgres | text | True | False | 'password': '' | -| comment | postgres | text | True | True | 'comment': '' | -| connection_string | elasticsearch | text | True | False | 'connection_string': '' | -| user_name | elasticsearch | text | True | False | 'user_name': '' | -| password | elasticsearch | text | True | False | 'password': '' | -| comment | elasticsearch | text | True | True | 'comment': '' | +| Option | Storage | Editable | Optional | Config Syntax | +| -------| --------- | -------- | -------- | ------------- | +| aws_role | s3 | True | True | 'aws_role': '' | +| external_id | s3 | True | True | 'external_id': '' | +| aws_access_key_id | s3 | True | True | 'aws_access_key_id': '' | +| aws_secret_access_key_id | s3 | True | True | 'aws_secret_access_key_id': '' | +| path_display_filter | s3 | True | True | 'path_display_filter': '' | +| path_display_filters | s3 | True | True | 'path_display_filters': ('', ...) | +| read_only | s3 | True | True | 'read_only': True/False | +| encryption_kms_key | s3 | True | True | 'encryption_kms_key': '' | +| encryption_customer_kms_key | s3 | True | True | 'encryption_customer_kms_key': '' | +| comment | s3 | True | True | 'comment': '' | +| host | kafka | False | False | 'host': '' | +| hosts | kafka | False | False | 'hosts': ('', ...) | +| consumer_properties | kafka | True | True | 'consumer_properties': '' | +| version | kafka | False | True | 'version': '' | +| require_static_ip | kafka | True | True | 'require_static_ip': True/False | +| ssl | kafka | True | True | 'ssl': True/False | +| topic_display_filter | kafka | True | True | 'topic_display_filter': '' | +| topic_display_filters | kafka | True | True | 'topic_display_filter': ('', ...) | +| comment | kafka | True | True | 'comment': '' | +| aws_role | glue_catalog | True | True | 'aws_role': '' | +| external_id | glue_catalog | True | True | 'external_id': '' | +| aws_access_key_id | glue_catalog | True | True | 'aws_access_key_id': '' | +| aws_secret_access_key | glue_catalog | True | True | 'aws_secret_access_key': '' | +| default_storage_connection | glue_catalog | False | False | 'default_storage_connection': '' | +| default_storage_location | glue_catalog | False | False | 'default_storage_location': '' | +| region | glue_catalog | False | True | 'region': '' | +| database_display_filter | glue_catalog | True | True | 'database_display_filter': '' | +| database_display_filters | glue_catalog | True | True | 'database_display_filters': ('', ...) | +| comment | glue_catalog | True | True | 'comment': '' | +| aws_role | kinesis | True | True | 'aws_role': '' | +| external_id | kinesis | True | True | 'external_id': '' | +| aws_access_key_id | kinesis | True | True | 'aws_access_key_id': '' | +| aws_secret_access_key | kinesis | True | True | 'aws_secret_access_key': '' | +| region | kinesis | False | False | 'region': '' | +| read_only | kinesis | False | True | 'read_only': True/False | +| max_writers | kinesis | True | True | 'max_writers': | +| stream_display_filter | kinesis | True | True | 'stream_display_filter': '' | +| stream_display_filters | kinesis | True | True | 'stream_display_filters': ('', ...) | +| comment | kinesis | True | True | 'comment': '' | +| connection_string | snowflake | True | False | 'connection_string': '' | +| user_name | snowflake | True | False | 'user_name': '' | +| password | snowflake | True | False | 'password': '' | +| max_concurrent_connections | snowflake | True | True | 'max_concurrent_connections': | +| comment | snowflake | True | True | 'comment': '' | +| connection_string | redshift | True | False | 'connection_string': '' | +| user_name | redshift | True | False | 'user_name': '' | +| password | redshift | True | False | 'password': '' | +| max_concurrent_connections | redshift | True | True | 'max_concurrent_connections': | +| comment | redshift | True | True | 'comment': '' | +| connection_string | mysql | True | False | 'connection_string': '' | +| user_name | mysql | True | False | 'user_name': '' | +| password | mysql | True | False | 'password': '' | +| comment | mysql | True | True | 'comment': '' | +| connection_string | postgres | True | False | 'connection_string': '' | +| user_name | postgres | True | False | 'user_name': '' | +| password | postgres | True | False | 'password': '' | +| comment | postgres | True | True | 'comment': '' | +| connection_string | elasticsearch | True | False | 'connection_string': '' | +| user_name | elasticsearch | True | False | 'user_name': '' | +| password | elasticsearch | True | False | 'password': '' | +| comment | elasticsearch | True | True | 'comment': '' | + ## Target options -| Option | Storage | Type | Editable | Optional | Config Syntax | -| --------| --------- | ---- | -------- | -------- | ------------- | -| globally_unique_keys | datalake | boolean | False | True | 'globally_unique_keys': True/False | -| storage_connection | datalake | identifier | False | True | 'storage_connection': '' | -| storage_location | datalake | text | False | True | 'storage_location': '' | -| compute_cluster | datalake | identifier | True | True | 'compute_cluster': '' | -| compression | datalake | value | True | True | 'compression': 'SNAPPY/GZIP' | -| compaction_processes | datalake | integer | True | True | 'compaction_processes': | -| disable_compaction | datalake | boolean | True | True | 'disable_compaction': True/False | -| retention_date_partition | datalake | identifier | False | True | 'retention_date_partition': '' | -| table_data_retention | datalake | identifier | True | True | 'table_data_retention': '' | -| column_data_retention | datalake | dict | True | True | 'column_data_retention': {'COLUMN' : '','DURATION': N DAYS} | -| comment | datalake | text | True | True | 'comment': '' | -| storage_connection | materialized_view | identifier | False | True | 'storage_connection': '' | -| storage_location | materialized_view | text | False | True | 'storage_location': '' | -| max_time_travel_duration | materialized_view | identifier | True | True | 'max_time_travel_duration': '' | -| compute_cluster | materialized_view | identifier | True | True | 'compute_cluster': '' | -| column_transformations | snowflake | dict | False | True | 'column_transformations': {'' : '' , ...} | -| deduplicate_with | snowflake | dict | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | -| exclude_columns | snowflake | list | False | True | 'exclude_columns': ('', ...) | -| create_table_if_missing | snowflake | boolean | False | True | 'create_table_if_missing': True/False} | -| run_interval | snowflake | integer | False | True | 'run_interval': '' | +| Option | Storage | Editable | Optional | Config Syntax | +| -------| --------- | -------- | -------- | ------------- | +| globally_unique_keys | datalake | False | True | 'globally_unique_keys': True/False | +| storage_connection | datalake | False | True | 'storage_connection': '' | +| storage_location | datalake | False | True | 'storage_location': '' | +| compute_cluster | datalake | True | True | 'compute_cluster': '' | +| compression | datalake | True | True | 'compression': 'SNAPPY/GZIP' | +| compaction_processes | datalake | True | True | 'compaction_processes': | +| disable_compaction | datalake | True | True | 'disable_compaction': True/False | +| retention_date_partition | datalake | False | True | 'retention_date_partition': '' | +| table_data_retention | datalake | True | True | 'table_data_retention': '' | +| column_data_retention | datalake | True | True | 'column_data_retention': ({'COLUMN' : '','DURATION': ''}) | +| comment | datalake | True | True | 'comment': '' | +| storage_connection | materialized_view | False | True | 'storage_connection': '' | +| storage_location | materialized_view | False | True | 'storage_location': '' | +| max_time_travel_duration | materialized_view | True | True | 'max_time_travel_duration': '' | +| compute_cluster | materialized_view | True | True | 'compute_cluster': '' | +| column_transformations | snowflake | False | True | 'column_transformations': {'' : '' , ...} | +| deduplicate_with | snowflake | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | +| exclude_columns | snowflake | False | True | 'exclude_columns': ('', ...) | +| create_table_if_missing | snowflake | False | True | 'create_table_if_missing': True/False} | +| run_interval | snowflake | False | True | 'run_interval': '' | + ## Transformation options -| Option | Storage | Type | Editable | Optional | Config Syntax | -| --------| --------- | ---- | -------- | -------- | ------------- | -| run_interval | s3 | ineger | False | True | 'run_interval': '' | -| start_from | s3 | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | s3 | value | True | True | 'end_at': '/NOW' | -| compute_cluster | s3 | identifier | True | True | 'compute_cluster': '' | -| comment | s3 | text | True | True | 'comment': '' | -| allow_cartesian_products | s3 | boolean | False | True | 'allow_cartesian_products': True/False | -| aggregation_parallelism | s3 | integer | True | True | 'aggregation_parallelism': | -| run_parallelism | s3 | integer | True | True | 'run_parallelism': | -| file_format | s3 | value | False | False | 'file_format': 'CSV/TSV ...' | -| compression | s3 | value | False | True | 'compression': 'SNAPPY/GZIP ...' | -| date_pattern | s3 | text | False | True | 'date_pattern': '' | -| output_offset | s3 | identifier | False | True | 'output_offset': '' | -| location | s3 | text | False | False | 'location': '' | -| run_interval | elasticsearch | identifier | False | True | 'run_interval': '' | -| start_from | elasticsearch | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | elasticsearch | value | True | True | 'end_at': '/NOW' | -| compute_cluster | elasticsearch | identifier | True | True | 'compute_cluster': '' | -| allow_cartesian_products | elasticsearch | boolean | False | True | 'allow_cartesian_products': True/False | -| aggregation_parallelism | elasticsearch | integer | True | True | 'aggregation_parallelism': | -| run_parallelism | elasticsearch | integer | True | True | 'run_parallelism': | -| bulk_max_size_bytes | elasticsearch | integer | True | True | 'bulk_max_size_bytes': | -| index_partition_size | elasticsearch | value | True | True | 'index_partition_size': 'HOURLY/DAILY ...' | -| comment | elasticsearch | text | True | True | 'comment': '' | -| custom_insert_expressions | snowflake | dict | True | True | 'custom_insert_expressions': {'INSERT_TIME' : 'CURRENT_TIMESTAMP()','MY_VALUE': ''} | -| custom_update_expressions | snowflake | dict | True | True | 'custom_update_expressions': {'UPDATE_TIME' : 'CURRENT_TIMESTAMP()','MY_VALUE': ''} | -| keep_existing_values_when_null | snowflake | boolean | True | True | 'keep_existing_values_when_null': True/False | -| add_missing_columns | snowflake | boolean | False | True | 'add_missing_columns': True/False | -| run_interval | snowflake | identifier | False | True | 'run_interval': '' | -| start_from | snowflake | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | snowflake | value | True | True | 'end_at': '/NOW' | -| compute_cluster | snowflake | identifier | True | True | 'compute_cluster': '' | -| allow_cartesian_products | snowflake | boolean | False | True | 'allow_cartesian_products': True/False | -| aggregation_parallelism | snowflake | integer | True | True | 'aggregation_parallelism': | -| run_parallelism | snowflake | integer | True | True | 'run_parallelism': | -| comment | snowflake | text | True | True | 'comment': '' | -| add_missing_columns | datalake | boolean | False | True | 'add_missing_columns': True/False | -| run_interval | datalake | identifier | False | True | 'run_interval': '' | -| start_from | datalake | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | datalake | value | True | True | 'end_at': '/NOW' | -| compute_cluster | datalake | identifier | True | True | 'compute_cluster': '' | -| allow_cartesian_products | datalake | boolean | False | True | 'allow_cartesian_products': True/False | -| aggregation_parallelism | datalake | integer | True | True | 'aggregation_parallelism': | -| run_parallelism | datalake | integer | True | True | 'run_parallelism': | -| comment | datalake | text | True | True | 'comment': '' | -| run_interval | redshift | identifier | False | True | 'run_interval': '' | -| start_from | redshift | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | redshift | value | True | True | 'end_at': '/NOW' | -| compute_cluster | redshift | identifier | True | True | 'compute_cluster': '' | -| allow_cartesian_products | redshift | boolean | False | True | 'allow_cartesian_products': True/False | -| aggregation_parallelism | redshift | integer | True | True | 'aggregation_parallelism': | -| run_parallelism | redshift | integer | True | True | 'run_parallelism': | -| skip_failed_files | redshift | boolean | False | True | 'skip_failed_files': True/False | -| fail_on_write_error | redshift | boolean | False | True | 'fail_on_write_error': True/False | -| comment | redshift | text | True | True | 'comment': '' | +| Option | Storage | Editable | Optional | Config Syntax | +| -------| --------- | -------- | -------- | ------------- | +| run_interval | s3 | False | True | 'run_interval': '' | +| start_from | s3 | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | s3 | True | True | 'end_at': '/NOW' | +| compute_cluster | s3 | True | True | 'compute_cluster': '' | +| comment | s3 | True | True | 'comment': '' | +| allow_cartesian_products | s3 | False | True | 'allow_cartesian_products': True/False | +| aggregation_parallelism | s3 | True | True | 'aggregation_parallelism': | +| run_parallelism | s3 | True | True | 'run_parallelism': | +| file_format | s3 | False | False | 'file_format': 'CSV/TSV ...' | +| compression | s3 | False | True | 'compression': 'SNAPPY/GZIP ...' | +| date_pattern | s3 | False | True | 'date_pattern': '' | +| output_offset | s3 | False | True | 'output_offset': '' | +| location | s3 | False | False | 'location': '' | +| run_interval | elasticsearch | False | True | 'run_interval': '' | +| start_from | elasticsearch | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | elasticsearch | True | True | 'end_at': '/NOW' | +| compute_cluster | elasticsearch | True | True | 'compute_cluster': '' | +| allow_cartesian_products | elasticsearch | False | True | 'allow_cartesian_products': True/False | +| aggregation_parallelism | elasticsearch | True | True | 'aggregation_parallelism': | +| run_parallelism | elasticsearch | True | True | 'run_parallelism': | +| bulk_max_size_bytes | elasticsearch | True | True | 'bulk_max_size_bytes': | +| index_partition_size | elasticsearch | True | True | 'index_partition_size': 'HOURLY/DAILY ...' | +| comment | elasticsearch | True | True | 'comment': '' | +| custom_insert_expressions | snowflake | True | True | 'custom_insert_expressions': {'INSERT_TIME' : 'CURRENT_TIMESTAMP()','MY_VALUE': ''} | +| custom_update_expressions | snowflake | True | True | 'custom_update_expressions': {'UPDATE_TIME' : 'CURRENT_TIMESTAMP()','MY_VALUE': ''} | +| keep_existing_values_when_null | snowflake | True | True | 'keep_existing_values_when_null': True/False | +| add_missing_columns | snowflake | False | True | 'add_missing_columns': True/False | +| run_interval | snowflake | False | True | 'run_interval': '' | +| start_from | snowflake | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | snowflake | True | True | 'end_at': '/NOW' | +| compute_cluster | snowflake | True | True | 'compute_cluster': '' | +| allow_cartesian_products | snowflake | False | True | 'allow_cartesian_products': True/False | +| aggregation_parallelism | snowflake | True | True | 'aggregation_parallelism': | +| run_parallelism | snowflake | True | True | 'run_parallelism': | +| comment | snowflake | True | True | 'comment': '' | +| add_missing_columns | datalake | False | True | 'add_missing_columns': True/False | +| run_interval | datalake | False | True | 'run_interval': '' | +| start_from | datalake | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | datalake | True | True | 'end_at': '/NOW' | +| compute_cluster | datalake | True | True | 'compute_cluster': '' | +| allow_cartesian_products | datalake | False | True | 'allow_cartesian_products': True/False | +| aggregation_parallelism | datalake | True | True | 'aggregation_parallelism': | +| run_parallelism | datalake | True | True | 'run_parallelism': | +| comment | datalake | True | True | 'comment': '' | +| run_interval | redshift | False | True | 'run_interval': '' | +| start_from | redshift | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | redshift | True | True | 'end_at': '/NOW' | +| compute_cluster | redshift | True | True | 'compute_cluster': '' | +| allow_cartesian_products | redshift | False | True | 'allow_cartesian_products': True/False | +| aggregation_parallelism | redshift | True | True | 'aggregation_parallelism': | +| run_parallelism | redshift | True | True | 'run_parallelism': | +| skip_failed_files | redshift | False | True | 'skip_failed_files': True/False | +| fail_on_write_error | redshift | False | True | 'fail_on_write_error': True/False | +| comment | redshift | True | True | 'comment': '' | + ## Copy options -| Option | Storage | Category | Type | Editable | Optional | Config Syntax | -| -------| ---------- | -------- | -----| -------- | -------- | ------------- | -| topic | kafka | source_options | text | False | False | 'comment': '' | -| exclude_columns | kafka | job_options | list | False | True | 'exclude_columns': ('', ...) | -| deduplicate_with | kafka | job_options | dict | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | -| consumer_properties | kafka | job_options | text | True | True | 'comment': '' | -| reader_shards | kafka | job_options | integer | True | True | 'reader_shards': | -| store_raw_data | kafka | job_options | boolean | False | True | 'store_raw_data': True/False | -| start_from | kafka | job_options | value | False | True | 'start_from': 'BEGINNING/NOW' | -| end_at | kafka | job_options | value | True | True | 'end_at': '/NOW' | -| compute_cluster | kafka | job_options | identifier | True | True | 'compute_cluster': '' | -| run_parallelism | kafka | job_options | integer | True | True | 'run_parallelism': | -| content_type | kafka | job_options | value | True | True | 'content_type': 'AUTO/CSV/...' | -| compression | kafka | job_options | value | False | True | 'compression': 'AUTO/GZIP/...' | -| comment | kafka | job_options | text | True | True | 'comment': '' | -| table_include_list | mysql | source_options | list | True | True | 'table_include_list': ('', ...) | -| column_exclude_list | mysql | source_options | list | True | True | 'column_exclude_list': ('', ...) | -| exclude_columns | mysql | job_options | list | False | True | 'exclude_columns': ('', ...) | -| column_transformations | mysql | job_options | dict | False | True | 'column_transformations': {'' : '' , ...} | -| skip_snapshots | mysql | job_options | boolean | True | True | 'skip_snapshots': True/False | -| end_at | mysql | job_options | value | True | True | 'end_at': '/NOW' | -| compute_cluster | mysql | job_options | identifier | True | True | 'compute_cluster': '' | -| comment | mysql | job_options | text | True | True | 'comment': '' | -| table_include_list | postgres | source_options | list | False | False | 'table_include_list': ('', ...) | -| column_exclude_list | postgres | source_options | list | False | True | 'column_exclude_list': ('', ...) | -| heartbeat_table | postgres | job_options | text | False | True | 'heartbeat_table': '' | -| skip_snapshots | postgres | job_options | boolean | False | True | 'skip_snapshots': True/False | -| publication_name | postgres | job_options | text | False | False | 'publication_name': '' | -| end_at | postgres | job_options | value | True | True | 'end_at': '/NOW' | -| start_from | postgres | job_options | value | False | True | 'start_from': '/NOW/BEGINNING' | -| compute_cluster | postgres | job_options | identifier | True | True | 'compute_cluster': '' | -| comment | postgres | job_options | text | True | True | 'comment': '' | -| parse_json_columns | postgres | job_options | boolean | False | False | 'parse_json_columns': True/False | -| column_transformations | postgres | job_options | dict | False | True | 'column_transformations': {'' : '' , ...} | -| exclude_columns | postgres | job_options | list | False | True | 'exclude_columns': ('', ...) | -| location | s3 | source_options | text | False | False | 'location': '' | -| date_pattern | s3 | job_options | text | False | True | 'date_pattern': '' | -| file_pattern | s3 | job_options | text | False | True | 'file_pattern': '' | -| initial_load_pattern | s3 | job_options | text | False | True | 'initial_load_pattern': '' | -| initial_load_prefix | s3 | job_options | text | False | True | 'initial_load_prefix': '' | -| delete_files_after_load | s3 | job_options | boolean | False | True | 'delete_files_after_load': True/False | -| deduplicate_with | s3 | job_options | dict | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | -| end_at | s3 | job_options | value | True | True | 'end_at': '/NOW' | -| start_from | s3 | job_options | value | False | True | 'start_from': '/NOW/BEGINNING' | -| compute_cluster | s3 | job_options | identifier | True | True | 'compute_cluster': '' | -| run_parallelism | s3 | job_options | integer | True | True | 'run_parallelism': | -| content_type | s3 | job_options | value | True | True | 'content_type': 'AUTO/CSV...' | -| compression | s3 | job_options | value | False | True | 'compression': 'AUTO/GZIP...' | -| comment | s3 | job_options | text | True | True | 'comment': '' | -| column_transformations | s3 | job_options | dict | False | True | 'column_transformations': {'' : '' , ...} | -| exclude_columns | s3 | job_options | list | False | True | 'exclude_columns': ('', ...) | -| stream | kinesis | source_options | text | False | False | 'stream': '' | -| reader_shards | kinesis | job_options | integer | True | True | 'reader_shards': | -| store_raw_data | kinesis | job_options | boolean | False | True | 'store_raw_data': True/False | -| start_from | kinesis | job_options | value | False | True | 'start_from': '/NOW/BEGINNING' | -| end_at | kinesis | job_options | value | False | True | 'end_at': '/NOW' | -| compute_cluster | kinesis | job_options | identifier | True | True | 'compute_cluster': '' | -| run_parallelism | kinesis | job_options | integer | False | True | 'run_parallelism': | -| content_type | kinesis | job_options | value | True | True | 'content_type': 'AUTO/CSV...' | -| compression | kinesis | job_options | value | False | True | 'compression': 'AUTO/GZIP...' | -| comment | kinesis | job_options | text | True | True | 'comment': '' | -| column_transformations | kinesis | job_options | text | True | True | 'column_transformations': {'' : '' , ...} | -| deduplicate_with | kinesis | job_options | dict | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | -| exclude_columns | kinesis | job_options | list | False | True | 'exclude_columns': ('', ...) | +| Option | Storage | Category | Editable | Optional | Config Syntax | +| -------| ---------- | -------- | -------- | -------- | ------------- | +| topic | kafka | source_options | False | False | 'comment': '' | +| exclude_columns | kafka | job_options | False | True | 'exclude_columns': ('', ...) | +| deduplicate_with | kafka | job_options | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | +| consumer_properties | kafka | job_options | True | True | 'comment': '' | +| reader_shards | kafka | job_options | True | True | 'reader_shards': | +| store_raw_data | kafka | job_options | False | True | 'store_raw_data': True/False | +| start_from | kafka | job_options | False | True | 'start_from': 'BEGINNING/NOW' | +| end_at | kafka | job_options | True | True | 'end_at': '/NOW' | +| compute_cluster | kafka | job_options | True | True | 'compute_cluster': '' | +| run_parallelism | kafka | job_options | True | True | 'run_parallelism': | +| content_type | kafka | job_options | True | True | 'content_type': 'AUTO/CSV/...' | +| compression | kafka | job_options | False | True | 'compression': 'AUTO/GZIP/...' | +| comment | kafka | job_options | True | True | 'comment': '' | +| table_include_list | mysql | source_options | True | True | 'table_include_list': ('', ...) | +| column_exclude_list | mysql | source_options | True | True | 'column_exclude_list': ('', ...) | +| exclude_columns | mysql | job_options | False | True | 'exclude_columns': ('', ...) | +| column_transformations | mysql | job_options | False | True | 'column_transformations': {'' : '' , ...} | +| skip_snapshots | mysql | job_options | True | True | 'skip_snapshots': True/False | +| end_at | mysql | job_options | True | True | 'end_at': '/NOW' | +| compute_cluster | mysql | job_options | True | True | 'compute_cluster': '' | +| comment | mysql | job_options | True | True | 'comment': '' | +| table_include_list | postgres | source_options | False | False | 'table_include_list': ('', ...) | +| column_exclude_list | postgres | source_options | False | True | 'column_exclude_list': ('', ...) | +| heartbeat_table | postgres | job_options | False | True | 'heartbeat_table': '' | +| skip_snapshots | postgres | job_options | False | True | 'skip_snapshots': True/False | +| publication_name | postgres | job_options | False | False | 'publication_name': '' | +| end_at | postgres | job_options | True | True | 'end_at': '/NOW' | +| start_from | postgres | job_options | False | True | 'start_from': '/NOW/BEGINNING' | +| compute_cluster | postgres | job_options | True | True | 'compute_cluster': '' | +| comment | postgres | job_options | True | True | 'comment': '' | +| parse_json_columns | postgres | job_options | False | False | 'parse_json_columns': True/False | +| column_transformations | postgres | job_options | False | True | 'column_transformations': {'' : '' , ...} | +| exclude_columns | postgres | job_options | False | True | 'exclude_columns': ('', ...) | +| location | s3 | source_options | False | False | 'location': '' | +| date_pattern | s3 | job_options | False | True | 'date_pattern': '' | +| file_pattern | s3 | job_options | False | True | 'file_pattern': '' | +| initial_load_pattern | s3 | job_options | False | True | 'initial_load_pattern': '' | +| initial_load_prefix | s3 | job_options | False | True | 'initial_load_prefix': '' | +| delete_files_after_load | s3 | job_options | False | True | 'delete_files_after_load': True/False | +| deduplicate_with | s3 | job_options | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | +| end_at | s3 | job_options | True | True | 'end_at': '/NOW' | +| start_from | s3 | job_options | False | True | 'start_from': '/NOW/BEGINNING' | +| compute_cluster | s3 | job_options | True | True | 'compute_cluster': '' | +| run_parallelism | s3 | job_options | True | True | 'run_parallelism': | +| content_type | s3 | job_options | True | True | 'content_type': 'AUTO/CSV...' | +| compression | s3 | job_options | False | True | 'compression': 'AUTO/GZIP...' | +| comment | s3 | job_options | True | True | 'comment': '' | +| column_transformations | s3 | job_options | False | True | 'column_transformations': {'' : '' , ...} | +| exclude_columns | s3 | job_options | False | True | 'exclude_columns': ('', ...) | +| stream | kinesis | source_options | False | False | 'stream': '' | +| reader_shards | kinesis | job_options | True | True | 'reader_shards': | +| store_raw_data | kinesis | job_options | False | True | 'store_raw_data': True/False | +| start_from | kinesis | job_options | False | True | 'start_from': '/NOW/BEGINNING' | +| end_at | kinesis | job_options | False | True | 'end_at': '/NOW' | +| compute_cluster | kinesis | job_options | True | True | 'compute_cluster': '' | +| run_parallelism | kinesis | job_options | False | True | 'run_parallelism': | +| content_type | kinesis | job_options | True | True | 'content_type': 'AUTO/CSV...' | +| compression | kinesis | job_options | False | True | 'compression': 'AUTO/GZIP...' | +| comment | kinesis | job_options | True | True | 'comment': '' | +| column_transformations | kinesis | job_options | True | True | 'column_transformations': {'' : '' , ...} | +| deduplicate_with | kinesis | job_options | False | True | 'deduplicate_with': {'COLUMNS' : ['col1', 'col2'],'WINDOW': 'N HOURS'} | +| exclude_columns | kinesis | job_options | False | True | 'exclude_columns': ('', ...) | diff --git a/dbt/adapters/upsolver/options/target_options.py b/dbt/adapters/upsolver/options/target_options.py index c258c0c..d0d0c4b 100644 --- a/dbt/adapters/upsolver/options/target_options.py +++ b/dbt/adapters/upsolver/options/target_options.py @@ -51,8 +51,8 @@ For example, if the current time is 2023-02-23 12:30:00 UTC, and you have defined TABLE_DATA_RETENTION = 2 days, you can expect data written during 2023-02-23, 2023-02-22, and 2023-02-21 to exist in the table. The retention threshold truncates data to the nearest day, so when the time changes to 2023-02-24 00:00:00 UTC, you can no longer expect data from 2023-02-21 to be present in the table, although it might be there for a while. Note that you need at least one date partition column for this option to work. Value: DAYS"""}, - "column_data_retention": {"type": "dict", "editable": True, "optional": True, - "syntax":"'column_data_retention': {'COLUMN' : '','DURATION': N DAYS}", + "column_data_retention": {"type": "list_dict", "editable": True, "optional": True, + "syntax":"'column_data_retention': ({'COLUMN' : '','DURATION': ''})", "description":"""When set, after the duration of a column elapses in a partition, the data is rewritten without the contents of that column. Number of days can range between 1 and 9999. Note that you need at least one date partition column for this to work. Type: list of (, DAYS) pairs"""}, diff --git a/dbt/adapters/upsolver/options/transformation_options.py b/dbt/adapters/upsolver/options/transformation_options.py index 64d6c34..81b2f6c 100644 --- a/dbt/adapters/upsolver/options/transformation_options.py +++ b/dbt/adapters/upsolver/options/transformation_options.py @@ -1,16 +1,12 @@ Transformation_options = { "s3": { - "run_interval": {"type": "ineger", "editable": False, "optional": True, + "run_interval": {"type": "integer", "editable": False, "optional": True, "syntax":"'run_interval': ''", "description":"""How often the job runs. The runs take place over a set period of time defined by this interval and they must be divisible by the number of hours in a day. For example, you can set RUN_INTERVAL to 2 hours (the job runs 12 times per day), but trying to set RUN_INTERVAL to 5 hours would fail since 24 hours is not evenly divisible by 5.RUN_INTERVAL Value: { MINUTE[S] | HOUR[S] | DAY[S] } - Default: 1 MINUTE - (Optional) How often the job runs. - The runs take place over a set period of time defined by this interval and they must be divisible by the number of hours in a day. - For example, you can set RUN_INTERVAL to 2 hours (the job runs 12 times per day), but trying to set RUN_INTERVAL to 5 hours would fail since 24 hours is not evenly divisible by 5. - Value: { MINUTE[S] | HOUR[S] | DAY[S] }"""}, + Default: 1 MINUTE"""}, "start_from": {"type": "value", "editable": False, "optional": True, "syntax":"'start_from': '/NOW/BEGINNING'", "description":"""Configures the time to start inserting data from. Data before the specified time is ignored. @@ -84,17 +80,13 @@ Usually, it's recommended to include padding to ensure alphabetical sorting of the output files."""} }, "elasticsearch": { - "run_interval": {"type": "identifier", "editable": False, "optional": True, + "run_interval": {"type": "integer", "editable": False, "optional": True, "syntax":"'run_interval': ''", "description":"""How often the job runs. The runs take place over a set period of time defined by this interval and they must be divisible by the number of hours in a day. For example, you can set RUN_INTERVAL to 2 hours (the job runs 12 times per day), but trying to set RUN_INTERVAL to 5 hours would fail since 24 hours is not evenly divisible by 5.RUN_INTERVAL Value: { MINUTE[S] | HOUR[S] | DAY[S] } - Default: 1 MINUTE - (Optional) How often the job runs. - The runs take place over a set period of time defined by this interval and they must be divisible by the number of hours in a day. - For example, you can set RUN_INTERVAL to 2 hours (the job runs 12 times per day), but trying to set RUN_INTERVAL to 5 hours would fail since 24 hours is not evenly divisible by 5. - Value: { MINUTE[S] | HOUR[S] | DAY[S] }"""}, + Default: 1 MINUTE"""}, "start_from": {"type": "value", "editable": False, "optional": True, "syntax":"'start_from': '/NOW/BEGINNING'", "description":"""Configures the time to start inserting data from. Data before the specified time is ignored. @@ -145,13 +137,13 @@ "description":"""A description or comment regarding this job."""} }, "snowflake": { - "custom_insert_expressions": {"type": "dict", "editable": True, "optional": True, + "custom_insert_expressions": {"type": "dict_str", "editable": True, "optional": True, "syntax":"'custom_insert_expressions': {'INSERT_TIME' : 'CURRENT_TIMESTAMP()','MY_VALUE': ''}", "description":""" Configure a list of custom expression transformations to apply to the value of each column when inserting unmatched (new) rows. Note this is only used in Merge Jobs. Note: You can use {} as a placeholder for the mapped value from the select statement. Type: array[(column, expression)] Default: ()"""}, - "custom_update_expressions": {"type": "dict", "editable": True, "optional": True, + "custom_update_expressions": {"type": "dict_str", "editable": True, "optional": True, "syntax":"'custom_update_expressions': {'UPDATE_TIME' : 'CURRENT_TIMESTAMP()','MY_VALUE': ''}", "description":"""Configure a list of custom expression transformations to apply to the value of each column when updating matched rows. Note this is only used in Merge Jobs. Note: You can use {} as a placeholder for the mapped value from the select statement. @@ -166,17 +158,13 @@ "description":"""When true, columns that don't exist in the target table are added automatically when encountered. When false, you cannot do SELECT * within the SELECT statement of your transformation job. Default: false"""}, - "run_interval": {"type": "identifier", "editable": False, "optional": True, + "run_interval": {"type": "integer", "editable": False, "optional": True, "syntax":"'run_interval': ''", "description":"""How often the job runs. The runs take place over a set period of time defined by this interval and they must be divisible by the number of hours in a day. For example, you can set RUN_INTERVAL to 2 hours (the job runs 12 times per day), but trying to set RUN_INTERVAL to 5 hours would fail since 24 hours is not evenly divisible by 5.RUN_INTERVAL Value: { MINUTE[S] | HOUR[S] | DAY[S] } - Default: 1 MINUTE - (Optional) How often the job runs. - The runs take place over a set period of time defined by this interval and they must be divisible by the number of hours in a day. - For example, you can set RUN_INTERVAL to 2 hours (the job runs 12 times per day), but trying to set RUN_INTERVAL to 5 hours would fail since 24 hours is not evenly divisible by 5. - Value: { MINUTE[S] | HOUR[S] | DAY[S] }"""}, + Default: 1 MINUTE"""}, "start_from": {"type": "value", "editable": False, "optional": True, "syntax":"'start_from': '/NOW/BEGINNING'", "description":"""Configures the time to start inserting data from. Data before the specified time is ignored. @@ -223,17 +211,13 @@ "description":""" When true, columns that don't exist in the target table are added automatically when encountered. When false, you cannot do SELECT * within the SELECT statement of your transformation job. Default: false"""}, - "run_interval": {"type": "identifier", "editable": False, "optional": True, + "run_interval": {"type": "integer", "editable": False, "optional": True, "syntax":"'run_interval': ''", "description":"""How often the job runs. The runs take place over a set period of time defined by this interval and they must be divisible by the number of hours in a day. For example, you can set RUN_INTERVAL to 2 hours (the job runs 12 times per day), but trying to set RUN_INTERVAL to 5 hours would fail since 24 hours is not evenly divisible by 5.RUN_INTERVAL Value: { MINUTE[S] | HOUR[S] | DAY[S] } - Default: 1 MINUTE - (Optional) How often the job runs. - The runs take place over a set period of time defined by this interval and they must be divisible by the number of hours in a day. - For example, you can set RUN_INTERVAL to 2 hours (the job runs 12 times per day), but trying to set RUN_INTERVAL to 5 hours would fail since 24 hours is not evenly divisible by 5. - Value: { MINUTE[S] | HOUR[S] | DAY[S] }"""}, + Default: 1 MINUTE"""}, "start_from": {"type": "value", "editable": False, "optional": True, "syntax":"'start_from': '/NOW/BEGINNING'", "description":"""Configures the time to start inserting data from. Data before the specified time is ignored. @@ -275,17 +259,13 @@ "description":"""A description or comment regarding this job."""} }, "redshift": { - "run_interval": {"type": "identifier", "editable": False, "optional": True, + "run_interval": {"type": "integer", "editable": False, "optional": True, "syntax":"'run_interval': ''", "description":"""How often the job runs. The runs take place over a set period of time defined by this interval and they must be divisible by the number of hours in a day. For example, you can set RUN_INTERVAL to 2 hours (the job runs 12 times per day), but trying to set RUN_INTERVAL to 5 hours would fail since 24 hours is not evenly divisible by 5.RUN_INTERVAL Value: { MINUTE[S] | HOUR[S] | DAY[S] } - Default: 1 MINUTE - (Optional) How often the job runs. - The runs take place over a set period of time defined by this interval and they must be divisible by the number of hours in a day. - For example, you can set RUN_INTERVAL to 2 hours (the job runs 12 times per day), but trying to set RUN_INTERVAL to 5 hours would fail since 24 hours is not evenly divisible by 5. - Value: { MINUTE[S] | HOUR[S] | DAY[S] }"""}, + Default: 1 MINUTE"""}, "start_from": {"type": "value", "editable": False, "optional": True, "syntax":"'start_from': '/NOW/BEGINNING'", "description":"""Configures the time to start inserting data from. Data before the specified time is ignored. diff --git a/examples/s3_to_athena_new/models/orders_raw_data_2.sql b/examples/s3_to_athena_new/models/orders_raw_data.sql similarity index 86% rename from examples/s3_to_athena_new/models/orders_raw_data_2.sql rename to examples/s3_to_athena_new/models/orders_raw_data.sql index d3f0e3e..0b39e10 100644 --- a/examples/s3_to_athena_new/models/orders_raw_data_2.sql +++ b/examples/s3_to_athena_new/models/orders_raw_data.sql @@ -9,4 +9,4 @@ ) }} -SELECT * FROM {{ ref('s3_connection_2') }} +SELECT * FROM {{ ref('s3_connection') }} diff --git a/examples/s3_to_athena_new/models/orders_transformed_data_2.sql b/examples/s3_to_athena_new/models/orders_transformed_data.sql similarity index 95% rename from examples/s3_to_athena_new/models/orders_transformed_data_2.sql rename to examples/s3_to_athena_new/models/orders_transformed_data.sql index 40fbb8b..1a485ed 100644 --- a/examples/s3_to_athena_new/models/orders_transformed_data_2.sql +++ b/examples/s3_to_athena_new/models/orders_transformed_data.sql @@ -18,7 +18,7 @@ SELECT nettotal AS total, $event_time AS partition_date - FROM {{ ref('orders_raw_data_2')}} + FROM {{ ref('orders_raw_data')}} LET customer_name = customer.firstname || ' ' || customer.lastname WHERE ordertype = 'SHIPPING' AND $event_time BETWEEN run_start_time() AND run_end_time() diff --git a/examples/s3_to_athena_new/models/s3_connection_2.sql b/examples/s3_to_athena_new/models/s3_connection.sql similarity index 100% rename from examples/s3_to_athena_new/models/s3_connection_2.sql rename to examples/s3_to_athena_new/models/s3_connection.sql