Skip to content

Commit

Permalink
chore(ingest): improve code formatting (datahub-project#11326)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hsheth2
hsheth2 authored Sep 11, 2024
1 parent 920a6ce commit 3755731
Show file tree
Hide file tree
Showing 58 changed files with 541 additions and 383 deletions.
1 change: 1 addition & 0 deletions metadata-ingestion/src/datahub/api/circuit_breaker/assertion_circuit_breaker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ class AssertionCircuitBreaker(AbstractCircuitBreaker):
The circuit breaker checks if there are passing assertion on the Dataset.
"""

config: AssertionCircuitBreakerConfig

def __init__(self, config: AssertionCircuitBreakerConfig):
Expand Down
8 changes: 5 additions & 3 deletions metadata-ingestion/src/datahub/api/circuit_breaker/circuit_breaker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@ def __init__(
# Select your transport with a defined url endpoint
self.transport = RequestsHTTPTransport(
url=datahub_host + "/api/graphql",
headers={"Authorization": "Bearer " + datahub_token}
if datahub_token is not None
else None,
headers=(
{"Authorization": "Bearer " + datahub_token}
if datahub_token is not None
else None
),
method="POST",
timeout=timeout,
)
Expand Down
26 changes: 15 additions & 11 deletions metadata-ingestion/src/datahub/api/entities/datacontract/datacontract.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,14 +179,16 @@ def generate_mcp(
aspects=[
DataContractPropertiesClass(
entity=self.entity,
schema=[SchemaContractClass(assertion=schema_assertion_urn)]
if schema_assertion_urn
else None,
freshness=[
FreshnessContractClass(assertion=freshness_assertion_urn)
]
if freshness_assertion_urn
else None,
schema=(
[SchemaContractClass(assertion=schema_assertion_urn)]
if schema_assertion_urn
else None
),
freshness=(
[FreshnessContractClass(assertion=freshness_assertion_urn)]
if freshness_assertion_urn
else None
),
dataQuality=[
DataQualityContractClass(assertion=dq_assertion_urn)
for dq_assertion_urn in dq_assertions
Expand All @@ -195,9 +197,11 @@ def generate_mcp(
# Also emit status.
StatusClass(removed=False),
# Emit the contract state as PENDING.
DataContractStatusClass(state=DataContractStateClass.PENDING)
if True
else None,
(
DataContractStatusClass(state=DataContractStateClass.PENDING)
if True
else None
),
],
)

Expand Down
22 changes: 13 additions & 9 deletions metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,14 +190,16 @@ def end_event_mcp(
timestampMillis=end_timestamp_millis,
result=DataProcessInstanceRunResultClass(
type=result,
nativeResultType=result_type
if result_type is not None
else self.orchestrator,
nativeResultType=(
result_type if result_type is not None else self.orchestrator
),
),
attempt=attempt,
durationMillis=(end_timestamp_millis - start_timestamp_millis)
if start_timestamp_millis
else None,
durationMillis=(
(end_timestamp_millis - start_timestamp_millis)
if start_timestamp_millis
else None
),
),
)
yield mcp
Expand Down Expand Up @@ -258,9 +260,11 @@ def generate_mcp(
aspect=DataProcessInstanceRelationships(
upstreamInstances=[str(urn) for urn in self.upstream_urns],
parentTemplate=str(self.template_urn) if self.template_urn else None,
parentInstance=str(self.parent_instance)
if self.parent_instance is not None
else None,
parentInstance=(
str(self.parent_instance)
if self.parent_instance is not None
else None
),
),
)
yield mcp
Expand Down
40 changes: 22 additions & 18 deletions metadata-ingestion/src/datahub/api/entities/dataproduct/dataproduct.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,27 +343,31 @@ def from_datahub(cls, graph: DataHubGraph, id: str) -> DataProduct:
tags: Optional[GlobalTagsClass] = graph.get_aspect(id, GlobalTagsClass)
return DataProduct(
id=id,
display_name=data_product_properties.name
if data_product_properties
else None,
display_name=(
data_product_properties.name if data_product_properties else None
),
domain=domains.domains[0],
description=data_product_properties.description
if data_product_properties
else None,
assets=[e.destinationUrn for e in data_product_properties.assets or []]
if data_product_properties
else None,
description=(
data_product_properties.description if data_product_properties else None
),
assets=(
[e.destinationUrn for e in data_product_properties.assets or []]
if data_product_properties
else None
),
owners=yaml_owners,
terms=[term.urn for term in glossary_terms.terms]
if glossary_terms
else None,
terms=(
[term.urn for term in glossary_terms.terms] if glossary_terms else None
),
tags=[tag.tag for tag in tags.tags] if tags else None,
properties=data_product_properties.customProperties
if data_product_properties
else None,
external_url=data_product_properties.externalUrl
if data_product_properties
else None,
properties=(
data_product_properties.customProperties
if data_product_properties
else None
),
external_url=(
data_product_properties.externalUrl if data_product_properties else None
),
)

def _patch_ownership(
Expand Down
78 changes: 45 additions & 33 deletions metadata-ingestion/src/datahub/api/entities/dataset/dataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,20 +94,24 @@ def from_schema_field(
description=schema_field.description,
label=schema_field.label,
created=schema_field.created.__dict__ if schema_field.created else None,
lastModified=schema_field.lastModified.__dict__
if schema_field.lastModified
else None,
lastModified=(
schema_field.lastModified.__dict__
if schema_field.lastModified
else None
),
recursive=schema_field.recursive,
globalTags=schema_field.globalTags.__dict__
if schema_field.globalTags
else None,
glossaryTerms=schema_field.glossaryTerms.__dict__
if schema_field.glossaryTerms
else None,
globalTags=(
schema_field.globalTags.__dict__ if schema_field.globalTags else None
),
glossaryTerms=(
schema_field.glossaryTerms.__dict__
if schema_field.glossaryTerms
else None
),
isPartitioningKey=schema_field.isPartitioningKey,
jsonProps=json.loads(schema_field.jsonProps)
if schema_field.jsonProps
else None,
jsonProps=(
json.loads(schema_field.jsonProps) if schema_field.jsonProps else None
),
)

@validator("urn", pre=True, always=True)
Expand Down Expand Up @@ -300,9 +304,11 @@ def generate_mcp(
properties=[
StructuredPropertyValueAssignmentClass(
propertyUrn=f"urn:li:structuredProperty:{prop_key}",
values=prop_value
if isinstance(prop_value, list)
else [prop_value],
values=(
prop_value
if isinstance(prop_value, list)
else [prop_value]
),
)
for prop_key, prop_value in field.structured_properties.items()
]
Expand Down Expand Up @@ -359,9 +365,11 @@ def generate_mcp(
properties=[
StructuredPropertyValueAssignmentClass(
propertyUrn=f"urn:li:structuredProperty:{prop_key}",
values=prop_value
if isinstance(prop_value, list)
else [prop_value],
values=(
prop_value
if isinstance(prop_value, list)
else [prop_value]
),
)
for prop_key, prop_value in self.structured_properties.items()
]
Expand Down Expand Up @@ -501,25 +509,29 @@ def from_datahub(cls, graph: DataHubGraph, urn: str) -> "Dataset":

return Dataset( # type: ignore[call-arg]
urn=urn,
description=dataset_properties.description
if dataset_properties and dataset_properties.description
else None,
name=dataset_properties.name
if dataset_properties and dataset_properties.name
else None,
description=(
dataset_properties.description
if dataset_properties and dataset_properties.description
else None
),
name=(
dataset_properties.name
if dataset_properties and dataset_properties.name
else None
),
schema=Dataset._schema_from_schema_metadata(graph, urn),
tags=[tag.tag for tag in tags.tags] if tags else None,
glossary_terms=[term.urn for term in glossary_terms.terms]
if glossary_terms
else None,
glossary_terms=(
[term.urn for term in glossary_terms.terms] if glossary_terms else None
),
owners=yaml_owners,
properties=dataset_properties.customProperties
if dataset_properties
else None,
properties=(
dataset_properties.customProperties if dataset_properties else None
),
subtypes=[subtype for subtype in subtypes.typeNames] if subtypes else None,
structured_properties=structured_properties_map
if structured_properties
else None,
structured_properties=(
structured_properties_map if structured_properties else None
),
)

def to_yaml(
Expand Down
20 changes: 12 additions & 8 deletions metadata-ingestion/src/datahub/api/entities/forms/forms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,11 +197,13 @@ def validate_prompts(self, emitter: DataHubGraph) -> List[FormPromptClass]:
title=prompt.title,
description=prompt.description,
type=prompt.type,
structuredPropertyParams=StructuredPropertyParamsClass(
urn=prompt.structured_property_urn
)
if prompt.structured_property_urn
else None,
structuredPropertyParams=(
StructuredPropertyParamsClass(
urn=prompt.structured_property_urn
)
if prompt.structured_property_urn
else None
),
required=prompt.required,
)
)
Expand Down Expand Up @@ -339,9 +341,11 @@ def from_datahub(graph: DataHubGraph, urn: str) -> "Forms":
title=prompt_raw.title,
description=prompt_raw.description,
type=prompt_raw.type,
structured_property_urn=prompt_raw.structuredPropertyParams.urn
if prompt_raw.structuredPropertyParams
else None,
structured_property_urn=(
prompt_raw.structuredPropertyParams.urn
if prompt_raw.structuredPropertyParams
else None
),
)
)
return Forms(
Expand Down
60 changes: 33 additions & 27 deletions metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,19 +126,23 @@ def create(file: str) -> None:
],
cardinality=structuredproperty.cardinality,
immutable=structuredproperty.immutable,
allowedValues=[
PropertyValueClass(
value=v.value, description=v.description
)
for v in structuredproperty.allowed_values
]
if structuredproperty.allowed_values
else None,
typeQualifier={
"allowedTypes": structuredproperty.type_qualifier.allowed_types
}
if structuredproperty.type_qualifier
else None,
allowedValues=(
[
PropertyValueClass(
value=v.value, description=v.description
)
for v in structuredproperty.allowed_values
]
if structuredproperty.allowed_values
else None
),
typeQualifier=(
{
"allowedTypes": structuredproperty.type_qualifier.allowed_types
}
if structuredproperty.type_qualifier
else None
),
),
)
emitter.emit_mcp(mcp)
Expand All @@ -160,20 +164,22 @@ def from_datahub(cls, graph: DataHubGraph, urn: str) -> "StructuredProperties":
description=structured_property.description,
entity_types=structured_property.entityTypes,
cardinality=structured_property.cardinality,
allowed_values=[
AllowedValue(
value=av.value,
description=av.description,
)
for av in structured_property.allowedValues or []
]
if structured_property.allowedValues is not None
else None,
type_qualifier={
"allowed_types": structured_property.typeQualifier.get("allowedTypes")
}
if structured_property.typeQualifier
else None,
allowed_values=(
[
AllowedValue(
value=av.value,
description=av.description,
)
for av in structured_property.allowedValues or []
]
if structured_property.allowedValues is not None
else None
),
type_qualifier=(
{"allowed_types": structured_property.typeQualifier.get("allowedTypes")}
if structured_property.typeQualifier
else None
),
)

def to_yaml(
Expand Down
8 changes: 5 additions & 3 deletions metadata-ingestion/src/datahub/api/graphql/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,11 @@ def __init__(
# Select your transport with a defined url endpoint
self.transport = RequestsHTTPTransport(
url=datahub_host + "/api/graphql",
headers={"Authorization": "Bearer " + datahub_token}
if datahub_token is not None
else None,
headers=(
{"Authorization": "Bearer " + datahub_token}
if datahub_token is not None
else None
),
method="POST",
timeout=timeout,
)
Expand Down
Loading

0 comments on commit 3755731

Please sign in to comment.