Skip to content

Commit

Permalink
[YAML] Better docs for Filter and MapToFields. (apache#33274)
Browse files Browse the repository at this point in the history
* [YAML] Better docs for Filter and MapToFields.

* Remove redundant optional indicators.

* Update sdks/python/apache_beam/yaml/yaml_mapping.py

Co-authored-by: Jeff Kinard <[email protected]>

---------

Co-authored-by: Jeff Kinard <[email protected]>
  • Loading branch information
robertwb and Polber authored Dec 18, 2024
1 parent d17f77a commit 330f57b
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 3 deletions.
2 changes: 1 addition & 1 deletion sdks/python/apache_beam/yaml/generate_yaml_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,7 @@ def main():
if options.markdown_file or options.html_file:
if '-' in transforms[0]:
extra_docs = 'Supported languages: ' + ', '.join(
t.split('-')[-1] for t in sorted(transforms))
t.split('-')[-1] for t in sorted(transforms)) + '.'
else:
extra_docs = ''
markdown_out.write(
Expand Down
30 changes: 28 additions & 2 deletions sdks/python/apache_beam/yaml/yaml_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from typing import Callable
from typing import Collection
from typing import Dict
from typing import Iterable
from typing import List
from typing import Mapping
from typing import Optional
Expand Down Expand Up @@ -619,6 +620,13 @@ def _PyJsFilter(
See more complete documentation on
[YAML Filtering](https://beam.apache.org/documentation/sdks/yaml-udf/#filtering).
Args:
keep: An expression evaluating to true for those records that should be kept.
language: The language of the above expression.
Defaults to generic.
error_handling: Whether and where to output records that throw errors when
the above expressions are evaluated.
""" # pylint: disable=line-too-long
keep_fn = _as_callable_for_pcoll(pcoll, keep, "keep", language or 'generic')
return pcoll | beam.Filter(keep_fn)
Expand Down Expand Up @@ -664,14 +672,32 @@ def normalize_fields(pcoll, fields, drop=(), append=False, language='generic'):

@beam.ptransform.ptransform_fn
@maybe_with_exception_handling_transform_fn
def _PyJsMapToFields(pcoll, language='generic', **mapping_args):
def _PyJsMapToFields(
pcoll,
fields: Mapping[str, Union[str, Mapping[str, str]]],
append: Optional[bool] = False,
drop: Optional[Iterable[str]] = None,
language: Optional[str] = None):
"""Creates records with new fields defined in terms of the input fields.
See more complete documentation on
[YAML Mapping Functions](https://beam.apache.org/documentation/sdks/yaml-udf/#mapping-functions).
Args:
fields: The output fields to compute, each mapping to the expression or
callable that creates them.
append: Whether to append the created fields to the set of
fields already present, outputting a union of both the new fields and
the original fields for each record. Defaults to False.
drop: If `append` is true, enumerates a subset of fields from the
original record that should not be kept
language: The language used to define (and execute) the
expressions and/or callables in `fields`. Defaults to generic.
error_handling: Whether and where to output records that throw errors when
the above expressions are evaluated.
""" # pylint: disable=line-too-long
input_schema, fields = normalize_fields(
pcoll, language=language, **mapping_args)
pcoll, fields, drop or (), append, language=language or 'generic')
if language == 'javascript':
options.YamlOptions.check_enabled(pcoll.pipeline, 'javascript')

Expand Down

0 comments on commit 330f57b

Please sign in to comment.