diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de8e7abb9..1764c9b55 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,8 +29,8 @@ repos: rev: 24.4.0 hooks: - id: black - - repo: https://github.com/pre-commit/mirrors-pylint - rev: 'v3.0.0a5' # Use the sha / tag you want to point at + - repo: https://github.com/PyCQA/pylint + rev: 'v3.2.0' # Use the sha / tag you want to point at hooks: - id: pylint - repo: https://github.com/PyCQA/flake8 diff --git a/apps/cloud/odc/apps/cloud/azure_to_tar.py b/apps/cloud/odc/apps/cloud/azure_to_tar.py index da1fc9bee..3702c441f 100644 --- a/apps/cloud/odc/apps/cloud/azure_to_tar.py +++ b/apps/cloud/odc/apps/cloud/azure_to_tar.py @@ -35,9 +35,10 @@ def cli( url_prefix = (account_url + "/" + container_name + "/")[len("https://") :] # jam it all in a tar - tar_opts = dict( - name=outfile, mode="w" + tar_mode(gzip=True, xz=True, is_pipe=False) - ) + tar_opts = { + "name": outfile, + "mode": "w" + tar_mode(gzip=True, xz=True, is_pipe=False), + } with tarfile.open(**tar_opts) as tar: for yaml in yamls: add_txt_file(tar=tar, content=yaml[0], fname=url_prefix + yaml[1]) diff --git a/apps/cloud/odc/apps/cloud/s3_find.py b/apps/cloud/odc/apps/cloud/s3_find.py index 673d2c4c1..b835def35 100755 --- a/apps/cloud/odc/apps/cloud/s3_find.py +++ b/apps/cloud/odc/apps/cloud/s3_find.py @@ -52,7 +52,9 @@ def cli(uri, skip_check, no_sign_request=None, request_payer=False): try: stream = s3_find_glob(uri, skip_check=skip_check, s3=s3, **opts) for i, o in enumerate(stream): - print(o.url, flush=(i % flush_freq == 0)) + print( + o.url, flush=(i % flush_freq == 0) + ) # pylint:disable=superfluous-parens except ValueError as ve: click.echo(str(ve), err=True) sys.exit(1) diff --git a/apps/cloud/odc/apps/cloud/s3_to_tar.py b/apps/cloud/odc/apps/cloud/s3_to_tar.py index 688ff885c..af18ea340 100755 --- a/apps/cloud/odc/apps/cloud/s3_to_tar.py +++ b/apps/cloud/odc/apps/cloud/s3_to_tar.py @@ -71,7 +71,7 @@ def dump_to_tar(data_stream, tar): fetcher = S3Fetcher(nconcurrent=nconnections, aws_unsigned=no_sign_request) is_pipe = outfile == "-" - tar_opts = dict(mode="w" + tar_mode(gzip=gzip, xz=xz, is_pipe=is_pipe)) + tar_opts = {"mode": "w" + tar_mode(gzip=gzip, xz=xz, is_pipe=is_pipe)} if is_pipe: if stdout.isatty(): click.echo("Will not write to a terminal", err=True) diff --git a/apps/cloud/odc/apps/cloud/thredds_to_tar.py b/apps/cloud/odc/apps/cloud/thredds_to_tar.py index ea4c4a83c..cb1f47da3 100644 --- a/apps/cloud/odc/apps/cloud/thredds_to_tar.py +++ b/apps/cloud/odc/apps/cloud/thredds_to_tar.py @@ -60,9 +60,10 @@ def cli(thredds_catalogue, skips, select, workers, outfile): yamls = download_yamls(urls, workers) # jam it all in a tar - tar_opts = dict( - name=outfile, mode="w" + tar_mode(gzip=True, xz=True, is_pipe=False) - ) + tar_opts = { + "name": outfile, + "mode": "w" + tar_mode(gzip=True, xz=True, is_pipe=False), + } with tarfile.open(**tar_opts) as tar: for yaml in yamls: add_txt_file(tar=tar, content=yaml[0], fname=yaml[1]) diff --git a/apps/dc_tools/odc/apps/dc_tools/_stac.py b/apps/dc_tools/odc/apps/dc_tools/_stac.py index d01e297bf..6793c9544 100644 --- a/apps/dc_tools/odc/apps/dc_tools/_stac.py +++ b/apps/dc_tools/odc/apps/dc_tools/_stac.py @@ -13,7 +13,7 @@ from eodatasets3.serialise import from_doc from eodatasets3.stac import to_stac_item from toolz import get_in -from urlpath import URL +from urllib.parse import urlparse from ._docs import odc_uuid @@ -179,10 +179,23 @@ def _find_self_href(item: Document) -> str: return self_uri[0] +def _get_relative_path(asset_href, self_link): + if self_link is None: + return asset_href + + self_path = urlparse(self_link).path + href_path = urlparse(asset_href).path + + try: + return str(Path(href_path).relative_to(Path(self_path).parent)) + except ValueError: + # if it's not relative, keep as an absolute link + return asset_href + + def _get_stac_bands( item: Document, default_grid: str, - relative: bool = False, proj_shape: Optional[str] = None, proj_transform: Optional[str] = None, ) -> Tuple[Document, Document, Document]: @@ -195,27 +208,6 @@ def _get_stac_bands( assets = item.get("assets", {}) - def _get_path(asset, force_relative=False): - path = URL(asset["href"]) - if relative: - try: - if self_link is None: - raise ValueError - path = path.relative_to(URL(self_link).parent) - # Value error is raised if the path is not relative to the parent - # or if the self link cannot be found. - except ValueError: - # If the path is not relative to the parent force_relative - # is still used for data assets, due to a historical assumption. - # TODO: Implement rewrite_assets (like in stac_to_dc) in all - # tools so that this is no longer necessary. - if force_relative: - path = path.name - else: - pass - - return str(path) - for asset_name, asset in assets.items(): image_types = ["jp2", "geotiff"] # If something's not in image_types, make it an accessory @@ -223,7 +215,9 @@ def _get_path(asset, force_relative=False): if not any( t in asset.get("type", []) for t in image_types ) or "thumbnail" in asset.get("roles", []): - accessories[asset_name] = {"path": _get_path(asset)} + accessories[asset_name] = { + "path": _get_relative_path(asset["href"], self_link) + } continue # If transform specified here in the asset it should override @@ -240,7 +234,7 @@ def _get_path(asset, force_relative=False): "transform": transform, } - path = _get_path(asset, force_relative=True) + path = _get_relative_path(asset["href"], self_link) band_index = asset.get("band", None) band_info = {"path": path} @@ -275,10 +269,6 @@ def round_coords(c1, c2): return None -def stac_transform_absolute(input_stac): - return stac_transform(input_stac, relative=False) - - def _convert_value_to_eo3_type(key: str, value): """ Convert return type as per EO3 specification. @@ -332,7 +322,7 @@ def _check_valid_uuid(uuid_string: str) -> bool: return False -def stac_transform(input_stac: Document, relative: bool = True) -> Document: +def stac_transform(input_stac: Document) -> Document: """Takes in a raw STAC 1.0 dictionary and returns an ODC dictionary""" # pylint: disable=too-many-locals @@ -371,7 +361,6 @@ def stac_transform(input_stac: Document, relative: bool = True) -> Document: bands, grids, accessories = _get_stac_bands( input_stac, default_grid, - relative=relative, proj_shape=proj_shape, proj_transform=proj_transform, ) diff --git a/apps/dc_tools/odc/apps/dc_tools/_version.py b/apps/dc_tools/odc/apps/dc_tools/_version.py index 699eb888d..aa0be2139 100644 --- a/apps/dc_tools/odc/apps/dc_tools/_version.py +++ b/apps/dc_tools/odc/apps/dc_tools/_version.py @@ -1 +1 @@ -__version__ = "0.2.16" +__version__ = "0.2.17" diff --git a/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py index 8474bc410..733515e81 100644 --- a/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/esa_worldcover_to_dc.py @@ -38,7 +38,7 @@ "{algo}/{year}/map/ESA_WorldCover_10m_{year}_{algo}_{ns}{ew}_Map.tif" ) -map_version = dict(algo="v100", year="2020") +map_version = {"algo": "v100", "year": "2020"} def _unpack_bbox(bounding_box: str) -> Tuple[int, int, int, int]: diff --git a/apps/dc_tools/odc/apps/dc_tools/index_from_tar.py b/apps/dc_tools/odc/apps/dc_tools/index_from_tar.py index 253c2f96f..5cc1c437d 100644 --- a/apps/dc_tools/odc/apps/dc_tools/index_from_tar.py +++ b/apps/dc_tools/odc/apps/dc_tools/index_from_tar.py @@ -126,13 +126,13 @@ def cli( if ignore_lineage: auto_add_lineage = False - ds_resolve_args = dict( - products=product_names, - exclude_products=exclude_product_names, - fail_on_missing_lineage=not auto_add_lineage, - verify_lineage=verify_lineage, - skip_lineage=ignore_lineage, - ) + ds_resolve_args = { + "products": product_names, + "exclude_products": exclude_product_names, + "fail_on_missing_lineage": not auto_add_lineage, + "verify_lineage": verify_lineage, + "skip_lineage": ignore_lineage, + } allowed_changes = {(): allow_any} diff --git a/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py index 2e4f0bd7a..cf3dc0fab 100755 --- a/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/s3_to_dc.py @@ -11,7 +11,7 @@ from datacube import Datacube from datacube.index.hl import Doc2Dataset from odc.apps.dc_tools._docs import parse_doc_stream -from odc.apps.dc_tools._stac import stac_transform, stac_transform_absolute +from odc.apps.dc_tools._stac import stac_transform from odc.apps.dc_tools.utils import ( IndexingException, SkippedException, @@ -26,7 +26,6 @@ statsd_gauge_reporting, statsd_setting, transform_stac, - transform_stac_absolute, update_flag, update_if_exists_flag, verify_lineage, @@ -59,12 +58,15 @@ def dump_to_odc( uris_docs = parse_doc_stream( ((doc.url, doc.data) for doc in document_stream), on_error=doc_error, - transform=transform, ) found_docs = False for uri, metadata in uris_docs: found_docs = True + stac_doc = None + if transform: + stac_doc = metadata + metadata = stac_transform(metadata) try: index_update_dataset( metadata, @@ -76,6 +78,7 @@ def dump_to_odc( allow_unsafe=allow_unsafe, archive_less_mature=archive_less_mature, publish_action=publish_action, + stac_doc=stac_doc, ) ds_added += 1 except IndexingException: @@ -103,7 +106,6 @@ def dump_to_odc( @fail_on_missing_lineage @verify_lineage @transform_stac -@transform_stac_absolute @update_flag @update_if_exists_flag @allow_unsafe @@ -121,7 +123,6 @@ def cli( fail_on_missing_lineage, verify_lineage, stac, - absolute, update, update_if_exists, allow_unsafe, @@ -151,13 +152,6 @@ def cli( datefmt="%m/%d/%Y %I:%M:%S", ) - transform = None - if stac: - if absolute: - transform = stac_transform_absolute - else: - transform = stac_transform - opts = {} if request_payer: opts["RequestPayer"] = "requester" @@ -215,7 +209,7 @@ def cli( skip_lineage=skip_lineage, fail_on_missing_lineage=fail_on_missing_lineage, verify_lineage=verify_lineage, - transform=transform, + transform=stac, update=update, update_if_exists=update_if_exists, allow_unsafe=allow_unsafe, diff --git a/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py index 6ddb321e1..a6c945eec 100644 --- a/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/sqs_to_dc.py @@ -33,14 +33,13 @@ statsd_setting, statsd_gauge_reporting, transform_stac, - transform_stac_absolute, archive_less_mature, update_flag, update_if_exists_flag, verify_lineage, publish_action, ) -from ._stac import stac_transform, stac_transform_absolute, ds_to_stac +from ._stac import stac_transform, ds_to_stac # Added log handler logging.basicConfig(level=logging.WARNING, handlers=[logging.StreamHandler()]) @@ -72,7 +71,7 @@ def extract_action_from_message(message): return None -def handle_json_message(metadata, transform, odc_metadata_link): +def handle_json_message(metadata, odc_metadata_link): odc_yaml_uri = None uri = None @@ -100,9 +99,6 @@ def handle_json_message(metadata, transform, odc_metadata_link): # if no odc_metadata_link provided, it will look for metadata dict "href" value with "rel==self" uri = get_uri(metadata, "self") - if transform: - metadata = transform(metadata) - return metadata, uri @@ -250,13 +246,12 @@ def queue_to_odc( do_archiving(metadata, dc, publish_action) else: if not record_path: - if transform: - stac_doc = metadata # Extract metadata and URI from a STAC or similar # json structure for indexing - metadata, uri = handle_json_message( - metadata, transform, odc_metadata_link - ) + metadata, uri = handle_json_message(metadata, odc_metadata_link) + if transform: + stac_doc = metadata + metadata = stac_transform(metadata) else: # Extract metadata from an S3 bucket notification # or similar for indexing @@ -315,7 +310,6 @@ def queue_to_odc( @fail_on_missing_lineage @verify_lineage @transform_stac -@transform_stac_absolute @update_flag @update_if_exists_flag @allow_unsafe @@ -352,7 +346,6 @@ def cli( fail_on_missing_lineage, verify_lineage, stac, - absolute, update, update_if_exists, allow_unsafe, @@ -370,13 +363,6 @@ def cli( ): """Iterate through messages on an SQS queue and add them to datacube""" - transform = None - if stac: - if absolute: - transform = stac_transform_absolute - else: - transform = stac_transform - candidate_products = product.split() sqs = boto3.resource("sqs") @@ -391,7 +377,7 @@ def cli( skip_lineage=skip_lineage, fail_on_missing_lineage=fail_on_missing_lineage, verify_lineage=verify_lineage, - transform=transform, + transform=stac, limit=limit, update=update, no_sign_request=no_sign_request, diff --git a/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py index b16c2379a..99e7378dd 100644 --- a/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py @@ -4,15 +4,13 @@ import concurrent import json import logging -import os import sys from typing import Any, Dict, Generator, Optional, Tuple import click -import pystac from datacube import Datacube from datacube.index.hl import Doc2Dataset -from odc.apps.dc_tools._stac import stac_transform, stac_transform_absolute +from odc.apps.dc_tools._stac import stac_transform from odc.apps.dc_tools.utils import ( SkippedException, allow_unsafe, @@ -63,57 +61,24 @@ def _parse_options(options: Optional[str]) -> Dict[str, Any]: return parsed_options -def _guess_location( - item: pystac.Item, rewrite: Optional[Tuple[str, str]] = None -) -> Tuple[str, bool]: - self_link = None - asset_link = None - relative = True - +def item_to_meta_uri( + item: Item, + rename_product: Optional[str] = None, +) -> Generator[Tuple[dict, str, bool], None, None]: for link in item.links: if link.rel == "self": - self_link = link.target + uri = link.target # Override self with canonical if link.rel == "canonical": - self_link = link.target + uri = link.target break - for _, asset in item.assets.items(): - if "geotiff" in asset.media_type: - asset_link = os.path.dirname(asset.href) - break - - if rewrite is not None: - for _, asset in item.assets.items(): - if "geotiff" in asset.media_type: - asset.href = asset.href.replace(rewrite[0], rewrite[1]) - - # If the metadata and the document are not on the same path, - # we need to use absolute links and not relative ones. - if (self_link and asset_link) and os.path.dirname(self_link) != os.path.dirname( - asset_link - ): - relative = False - - return self_link, relative - - -def item_to_meta_uri( - item: Item, - rewrite: Optional[Tuple[str, str]] = None, - rename_product: Optional[str] = None, -) -> Generator[Tuple[dict, str, bool], None, None]: - uri, relative = _guess_location(item, rewrite) metadata = item.to_dict() if rename_product is not None: metadata["properties"]["odc:product"] = rename_product stac = metadata - - if relative: - metadata = stac_transform(metadata) - else: - metadata = stac_transform_absolute(metadata) + metadata = stac_transform(metadata) return (metadata, uri, stac) @@ -124,12 +89,11 @@ def process_item( doc2ds: Doc2Dataset, update_if_exists: bool, allow_unsafe: bool, - rewrite: Optional[Tuple[str, str]] = None, rename_product: Optional[str] = None, archive_less_mature: int = None, publish_action: bool = False, ): - meta, uri, stac = item_to_meta_uri(item, rewrite, rename_product) + meta, uri, stac = item_to_meta_uri(item, rename_product) index_update_dataset( meta, uri, @@ -149,7 +113,6 @@ def stac_api_to_odc( config: dict, catalog_href: str, allow_unsafe: bool = True, - rewrite: Optional[Tuple[str, str]] = None, rename_product: Optional[str] = None, archive_less_mature: int = None, publish_action: Optional[str] = None, @@ -182,7 +145,6 @@ def stac_api_to_odc( doc2ds, update_if_exists=update_if_exists, allow_unsafe=allow_unsafe, - rewrite=rewrite, rename_product=rename_product, archive_less_mature=archive_less_mature, publish_action=publish_action, @@ -235,15 +197,6 @@ def stac_api_to_odc( default=None, help="Other search terms, as a # separated list, i.e., --options=cloud_cover=0,100#sky=green", ) -@click.option( - "--rewrite-assets", - type=str, - default=None, - help=( - "Rewrite asset hrefs, for example, to change from " - "HTTPS to S3 URIs, --rewrite-assets=https://example.com/,s3://" - ), -) @rename_product @archive_less_mature @publish_action @@ -257,7 +210,6 @@ def cli( bbox, datetime, options, - rewrite_assets, rename_product, statsd_setting, archive_less_mature, @@ -267,7 +219,6 @@ def cli( Iterate through STAC items from a STAC API and add them to datacube. """ config = _parse_options(options) - rewrite = None # Format the search terms if bbox: @@ -283,13 +234,6 @@ def cli( # number if max_items is not None. config["max_items"] = limit - if rewrite_assets is not None: - rewrite = list(rewrite_assets.split(",")) - if len(rewrite) != 2: - raise ValueError( - "Rewrite assets argument needs to be two strings split by ','" - ) - # Do the thing dc = Datacube() added, failed, skipped = stac_api_to_odc( @@ -298,7 +242,6 @@ def cli( config, catalog_href, allow_unsafe=allow_unsafe, - rewrite=rewrite, rename_product=rename_product, archive_less_mature=archive_less_mature, publish_action=publish_action, diff --git a/apps/dc_tools/tests/test_sqs_to_dc.py b/apps/dc_tools/tests/test_sqs_to_dc.py index 43a3741f3..13ca3a7f5 100644 --- a/apps/dc_tools/tests/test_sqs_to_dc.py +++ b/apps/dc_tools/tests/test_sqs_to_dc.py @@ -140,7 +140,7 @@ def test_extract_metadata_from_message(aws_credentials, odc_test_db_with_product def test_handle_json_message(ga_ls8c_ard_3_message, ga_ls8c_ard_3_yaml): actual_doc, uri = handle_json_message( - ga_ls8c_ard_3_message, None, "STAC-LINKS-REL:odc_yaml" + ga_ls8c_ard_3_message, "STAC-LINKS-REL:odc_yaml" ) assert type(actual_doc) is dict @@ -170,7 +170,7 @@ def test_handle_json_message(ga_ls8c_ard_3_message, ga_ls8c_ard_3_yaml): def test_odc_metadata_link(ga_ls8c_ard_3_message): actual_doc, uri = handle_json_message( - ga_ls8c_ard_3_message, None, "STAC-LINKS-REL:odc_yaml" + ga_ls8c_ard_3_message, "STAC-LINKS-REL:odc_yaml" ) assert ( uri == "https://dea-public-data.s3-ap-southeast-2.amazonaws.com/" @@ -180,7 +180,7 @@ def test_odc_metadata_link(ga_ls8c_ard_3_message): def test_stac_link(ga_ls8c_ard_3_message): - metadata, uri = handle_json_message(ga_ls8c_ard_3_message, stac_transform, None) + metadata, uri = handle_json_message(ga_ls8c_ard_3_message, None) assert ( uri != "https://dea-public-data.s3-ap-southeast-2.amazonaws.com/" "baseline/ga_ls8c_ard_3/088/080/2020/05/25/" @@ -194,7 +194,8 @@ def test_stac_link(ga_ls8c_ard_3_message): def test_transform(ga_ls8c_ard_3_message, ga_ls8c_ard_3_yaml): - actual_doc, uri = handle_json_message(ga_ls8c_ard_3_message, stac_transform, None) + actual_doc, uri = handle_json_message(ga_ls8c_ard_3_message, None) + actual_doc = stac_transform(actual_doc) assert ga_ls8c_ard_3_yaml["id"] == actual_doc["id"] assert ga_ls8c_ard_3_yaml["crs"] == actual_doc["crs"] diff --git a/libs/cloud/odc/aio.py b/libs/cloud/odc/aio.py index b77bcf643..428e00eb4 100644 --- a/libs/cloud/odc/aio.py +++ b/libs/cloud/odc/aio.py @@ -44,7 +44,7 @@ def result(data=None, last_modified=None, error=None): ) bucket, key = s3_url_parse(url) - extra_args = dict(**kw) + extra_args = {**kw} if _range is not None: try: @@ -327,7 +327,7 @@ def __init__( s3_cfg = AioConfig( max_pool_connections=nconcurrent, **opts, - s3=dict(addressing_style=addressing_style), + s3={"addressing_style": addressing_style}, ) self._nconcurrent = nconcurrent @@ -582,4 +582,4 @@ def dirs_pred(f): else: stream = do_dir_query(qq, dirs_pred=dirs_pred) - return stream + return stream # pylint: disable=possibly-used-before-assignment diff --git a/libs/cloud/odc/aws/dns.py b/libs/cloud/odc/aws/dns.py index 2ea73cec0..a2f351b77 100644 --- a/libs/cloud/odc/aws/dns.py +++ b/libs/cloud/odc/aws/dns.py @@ -1,6 +1,8 @@ """ Tools for interacting with route53 """ +import sys + from . import _fetch_text, ec2_tags, mk_boto_session @@ -134,6 +136,4 @@ def display_help(): if __name__ == "__main__": - import sys - sys.exit(cli(sys.argv[1:])) diff --git a/libs/cloud/odc/aws/misc.py b/libs/cloud/odc/aws/misc.py index 3b103d730..eae8d9a0e 100644 --- a/libs/cloud/odc/aws/misc.py +++ b/libs/cloud/odc/aws/misc.py @@ -71,6 +71,6 @@ def build_request( auth.add_auth(req) - return Request(req.url, headers=dict(**req.headers), method="GET") + return Request(req.url, headers={**req.headers}, method="GET") return build_request diff --git a/libs/cloud/odc/aws/queue.py b/libs/cloud/odc/aws/queue.py index c8d0e1108..773d80021 100644 --- a/libs/cloud/odc/aws/queue.py +++ b/libs/cloud/odc/aws/queue.py @@ -136,8 +136,7 @@ def _sqs_message_stream(queue, **kw): if len(messages) == 0: return - for msg in messages: - yield msg + yield from messages def get_messages( diff --git a/libs/io/odc/io/text.py b/libs/io/odc/io/text.py index fde7d9788..c92d19231 100644 --- a/libs/io/odc/io/text.py +++ b/libs/io/odc/io/text.py @@ -73,7 +73,7 @@ def slurp(fname: PathLike, binary: bool = False) -> RawDoc: """ mode = "rb" if binary else "rt" - with open(fname, mode) as f: + with open(fname, mode) as f: # pylint: disable=unspecified-encoding return f.read() @@ -82,7 +82,7 @@ def slurp_lines(fname: str, *args, **kwargs) -> List[str]: if len(args) > 0 or len(kwargs) > 0: fname = fname.format(*args, **kwargs) - with open(fname, "rt") as f: + with open(fname, "rt") as f: # pylint: disable=unspecified-encoding return [s.rstrip() for s in f.readlines()] diff --git a/libs/ui/odc/ui/_dc_explore.py b/libs/ui/odc/ui/_dc_explore.py index f533b6fc9..9fdab60c4 100644 --- a/libs/ui/odc/ui/_dc_explore.py +++ b/libs/ui/odc/ui/_dc_explore.py @@ -59,7 +59,7 @@ def __init__( products = list(p.name for p, c in dc.index.datasets.count_by_product()) if style is None: - style = dict(fillOpacity=0.1, weight=1) + style = {"fillOpacity": 0.1, "weight": 1} state, gui = self._build_ui( products, time, zoom=zoom, center=center, height=height, width=width @@ -169,7 +169,7 @@ def bounds_handler(event): lat1 = max(lat1, -90) lat2 = min(lat2, +90) - state.bounds = dict(lat=(lat1, lat2), lon=(lon1, lon2)) + state.bounds = {"lat": (lat1, lat2), "lon": (lon1, lon2)} self.on_bounds(state.bounds) @@ -228,7 +228,7 @@ def _update_footprints(self): self._clear_footprints() self._dss_layer = new_layer - self._last_query_bounds = dict(**s.bounds) + self._last_query_bounds = {**s.bounds} self._last_query_polygon = _query_polygon(**s.bounds) else: self._clear_footprints() diff --git a/libs/ui/odc/ui/_images.py b/libs/ui/odc/ui/_images.py index 4c1ecf692..2dd2d00a3 100644 --- a/libs/ui/odc/ui/_images.py +++ b/libs/ui/odc/ui/_images.py @@ -71,7 +71,14 @@ def _compress_image(im: np.ndarray, driver="PNG", **opts) -> bytes: else: raise ValueError(f"Expect 2 or 3 dimensional array got: {im.ndim}") - rio_opts = dict(width=w, height=h, count=nc, driver=driver, dtype="uint8", **opts) + rio_opts = { + "width": w, + "height": h, + "count": nc, + "driver": driver, + "dtype": "uint8", + **opts, + } with warnings.catch_warnings(): warnings.simplefilter("ignore", rasterio.errors.NotGeoreferencedWarning) @@ -139,11 +146,11 @@ def mk_image_overlay( ipyleaflet.ImageOverlay or a list of them one per time slice """ - comp, mime = dict( - png=(to_png_data, "image/png"), - jpg=(to_jpeg_data, "image/jpeg"), - jpeg=(to_jpeg_data, "image/jpeg"), - ).get(fmt.lower(), (None, None)) + comp, mime = { + "png": (to_png_data, "image/png"), + "jpg": (to_jpeg_data, "image/jpeg"), + "jpeg": (to_jpeg_data, "image/jpeg"), + }.get(fmt.lower(), (None, None)) if comp is None or mime is None: raise ValueError("Only support png an jpeg formats") diff --git a/libs/ui/odc/ui/_map.py b/libs/ui/odc/ui/_map.py index 7da680163..954262929 100644 --- a/libs/ui/odc/ui/_map.py +++ b/libs/ui/odc/ui/_map.py @@ -33,15 +33,15 @@ def gridspec_to_geojson(gs, xx, yy, styles): def to_geojson(gs, tidx): bbox = gs.tile_geobox(tidx) - return dict( - geometry=bbox.geographic_extent.__geo_interface__, - type="Feature", - properties=dict( + return { + "geometry": bbox.geographic_extent.__geo_interface__, + "type": "Feature", + "properties": { # pylint:disable=consider-using-f-string - title="{:+d},{:+d}".format(*tidx), + "title": "{:+d},{:+d}".format(*tidx), **styles, - ), - ) + }, + } tiles = itertools.product(range(*xx), range(*yy)) @@ -179,9 +179,9 @@ def render_bounds(bounds): shape_opts = {"fillColor": "#fca45d", "color": "#000000", "fillOpacity": 0.1} draw.rectangle = {"shapeOptions": shape_opts, "metric": ["km", "m"]} - poly_opts = {"shapeOptions": dict(**shape_opts)} - poly_opts["shapeOptions"]["original"] = dict(**shape_opts) - poly_opts["shapeOptions"]["editing"] = dict(**shape_opts) + poly_opts = {"shapeOptions": {**shape_opts}} + poly_opts["shapeOptions"]["original"] = {**shape_opts} + poly_opts["shapeOptions"]["editing"] = {**shape_opts} draw.polygon = poly_opts draw.edit = True @@ -200,7 +200,7 @@ def bounds_handler(event): bounds = event["new"] render_bounds(bounds) (lat1, lon1), (lat2, lon2) = bounds - state.bounds = dict(lat=(lat1, lat2), lon=(lon1, lon2)) + state.bounds = {"lat": (lat1, lat2), "lon": (lon1, lon2)} def on_draw(event): v = event["new"] diff --git a/libs/ui/odc/ui/plt_tools.py b/libs/ui/odc/ui/plt_tools.py index 4cf8e9f30..d27235908 100644 --- a/libs/ui/odc/ui/plt_tools.py +++ b/libs/ui/odc/ui/plt_tools.py @@ -27,7 +27,7 @@ def compare_masks( - ``removed`` are pixels that turned False from A->B """ fig, axs = plt.subplots(2, 2, figsize=figsize) - opts = dict(interpolation=interpolation, cmap=cmap, **kw) + opts = {"interpolation": interpolation, "cmap": cmap, **kw} axs[0][0].set_title(names[0]) axs[0][0].imshow(a, **opts)