From 99739556c6e607b5254b1ab14ecc0448db550623 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Fri, 13 Dec 2024 15:42:38 -0800 Subject: [PATCH 1/5] skeleton: components module from dynamic text input --- .../parsers/model_to_component_factory.py | 55 ++++++++++++++++--- .../test/utils/manifest_only_fixtures.py | 20 +++++++ 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 215d6fff..65b1e0ea 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -8,6 +8,7 @@ import importlib import inspect import re +import types from functools import partial from typing import ( Any, @@ -986,8 +987,25 @@ def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> :param config: The custom defined connector config :return: The declarative component built from the Pydantic model to be used at runtime """ + INJECTED_COMPONENTS_PY = "__injected_components_py" - custom_component_class = self._get_class_from_fully_qualified_class_name(model.class_name) + components_module: Optional[types.ModuleType] = None + if INJECTED_COMPONENTS_PY in config: + # declares a dynamic module `components` from provided text + python_text = config[INJECTED_COMPONENTS_PY] + module_name = "components" + + # Create a new module object + components_module = types.ModuleType(module_name) + # Execute the module text in the module's namespace + exec(python_text, components_module.__dict__) + # Skip insert the module into sys.modules because we pass by reference below + # sys.modules[module_name] = components_module + + custom_component_class = self._get_class_from_fully_qualified_class_name( + full_qualified_class_name=model.class_name, + components_module=components_module, + ) component_fields = get_type_hints(custom_component_class) model_args = model.dict() model_args["config"] = config @@ -1039,15 +1057,38 @@ def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> } return custom_component_class(**kwargs) - @staticmethod - def _get_class_from_fully_qualified_class_name(full_qualified_class_name: str) -> Any: + def _get_class_from_fully_qualified_class_name( + full_qualified_class_name: str, + components_module: Optional[types.ModuleType] = None, + ) -> Any: + """ + Get a class from its fully qualified name, optionally using a pre-parsed module. + + Args: + full_qualified_class_name (str): The fully qualified name of the class (e.g., "module.ClassName"). + components_module (Optional[ModuleType]): An optional pre-parsed module. + + Returns: + Any: The class object. + + Raises: + ValueError: If the class cannot be loaded. + """ split = full_qualified_class_name.split(".") - module = ".".join(split[:-1]) + module_name = ".".join(split[:-1]) class_name = split[-1] + try: - return getattr(importlib.import_module(module), class_name) - except AttributeError: - raise ValueError(f"Could not load class {full_qualified_class_name}.") + # Use the provided module if available and if module name matches + if components_module and components_module.__name__ == module_name: + return getattr(components_module, class_name) + + # Fallback to importing the module dynamically + module = importlib.import_module(module_name) + return getattr(module, class_name) + + except (AttributeError, ModuleNotFoundError) as e: + raise ValueError(f"Could not load class {full_qualified_class_name}.") from e @staticmethod def _derive_component_type_from_type_hints(field_type: Any) -> Optional[str]: diff --git a/airbyte_cdk/test/utils/manifest_only_fixtures.py b/airbyte_cdk/test/utils/manifest_only_fixtures.py index 47620e7c..01b2b393 100644 --- a/airbyte_cdk/test/utils/manifest_only_fixtures.py +++ b/airbyte_cdk/test/utils/manifest_only_fixtures.py @@ -2,6 +2,7 @@ import importlib.util +import types from pathlib import Path from types import ModuleType from typing import Optional @@ -51,6 +52,25 @@ def components_module(connector_dir: Path) -> Optional[ModuleType]: return components_module +def components_module_from_string(components_py_text: str) -> Optional[ModuleType]: + """Load and return the components module from a provided string containing the python code. + + This assumes the components module is located at /components.py. + + TODO: Make new unit test to leverage this fixture + """ + module_name = "components" + + # Create a new module object + components_module = types.ModuleType(name=module_name) + + # Execute the module text in the module's namespace + exec(components_py_text, components_module.__dict__) + + # Now you can import and use the module + return components_module + + @pytest.fixture(scope="session") def manifest_path(connector_dir: Path) -> Path: """Return the path to the connector's manifest file.""" From 8309f7910c223568a8b516ae1336f9a20a80bd9e Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 16 Dec 2024 08:52:17 -0800 Subject: [PATCH 2/5] refactor / clean up --- .../parsers/model_to_component_factory.py | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 65b1e0ea..8d4ded84 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -987,21 +987,7 @@ def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> :param config: The custom defined connector config :return: The declarative component built from the Pydantic model to be used at runtime """ - INJECTED_COMPONENTS_PY = "__injected_components_py" - - components_module: Optional[types.ModuleType] = None - if INJECTED_COMPONENTS_PY in config: - # declares a dynamic module `components` from provided text - python_text = config[INJECTED_COMPONENTS_PY] - module_name = "components" - - # Create a new module object - components_module = types.ModuleType(module_name) - # Execute the module text in the module's namespace - exec(python_text, components_module.__dict__) - # Skip insert the module into sys.modules because we pass by reference below - # sys.modules[module_name] = components_module - + components_module = self._get_components_module_object(config=config) custom_component_class = self._get_class_from_fully_qualified_class_name( full_qualified_class_name=model.class_name, components_module=components_module, @@ -1057,9 +1043,31 @@ def create_custom_component(self, model: Any, config: Config, **kwargs: Any) -> } return custom_component_class(**kwargs) + def _get_components_module_object( + config: Config, + ) -> None: + """Get a components module object based on the provided config. + + If custom python components is provided, this will be loaded. Otherwise, we will + attempt to load from the `components` module already imported. + """ + INJECTED_COMPONENTS_PY = "__injected_components_py" + COMPONENTS_MODULE_NAME = "components" + + components_module: types.ModuleType + if INJECTED_COMPONENTS_PY in config: + # Create a new module object and execute the provided Python code text within it + components_module = types.ModuleType(name=COMPONENTS_MODULE_NAME) + python_text = config[INJECTED_COMPONENTS_PY] + exec(python_text, components_module.__dict__) + # Skip insert the module into sys.modules because we pass by reference below + # sys.modules[module_name] = components_module + else: + components_module = importlib.import_module(name=COMPONENTS_MODULE_NAME) + def _get_class_from_fully_qualified_class_name( full_qualified_class_name: str, - components_module: Optional[types.ModuleType] = None, + components_module: types.ModuleType, ) -> Any: """ Get a class from its fully qualified name, optionally using a pre-parsed module. @@ -1075,18 +1083,17 @@ def _get_class_from_fully_qualified_class_name( ValueError: If the class cannot be loaded. """ split = full_qualified_class_name.split(".") - module_name = ".".join(split[:-1]) + module_name_full = ".".join(split[:-1]) + module_name = split[:-2] class_name = split[-1] - try: - # Use the provided module if available and if module name matches - if components_module and components_module.__name__ == module_name: - return getattr(components_module, class_name) - - # Fallback to importing the module dynamically - module = importlib.import_module(module_name) - return getattr(module, class_name) + if module_name != "components": + raise ValueError( + f"Custom components must be defined in a module named `components`. Found {module_name} instead." + ) + try: + return getattr(components_module, class_name) except (AttributeError, ModuleNotFoundError) as e: raise ValueError(f"Could not load class {full_qualified_class_name}.") from e From 399dd7ba5a36d93891ff1592158b0ef46a87716c Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Wed, 18 Dec 2024 16:07:47 -0800 Subject: [PATCH 3/5] add test resource for py_components unit test --- .../resources/valid_py_components_code.py | 15 + .../resources/valid_py_components_config.json | 3 + .../valid_py_components_manifest.yaml | 1368 +++++++++++++++++ 3 files changed, 1386 insertions(+) create mode 100644 unit_tests/source_declarative_manifest/resources/valid_py_components_code.py create mode 100644 unit_tests/source_declarative_manifest/resources/valid_py_components_config.json create mode 100644 unit_tests/source_declarative_manifest/resources/valid_py_components_manifest.yaml diff --git a/unit_tests/source_declarative_manifest/resources/valid_py_components_code.py b/unit_tests/source_declarative_manifest/resources/valid_py_components_code.py new file mode 100644 index 00000000..06c95e78 --- /dev/null +++ b/unit_tests/source_declarative_manifest/resources/valid_py_components_code.py @@ -0,0 +1,15 @@ +"""Custom Python components.py file for testing. + +This file is mostly a no-op (for now) but should trigger a failure if code file is not +correctly parsed. +""" + +from airbyte_cdk.sources.declarative.models import DeclarativeStream + + +class CustomDeclarativeStream(DeclarativeStream): + """Custom declarative stream class. + + We don't change anything from the base class, but this should still be enough to confirm + that the components.py file is correctly parsed. + """ diff --git a/unit_tests/source_declarative_manifest/resources/valid_py_components_config.json b/unit_tests/source_declarative_manifest/resources/valid_py_components_config.json new file mode 100644 index 00000000..214fc684 --- /dev/null +++ b/unit_tests/source_declarative_manifest/resources/valid_py_components_config.json @@ -0,0 +1,3 @@ +{ + "pokemon_name": "blastoise" +} diff --git a/unit_tests/source_declarative_manifest/resources/valid_py_components_manifest.yaml b/unit_tests/source_declarative_manifest/resources/valid_py_components_manifest.yaml new file mode 100644 index 00000000..bf15e313 --- /dev/null +++ b/unit_tests/source_declarative_manifest/resources/valid_py_components_manifest.yaml @@ -0,0 +1,1368 @@ +version: 3.9.6 + +type: DeclarativeSource + +description: This is just a test, with custom Python components enabled. Copied from Pokemon example. + +check: + type: CheckStream + stream_names: + - pokemon + +definitions: + streams: + pokemon: + type: components.CustomDeclarativeStream + name: pokemon + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/base_requester" + path: /{{config['pokemon_name']}} + http_method: GET + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $ref: "#/schemas/pokemon" + base_requester: + type: HttpRequester + url_base: https://pokeapi.co/api/v2/pokemon + +streams: + - $ref: "#/definitions/streams/pokemon" + +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - pokemon_name + properties: + pokemon_name: + type: string + description: Pokemon requested from the API. + enum: + - bulbasaur + - ivysaur + - venusaur + - charmander + - charmeleon + - charizard + - squirtle + - wartortle + - blastoise + - caterpie + - metapod + - butterfree + - weedle + - kakuna + - beedrill + - pidgey + - pidgeotto + - pidgeot + - rattata + - raticate + - spearow + - fearow + - ekans + - arbok + - pikachu + - raichu + - sandshrew + - sandslash + - nidoranf + - nidorina + - nidoqueen + - nidoranm + - nidorino + - nidoking + - clefairy + - clefable + - vulpix + - ninetales + - jigglypuff + - wigglytuff + - zubat + - golbat + - oddish + - gloom + - vileplume + - paras + - parasect + - venonat + - venomoth + - diglett + - dugtrio + - meowth + - persian + - psyduck + - golduck + - mankey + - primeape + - growlithe + - arcanine + - poliwag + - poliwhirl + - poliwrath + - abra + - kadabra + - alakazam + - machop + - machoke + - machamp + - bellsprout + - weepinbell + - victreebel + - tentacool + - tentacruel + - geodude + - graveler + - golem + - ponyta + - rapidash + - slowpoke + - slowbro + - magnemite + - magneton + - farfetchd + - doduo + - dodrio + - seel + - dewgong + - grimer + - muk + - shellder + - cloyster + - gastly + - haunter + - gengar + - onix + - drowzee + - hypno + - krabby + - kingler + - voltorb + - electrode + - exeggcute + - exeggutor + - cubone + - marowak + - hitmonlee + - hitmonchan + - lickitung + - koffing + - weezing + - rhyhorn + - rhydon + - chansey + - tangela + - kangaskhan + - horsea + - seadra + - goldeen + - seaking + - staryu + - starmie + - mrmime + - scyther + - jynx + - electabuzz + - magmar + - pinsir + - tauros + - magikarp + - gyarados + - lapras + - ditto + - eevee + - vaporeon + - jolteon + - flareon + - porygon + - omanyte + - omastar + - kabuto + - kabutops + - aerodactyl + - snorlax + - articuno + - zapdos + - moltres + - dratini + - dragonair + - dragonite + - mewtwo + - mew + - chikorita + - bayleef + - meganium + - cyndaquil + - quilava + - typhlosion + - totodile + - croconaw + - feraligatr + - sentret + - furret + - hoothoot + - noctowl + - ledyba + - ledian + - spinarak + - ariados + - crobat + - chinchou + - lanturn + - pichu + - cleffa + - igglybuff + - togepi + - togetic + - natu + - xatu + - mareep + - flaaffy + - ampharos + - bellossom + - marill + - azumarill + - sudowoodo + - politoed + - hoppip + - skiploom + - jumpluff + - aipom + - sunkern + - sunflora + - yanma + - wooper + - quagsire + - espeon + - umbreon + - murkrow + - slowking + - misdreavus + - unown + - wobbuffet + - girafarig + - pineco + - forretress + - dunsparce + - gligar + - steelix + - snubbull + - granbull + - qwilfish + - scizor + - shuckle + - heracross + - sneasel + - teddiursa + - ursaring + - slugma + - magcargo + - swinub + - piloswine + - corsola + - remoraid + - octillery + - delibird + - mantine + - skarmory + - houndour + - houndoom + - kingdra + - phanpy + - donphan + - porygon2 + - stantler + - smeargle + - tyrogue + - hitmontop + - smoochum + - elekid + - magby + - miltank + - blissey + - raikou + - entei + - suicune + - larvitar + - pupitar + - tyranitar + - lugia + - ho-oh + - celebi + - treecko + - grovyle + - sceptile + - torchic + - combusken + - blaziken + - mudkip + - marshtomp + - swampert + - poochyena + - mightyena + - zigzagoon + - linoone + - wurmple + - silcoon + - beautifly + - cascoon + - dustox + - lotad + - lombre + - ludicolo + - seedot + - nuzleaf + - shiftry + - taillow + - swellow + - wingull + - pelipper + - ralts + - kirlia + - gardevoir + - surskit + - masquerain + - shroomish + - breloom + - slakoth + - vigoroth + - slaking + - nincada + - ninjask + - shedinja + - whismur + - loudred + - exploud + - makuhita + - hariyama + - azurill + - nosepass + - skitty + - delcatty + - sableye + - mawile + - aron + - lairon + - aggron + - meditite + - medicham + - electrike + - manectric + - plusle + - minun + - volbeat + - illumise + - roselia + - gulpin + - swalot + - carvanha + - sharpedo + - wailmer + - wailord + - numel + - camerupt + - torkoal + - spoink + - grumpig + - spinda + - trapinch + - vibrava + - flygon + - cacnea + - cacturne + - swablu + - altaria + - zangoose + - seviper + - lunatone + - solrock + - barboach + - whiscash + - corphish + - crawdaunt + - baltoy + - claydol + - lileep + - cradily + - anorith + - armaldo + - feebas + - milotic + - castform + - kecleon + - shuppet + - banette + - duskull + - dusclops + - tropius + - chimecho + - absol + - wynaut + - snorunt + - glalie + - spheal + - sealeo + - walrein + - clamperl + - huntail + - gorebyss + - relicanth + - luvdisc + - bagon + - shelgon + - salamence + - beldum + - metang + - metagross + - regirock + - regice + - registeel + - latias + - latios + - kyogre + - groudon + - rayquaza + - jirachi + - deoxys + - turtwig + - grotle + - torterra + - chimchar + - monferno + - infernape + - piplup + - prinplup + - empoleon + - starly + - staravia + - staraptor + - bidoof + - bibarel + - kricketot + - kricketune + - shinx + - luxio + - luxray + - budew + - roserade + - cranidos + - rampardos + - shieldon + - bastiodon + - burmy + - wormadam + - mothim + - combee + - vespiquen + - pachirisu + - buizel + - floatzel + - cherubi + - cherrim + - shellos + - gastrodon + - ambipom + - drifloon + - drifblim + - buneary + - lopunny + - mismagius + - honchkrow + - glameow + - purugly + - chingling + - stunky + - skuntank + - bronzor + - bronzong + - bonsly + - mimejr + - happiny + - chatot + - spiritomb + - gible + - gabite + - garchomp + - munchlax + - riolu + - lucario + - hippopotas + - hippowdon + - skorupi + - drapion + - croagunk + - toxicroak + - carnivine + - finneon + - lumineon + - mantyke + - snover + - abomasnow + - weavile + - magnezone + - lickilicky + - rhyperior + - tangrowth + - electivire + - magmortar + - togekiss + - yanmega + - leafeon + - glaceon + - gliscor + - mamoswine + - porygon-z + - gallade + - probopass + - dusknoir + - froslass + - rotom + - uxie + - mesprit + - azelf + - dialga + - palkia + - heatran + - regigigas + - giratina + - cresselia + - phione + - manaphy + - darkrai + - shaymin + - arceus + - victini + - snivy + - servine + - serperior + - tepig + - pignite + - emboar + - oshawott + - dewott + - samurott + - patrat + - watchog + - lillipup + - herdier + - stoutland + - purrloin + - liepard + - pansage + - simisage + - pansear + - simisear + - panpour + - simipour + - munna + - musharna + - pidove + - tranquill + - unfezant + - blitzle + - zebstrika + - roggenrola + - boldore + - gigalith + - woobat + - swoobat + - drilbur + - excadrill + - audino + - timburr + - gurdurr + - conkeldurr + - tympole + - palpitoad + - seismitoad + - throh + - sawk + - sewaddle + - swadloon + - leavanny + - venipede + - whirlipede + - scolipede + - cottonee + - whimsicott + - petilil + - lilligant + - basculin + - sandile + - krokorok + - krookodile + - darumaka + - darmanitan + - maractus + - dwebble + - crustle + - scraggy + - scrafty + - sigilyph + - yamask + - cofagrigus + - tirtouga + - carracosta + - archen + - archeops + - trubbish + - garbodor + - zorua + - zoroark + - minccino + - cinccino + - gothita + - gothorita + - gothitelle + - solosis + - duosion + - reuniclus + - ducklett + - swanna + - vanillite + - vanillish + - vanilluxe + - deerling + - sawsbuck + - emolga + - karrablast + - escavalier + - foongus + - amoonguss + - frillish + - jellicent + - alomomola + - joltik + - galvantula + - ferroseed + - ferrothorn + - klink + - klang + - klinklang + - tynamo + - eelektrik + - eelektross + - elgyem + - beheeyem + - litwick + - lampent + - chandelure + - axew + - fraxure + - haxorus + - cubchoo + - beartic + - cryogonal + - shelmet + - accelgor + - stunfisk + - mienfoo + - mienshao + - druddigon + - golett + - golurk + - pawniard + - bisharp + - bouffalant + - rufflet + - braviary + - vullaby + - mandibuzz + - heatmor + - durant + - deino + - zweilous + - hydreigon + - larvesta + - volcarona + - cobalion + - terrakion + - virizion + - tornadus + - thundurus + - reshiram + - zekrom + - landorus + - kyurem + - keldeo + - meloetta + - genesect + - chespin + - quilladin + - chesnaught + - fennekin + - braixen + - delphox + - froakie + - frogadier + - greninja + - bunnelby + - diggersby + - fletchling + - fletchinder + - talonflame + - scatterbug + - spewpa + - vivillon + - litleo + - pyroar + - flabebe + - floette + - florges + - skiddo + - gogoat + - pancham + - pangoro + - furfrou + - espurr + - meowstic + - honedge + - doublade + - aegislash + - spritzee + - aromatisse + - swirlix + - slurpuff + - inkay + - malamar + - binacle + - barbaracle + - skrelp + - dragalge + - clauncher + - clawitzer + - helioptile + - heliolisk + - tyrunt + - tyrantrum + - amaura + - aurorus + - sylveon + - hawlucha + - dedenne + - carbink + - goomy + - sliggoo + - goodra + - klefki + - phantump + - trevenant + - pumpkaboo + - gourgeist + - bergmite + - avalugg + - noibat + - noivern + - xerneas + - yveltal + - zygarde + - diancie + - hoopa + - volcanion + - rowlet + - dartrix + - decidueye + - litten + - torracat + - incineroar + - popplio + - brionne + - primarina + - pikipek + - trumbeak + - toucannon + - yungoos + - gumshoos + - grubbin + - charjabug + - vikavolt + - crabrawler + - crabominable + - oricorio + - cutiefly + - ribombee + - rockruff + - lycanroc + - wishiwashi + - mareanie + - toxapex + - mudbray + - mudsdale + - dewpider + - araquanid + - fomantis + - lurantis + - morelull + - shiinotic + - salandit + - salazzle + - stufful + - bewear + - bounsweet + - steenee + - tsareena + - comfey + - oranguru + - passimian + - wimpod + - golisopod + - sandygast + - palossand + - pyukumuku + - typenull + - silvally + - minior + - komala + - turtonator + - togedemaru + - mimikyu + - bruxish + - drampa + - dhelmise + - jangmo-o + - hakamo-o + - kommo-o + - tapukoko + - tapulele + - tapubulu + - tapufini + - cosmog + - cosmoem + - solgaleo + - lunala + - nihilego + - buzzwole + - pheromosa + - xurkitree + - celesteela + - kartana + - guzzlord + - necrozma + - magearna + - marshadow + - poipole + - naganadel + - stakataka + - blacephalon + - zeraora + - meltan + - melmetal + - grookey + - thwackey + - rillaboom + - scorbunny + - raboot + - cinderace + - sobble + - drizzile + - inteleon + - skwovet + - greedent + - rookidee + - corvisquire + - corviknight + - blipbug + - dottler + - orbeetle + - nickit + - thievul + - gossifleur + - eldegoss + - wooloo + - dubwool + - chewtle + - drednaw + - yamper + - boltund + - rolycoly + - carkol + - coalossal + - applin + - flapple + - appletun + - silicobra + - sandaconda + - cramorant + - arrokuda + - barraskewda + - toxel + - toxtricity + - sizzlipede + - centiskorch + - clobbopus + - grapploct + - sinistea + - polteageist + - hatenna + - hattrem + - hatterene + - impidimp + - morgrem + - grimmsnarl + - obstagoon + - perrserker + - cursola + - sirfetchd + - mrrime + - runerigus + - milcery + - alcremie + - falinks + - pincurchin + - snom + - frosmoth + - stonjourner + - eiscue + - indeedee + - morpeko + - cufant + - copperajah + - dracozolt + - arctozolt + - dracovish + - arctovish + - duraludon + - dreepy + - drakloak + - dragapult + - zacian + - zamazenta + - eternatus + - kubfu + - urshifu + - zarude + - regieleki + - regidrago + - glastrier + - spectrier + - calyrex + order: 0 + title: Pokemon Name + pattern: ^[a-z0-9_\-]+$ + examples: + - ditto + - luxray + - snorlax + additionalProperties: true + +metadata: + testedStreams: + pokemon: + hasRecords: true + streamHash: f619395f8c7a553f51cec2a7274a4ce517ab46c8 + hasResponse: true + primaryKeysAreUnique: true + primaryKeysArePresent: true + responsesAreSuccessful: true + autoImportSchema: + pokemon: false + +schemas: + pokemon: + type: object + $schema: http://json-schema.org/draft-07/schema# + properties: + id: + type: + - "null" + - integer + name: + type: + - "null" + - string + forms: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + moves: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + move: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + version_group_details: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + version_group: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + level_learned_at: + type: + - "null" + - integer + move_learn_method: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + additionalProperties: true + additionalProperties: true + order: + type: + - "null" + - integer + stats: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + stat: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + effort: + type: + - "null" + - integer + base_stat: + type: + - "null" + - integer + additionalProperties: true + types: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + type: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + slot: + type: + - "null" + - integer + additionalProperties: true + height: + type: + - "null" + - integer + weight: + type: + - "null" + - integer + species: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + sprites: + type: + - "null" + - object + properties: + back_shiny: + type: + - "null" + - string + back_female: + type: + - "null" + - string + front_shiny: + type: + - "null" + - string + back_default: + type: + - "null" + - string + front_female: + type: + - "null" + - string + front_default: + type: + - "null" + - string + back_shiny_female: + type: + - "null" + - string + front_shiny_female: + type: + - "null" + - string + additionalProperties: true + abilities: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + slot: + type: + - "null" + - integer + ability: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + is_hidden: + type: + - "null" + - boolean + additionalProperties: true + held_items: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + item: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + version_details: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + version: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + rarity: + type: + - "null" + - integer + additionalProperties: true + additionalProperties: true + is_default: + type: + - "null" + - boolean + past_types: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + types: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + type: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + slot: + type: + - "null" + - integer + additionalProperties: true + generation: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + additionalProperties: true + game_indices: + type: + - "null" + - array + items: + type: + - "null" + - object + properties: + version: + type: + - "null" + - object + properties: + url: + type: + - "null" + - string + name: + type: + - "null" + - string + additionalProperties: true + game_index: + type: + - "null" + - integer + additionalProperties: true + base_experience: + type: + - "null" + - integer + location_area_encounters: + type: + - "null" + - string + additionalProperties: true From 9115757b137524cc33df870e756ed26ed2f2fa61 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Wed, 18 Dec 2024 16:29:17 -0800 Subject: [PATCH 4/5] add fixture for custom py components scenario --- .../source_declarative_manifest/conftest.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/unit_tests/source_declarative_manifest/conftest.py b/unit_tests/source_declarative_manifest/conftest.py index 3d61e65e..a2598822 100644 --- a/unit_tests/source_declarative_manifest/conftest.py +++ b/unit_tests/source_declarative_manifest/conftest.py @@ -2,13 +2,26 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. # +import hashlib import os +from pathlib import Path +from typing import Any, Literal import pytest import yaml -def get_fixture_path(file_name): +def hash_text(input_text: str, hash_type: Literal["md5", "sha256"] = "md5") -> str: + hashers = { + "md5": hashlib.md5, + "sha256": hashlib.sha256, + } + hash_object = hashers[hash_type]() + hash_object.update(input_text.encode()) + return hash_object.hexdigest() + + +def get_fixture_path(file_name) -> str: return os.path.join(os.path.dirname(__file__), file_name) @@ -52,3 +65,21 @@ def valid_local_config_file(): @pytest.fixture def invalid_local_config_file(): return get_fixture_path("resources/invalid_local_pokeapi_config.json") + + +@pytest.fixture +def py_components_config_dict() -> dict[str, Any]: + manifest_dict = yaml.safe_load( + get_fixture_path("resources/valid_py_components.yaml"), + ) + custom_py_code_path = get_fixture_path("resources/valid_py_components_code.py") + custom_py_code = Path(custom_py_code_path).read_text() + combined_config_dict = { + "__injected_declarative_manifest": manifest_dict, + "__injected_components_py": custom_py_code, + "__injected_components_py_checksum": { + "md5": hash_text(custom_py_code, "md5"), + "sha256": hash_text(custom_py_code, "sha256"), + }, + } + return combined_config_dict From 5dc664c95b3bea0eab71646527f39e74352224df Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Wed, 18 Dec 2024 16:47:52 -0800 Subject: [PATCH 5/5] add test --- ..._source_declarative_w_custom_components.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py diff --git a/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py b/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py new file mode 100644 index 00000000..c3ea2059 --- /dev/null +++ b/unit_tests/source_declarative_manifest/test_source_declarative_w_custom_components.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +# + +import json +from tempfile import NamedTemporaryFile +from typing import Any + +from airbyte_cdk.cli.source_declarative_manifest._run import ( + create_declarative_source, +) +from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource + + +def test_given_injected_declarative_manifest_and_py_components_then_return_declarative_manifest( + py_components_config_dict: dict[str, Any], +): + with NamedTemporaryFile(delete=False, suffix=".json") as temp_config_file: + json.dump(py_components_config_dict, temp_config_file) + temp_config_file.flush() + source = create_declarative_source( + ["check", "--config", temp_config_file.name], + ) + assert isinstance(source, ManifestDeclarativeSource)