Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[chore] resoto -> fix #16

Merged
merged 2 commits into from
Feb 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions .github/workflows/build_and_publish.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Check PR resotodatalink
name: Check PR fixdatalink
on:
push:
tags:
Expand All @@ -11,8 +11,8 @@ on:
workflow_dispatch:

jobs:
resotodatalink:
name: 'resotodatalink'
fixdatalink:
name: 'fixdatalink'
runs-on: ubuntu-latest
steps:
- name: Checkout
Expand Down Expand Up @@ -50,5 +50,5 @@ jobs:
uses: pypa/gh-action-pypi-publish@release/v1
with:
user: __token__
password: ${{ secrets.PYPI_RESOTODATALINK }}
password: ${{ secrets.PYPI_FIXDATALINK }}
packages_dir: ./dist/
2 changes: 1 addition & 1 deletion LICENSE
Original file line number Diff line number Diff line change
Expand Up @@ -629,7 +629,7 @@ to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.

resotodatalink.
fixdatalink.
Copyright (C) 2023 Some Engineering Inc.

This program is free software: you can redistribute it and/or modify
Expand Down
12 changes: 6 additions & 6 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ clean-env: ## remove environment
rm -fr venv-pypy

lint: ## static code analysis
black --line-length 120 --check resotodatalink tests
flake8 resotodatalink
mypy --python-version 3.9 --strict resotodatalink tests
black --line-length 120 --check fixdatalink tests
flake8 fixdatalink
mypy --python-version 3.9 --strict fixdatalink tests

test: ## run tests quickly with the default Python
pytest
Expand All @@ -67,15 +67,15 @@ test-all: ## run tests on every Python version with tox
tox

coverage: ## check code coverage quickly with the default Python
coverage run --source resotolib -m pytest
coverage run --source fixlib -m pytest
coverage combine
coverage report -m
coverage html
$(BROWSER) htmlcov/index.html

setup:
rm -fr venv
python3 -m venv venv --prompt "resotodatalink venv"
python3 -m venv venv --prompt "fixdatalink venv"
./venv/bin/python3 -m pip install --upgrade pip tox
./venv/bin/pip3 install -r requirements-all.txt
./venv/bin/pip3 install -e "."
Expand All @@ -84,7 +84,7 @@ setup:

update:
rm -fr venv
python3 -m venv venv --prompt "resotodatalink venv"
python3 -m venv venv --prompt "fixdatalink venv"
./venv/bin/python3 -m pip install --upgrade pip tox
./venv/bin/pip3 install -e ".[dev,test,snowflake,mysql,parquet,postgres]"
pip-compile -q --no-annotate --resolver=backtracking --upgrade --allow-unsafe --no-header -o requirements.txt --extra=extra
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# resotodatalink
Data Pipelines for Resoto infrastructure data
# fixdatalink
Data Pipelines for Fix infrastructure data


File renamed without changes.
2 changes: 1 addition & 1 deletion resotodatalink/analytics.py → fixdatalink/analytics.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from requests import get
from posthog.client import Client

log = logging.getLogger("resoto.datalink")
log = logging.getLogger("fix.datalink")


class AnalyticsEventSender(ABC):
Expand Down
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
from resotoclient.models import Kind, Model
from fixclient.models import Kind, Model
from typing import Dict, List, Tuple, Literal
import pyarrow as pa
from resotodatalink.schema_utils import (
from fixdatalink.schema_utils import (
base_kinds,
get_table_name,
get_link_table_name,
kind_properties,
)
from resotodatalink.arrow.type_converter import parquet_pyarrow_type, csv_pyarrow_type
from fixdatalink.arrow.type_converter import parquet_pyarrow_type, csv_pyarrow_type
from functools import partial


Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import pyarrow as pa
from resotoclient.models import Model
from resotodatalink.schema_utils import kind_properties
from fixclient.models import Model
from fixdatalink.schema_utils import kind_properties


def parquet_pyarrow_type(kind: str, model: Model) -> pa.lib.DataType:
Expand Down
12 changes: 6 additions & 6 deletions resotodatalink/arrow/writer.py → fixdatalink/arrow/writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,19 @@
import pyarrow.csv as csv
import pyarrow.parquet as pq
from google.cloud import storage
from resotolib.json import value_in_path
from resotolib.types import Json
from fixlib.json import value_in_path
from fixlib.types import Json

from resotodatalink.arrow.config import (
from fixdatalink.arrow.config import (
ArrowOutputConfig,
FileDestination,
CloudBucketDestination,
S3Bucket,
GCSBucket,
ArrowDestination,
)
from resotodatalink.arrow.model import ArrowModel
from resotodatalink.schema_utils import get_table_name, get_link_table_name, carz_access
from fixdatalink.arrow.model import ArrowModel
from fixdatalink.schema_utils import get_table_name, get_link_table_name, carz_access


class WriteResult(NamedTuple):
Expand Down Expand Up @@ -233,7 +233,7 @@ def sha(input: str) -> str:
result_dir = ensure_path(output_config.destination.path)
else:
hashed_url = sha(output_config.destination.bucket_name)
result_dir = ensure_path(Path(f"/tmp/resotodatalink-uploads/{hashed_url}"))
result_dir = ensure_path(Path(f"/tmp/fixdatalink-uploads/{hashed_url}"))

file_writer_format: Union[Parquet, CSV]
file_path: Path
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
from itertools import chain
from typing import AsyncIterator, Any, Callable, Dict, List, Iterator, Union

from resotolib.baseplugin import BaseCollectorPlugin
from resotolib.baseresources import EdgeType
from fixlib.baseplugin import BaseCollectorPlugin
from fixlib.baseresources import EdgeType

from resotodatalink.schema_utils import prepare_node, prepare_edge
from fixdatalink.schema_utils import prepare_node, prepare_edge


async def sync_to_async_iterator(it: Iterator[Any]) -> AsyncIterator[Any]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,26 +4,26 @@
from typing import List, Tuple, Set, Optional, AsyncIterator, Union, cast, Dict, TypeVar, Awaitable, Any

import jsons
from resotoclient import Kind, Model
from resotolib.baseplugin import BaseCollectorPlugin
from resotolib.baseresources import BaseResource, EdgeType
from resotolib.core.actions import CoreFeedback
from resotolib.types import Json
from fixclient import Kind, Model
from fixlib.baseplugin import BaseCollectorPlugin
from fixlib.baseresources import BaseResource, EdgeType
from fixlib.core.actions import CoreFeedback
from fixlib.types import Json
from sqlalchemy import create_engine, text

from resotodatalink import EngineConfig
from resotodatalink.arrow.config import ArrowOutputConfig
from resotodatalink.batch_stream import BatchStream
from resotodatalink.sql import sql_updater
from fixdatalink import EngineConfig
from fixdatalink.arrow.config import ArrowOutputConfig
from fixdatalink.batch_stream import BatchStream
from fixdatalink.sql import sql_updater

try:
from resotodatalink.arrow.model import ArrowModel
from resotodatalink.arrow.writer import ArrowWriter
from fixdatalink.arrow.model import ArrowModel
from fixdatalink.arrow.writer import ArrowWriter
except ImportError:
ArrowModel = type(None) # type: ignore
ArrowWriter = type(None) # type: ignore

log = getLogger("resoto.datalink")
log = getLogger("fix.datalink")
T = TypeVar("T")


Expand Down
File renamed without changes.
28 changes: 14 additions & 14 deletions resotodatalink/remote_graph.py → fixdatalink/remote_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,31 @@
from typing import Optional, ClassVar

from attr import define, field
from resotoclient import ResotoClient, JsObject
from resotolib.baseplugin import BaseCollectorPlugin
from resotolib.baseresources import (
from fixclient import FixClient, JsObject
from fixlib.baseplugin import BaseCollectorPlugin
from fixlib.baseresources import (
BaseResource,
Cloud,
EdgeType,
UnknownZone,
UnknownRegion,
UnknownAccount,
)
from resotolib.config import Config
from resotolib.core.actions import CoreFeedback
from resotolib.core.model_export import node_from_dict
from resotolib.graph import Graph
from resotolib.json import value_in_path
from resotolib.logger import log
from resotolib.types import Json
from fixlib.config import Config
from fixlib.core.actions import CoreFeedback
from fixlib.core.model_export import node_from_dict
from fixlib.graph import Graph
from fixlib.json import value_in_path
from fixlib.logger import log
from fixlib.types import Json


@define
class RemoteGraphConfig:
kind: ClassVar[str] = "remote_graph"
resoto_url: str = field(default="https://localhost:8900", metadata={"description": "URL of the resoto server"})
psk: Optional[str] = field(default=None, metadata={"description": "Pre-shared key for the resoto server"})
graph: str = field(default="resoto", metadata={"description": "Name of the graph to use"})
fix_url: str = field(default="https://localhost:8900", metadata={"description": "URL of the fix server"})
psk: Optional[str] = field(default=None, metadata={"description": "Pre-shared key for the fix server"})
graph: str = field(default="fix", metadata={"description": "Name of the graph to use"})
search: Optional[str] = field(
default=None, metadata={"description": "Search string to filter resources. None to get all resources."}
)
Expand Down Expand Up @@ -58,7 +58,7 @@ def collect(self) -> None:

def _collect_remote_graph(self) -> Graph:
config: RemoteGraphConfig = Config.remote_graph
client = ResotoClient(config.resoto_url, psk=config.psk)
client = FixClient(config.fix_url, psk=config.psk)
search = config.search or "is(graph_root) -[2:]->"
return self._collect_from_graph_iterator(client.search_graph(search, graph=config.graph))

Expand Down
12 changes: 6 additions & 6 deletions resotodatalink/schema_utils.py → fixdatalink/schema_utils.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import inspect
from typing import List, Dict, Tuple

from resotoclient.models import Property, Kind, Model
from resotolib import baseresources
from resotolib.baseplugin import BaseCollectorPlugin
from resotolib.baseresources import BaseResource, EdgeType
from resotolib.core.model_export import node_to_dict
from resotolib.types import Json
from fixclient.models import Property, Kind, Model
from fixlib import baseresources
from fixlib.baseplugin import BaseCollectorPlugin
from fixlib.baseresources import BaseResource, EdgeType
from fixlib.core.model_export import node_to_dict
from fixlib.types import Json

# This set will hold the names of all "base" resources
# Since that are abstract classes, there will be no instances of them - hence we do not need a table for them.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import shutil
from typing import Optional, Dict, Any, List

from resotolib.core.progress import ProgressTree, Progress
from resotolib.types import Json
from fixlib.core.progress import ProgressTree, Progress
from fixlib.types import Json
from rich.console import Group
from rich.markdown import Markdown
from rich.text import Text
Expand Down
10 changes: 5 additions & 5 deletions resotodatalink/snowflake.py → fixdatalink/snowflake.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,18 @@
import logging
from typing import Any, List, Iterator

from resotoclient import Model
from resotolib.types import Json
from fixclient import Model
from fixlib.types import Json
from snowflake.sqlalchemy import ARRAY, OBJECT
from sqlalchemy import Integer, Float, String, Boolean, column
from sqlalchemy import select
from sqlalchemy.sql import Values
from sqlalchemy.sql.dml import ValuesBase

from resotodatalink.schema_utils import kind_properties, get_table_name
from resotodatalink.sql import SqlDefaultUpdater, DialectUpdater
from fixdatalink.schema_utils import kind_properties, get_table_name
from fixdatalink.sql import SqlDefaultUpdater, DialectUpdater

log = logging.getLogger("resoto.datalink")
log = logging.getLogger("fix.datalink")


def kind_to_snowflake_type(kind_name: str, model: Model, **kwargs: Any) -> Any: # Type[TypeEngine[Any]]
Expand Down
14 changes: 7 additions & 7 deletions resotodatalink/sql.py → fixdatalink/sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
from datetime import datetime, date
from typing import List, Any, Type, Tuple, Dict, Iterator, Optional, Union

from resotoclient.models import Kind, Model, Property
from resotolib.types import Json
from resotolib.utils import UTC_Date_Format
from fixclient.models import Kind, Model, Property
from fixlib.types import Json
from fixlib.utils import UTC_Date_Format
from sqlalchemy import (
Boolean,
Column,
Expand All @@ -25,19 +25,19 @@
from sqlalchemy.sql.ddl import DropTable, DropConstraint
from sqlalchemy.sql.dml import ValuesBase

from resotodatalink import EngineConfig
from resotodatalink.schema_utils import (
from fixdatalink import EngineConfig
from fixdatalink.schema_utils import (
base_kinds,
temp_prefix,
carz_access,
get_table_name,
get_link_table_name,
kind_properties,
)
from resotolib.json import value_in_path
from fixlib.json import value_in_path
from sqlalchemy import create_engine

log = logging.getLogger("resoto.datalink")
log = logging.getLogger("fix.datalink")


class DateTimeString(TypeDecorator): # type: ignore
Expand Down
18 changes: 9 additions & 9 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
[project]
name = "resotodatalink"
version = "1.2.1"
name = "fixdatalink"
version = "2.0.0"
authors = [{name="Some Engineering Inc."}]
description = "Data Pipelines for Resoto infrastructure data."
description = "Data Pipelines for Fix infrastructure data."
license = {file="LICENSE"}
requires-python = ">=3.9"
classifiers = [
Expand All @@ -29,8 +29,8 @@ keywords = ["data source", "data sink", "data synchronization", "data pipeline"]

dependencies = [
"networkx",
"resotoclient>=1.5.0",
"resotolib",
"fixclient>=2.0.0",
"fixinventorylib",
"rich",
"sqlalchemy",
]
Expand Down Expand Up @@ -78,16 +78,16 @@ test = [
"pytest-runner",
"tox",
"wheel",
"resoto-plugin-example-collector"
"fix-plugin-example-collector"
]

[project.urls]
Documentation = "https://resoto.com"
Source = "https://github.com/someengineering/resotodatalink"
Documentation = "https://fix.com"
Source = "https://github.com/someengineering/fixdatalink"

[build-system]
requires = ["setuptools>=67.8.0", "wheel>=0.40.0", "build>=0.10.0"]
build-backend = "setuptools.build_meta"

[tool.setuptools.package-data]
resotodatalink = ["py.typed"]
fixdatalink = ["py.typed"]
Loading
Loading