From 4965c2d9354dc5533e853972924b2c1cc62e2130 Mon Sep 17 00:00:00 2001 From: Shreya Shankar Date: Tue, 20 Aug 2024 15:14:50 -0700 Subject: [PATCH] Add pre-commit hooks --- .pre-commit-config.yaml | 56 ++++++ motion/builder.py | 19 ++- motion/cli.py | 7 +- motion/operations/base.py | 3 +- motion/operations/equijoin.py | 17 +- motion/operations/filter.py | 9 +- motion/operations/map.py | 9 +- motion/operations/reduce.py | 25 +-- motion/operations/resolve.py | 22 ++- motion/operations/split.py | 12 +- motion/operations/unnest.py | 1 + motion/operations/utils.py | 15 +- motion/optimizers/join_optimizer.py | 16 +- .../map_optimizer/config_generators.py | 12 +- motion/optimizers/map_optimizer/evaluator.py | 2 +- .../map_optimizer/operation_creators.py | 2 +- motion/optimizers/map_optimizer/optimizer.py | 13 +- .../map_optimizer/plan_generators.py | 11 +- .../map_optimizer/prompt_generators.py | 16 +- motion/optimizers/map_optimizer/utils.py | 9 +- motion/optimizers/reduce_optimizer.py | 26 +-- motion/optimizers/utils.py | 7 +- motion/runner.py | 8 +- motion/utils.py | 3 +- poetry.lock | 161 +++++++++++++++++- pyproject.toml | 21 +++ 26 files changed, 389 insertions(+), 113 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..c8978635 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,56 @@ +ci: + autofix_prs: false + +files: "^(motion|ui)/" +exclude: '\__init__.py$' + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + exclude: ^.*\.egg-info/ + - id: check-merge-conflict + - id: check-case-conflict + - id: pretty-format-json + args: [--autofix, --no-ensure-ascii, --no-sort-keys] + - id: check-ast + - id: debug-statements + - id: check-docstring-first + + - repo: https://github.com/hadialqattan/pycln + rev: v2.4.0 + hooks: + - id: pycln + args: [--all] + + - repo: https://github.com/psf/black + rev: 24.1.1 + hooks: + - id: black + + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: "isort (python)" + types: [python] + args: [--profile, black] + + - repo: https://github.com/charliermarsh/ruff-pre-commit + # Ruff version. + rev: "v0.2.1" + hooks: + - id: ruff + + - repo: https://github.com/pre-commit/pre-commit + rev: v3.6.0 + hooks: + - id: validate_manifest + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v4.0.0-alpha.8" # Prettier version + hooks: + - id: prettier + files: "^ui/" diff --git a/motion/builder.py b/motion/builder.py index 2bebb9ee..189e09ef 100644 --- a/motion/builder.py +++ b/motion/builder.py @@ -1,19 +1,20 @@ -from collections import Counter, defaultdict import copy +import json +import os +import random +from collections import Counter, defaultdict +from typing import Any, Dict, List, Optional, Tuple, Union + import yaml -from typing import Dict, List, Any, Optional, Tuple, Union +from rich.console import Console + from motion.operations import get_operation from motion.operations.base import BaseOperation +from motion.optimizers.join_optimizer import JoinOptimizer from motion.optimizers.map_optimizer import MapOptimizer from motion.optimizers.reduce_optimizer import ReduceOptimizer -from motion.optimizers.join_optimizer import JoinOptimizer -from motion.utils import load_config -from rich.console import Console -import random -import json -import os from motion.optimizers.utils import LLMClient - +from motion.utils import load_config SUPPORTED_OPS = ["map", "resolve", "reduce", "equijoin", "filter"] diff --git a/motion/cli.py b/motion/cli.py index 08aaa51c..a026a940 100644 --- a/motion/cli.py +++ b/motion/cli.py @@ -1,9 +1,10 @@ -import typer -from typing import Optional from pathlib import Path +from typing import Optional + +import typer -from motion.runner import DSLRunner from motion.builder import Optimizer +from motion.runner import DSLRunner app = typer.Typer() diff --git a/motion/operations/base.py b/motion/operations/base.py index 30e6b722..025c745c 100644 --- a/motion/operations/base.py +++ b/motion/operations/base.py @@ -3,7 +3,8 @@ """ from abc import ABC, abstractmethod -from typing import Dict, List, Tuple, Optional +from typing import Dict, List, Optional, Tuple + from rich.console import Console diff --git a/motion/operations/equijoin.py b/motion/operations/equijoin.py index 34cc240d..508735ec 100644 --- a/motion/operations/equijoin.py +++ b/motion/operations/equijoin.py @@ -2,20 +2,23 @@ The `EquijoinOperation` class is a subclass of `BaseOperation` that performs an equijoin operation on two datasets. It uses a combination of blocking techniques and LLM-based comparisons to efficiently join the datasets. """ -from typing import Dict, List, Any, Tuple +import json +from collections import defaultdict from concurrent.futures import ThreadPoolExecutor +from typing import Any, Dict, List, Tuple + from jinja2 import Template -from collections import defaultdict -import json +from litellm import completion_cost +from sklearn.metrics.pairwise import cosine_similarity + from motion.operations.base import BaseOperation from motion.operations.utils import ( call_llm, - parse_llm_response, embedding, + parse_llm_response, + rich_as_completed, + validate_output, ) -from motion.operations.utils import validate_output, rich_as_completed -from litellm import completion_cost -from sklearn.metrics.pairwise import cosine_similarity def compare_pair( diff --git a/motion/operations/filter.py b/motion/operations/filter.py index ae3f5d38..e6feb277 100644 --- a/motion/operations/filter.py +++ b/motion/operations/filter.py @@ -1,15 +1,18 @@ """The `FilterOperation` class is a subclass of `BaseOperation` that implements a filtering operation on input data using a language model.""" -from typing import Dict, List, Any, Tuple, Optional from concurrent.futures import ThreadPoolExecutor +from typing import Any, Dict, List, Optional, Tuple + from jinja2 import Template + from motion.operations.base import BaseOperation from motion.operations.utils import ( + RichLoopBar, call_llm, - parse_llm_response, call_llm_with_validation, + parse_llm_response, + validate_output, ) -from motion.operations.utils import validate_output, RichLoopBar class FilterOperation(BaseOperation): diff --git a/motion/operations/map.py b/motion/operations/map.py index 291782bf..ec552989 100644 --- a/motion/operations/map.py +++ b/motion/operations/map.py @@ -2,17 +2,20 @@ The `MapOperation` and `ParallelMapOperation` classes are subclasses of `BaseOperation` that perform mapping operations on input data. They use LLM-based processing to transform input items into output items based on specified prompts and schemas. """ -from typing import Dict, List, Any, Tuple, Optional from concurrent.futures import ThreadPoolExecutor +from typing import Any, Dict, List, Optional, Tuple + from jinja2 import Template + from motion.operations.base import BaseOperation from motion.operations.utils import ( + RichLoopBar, call_llm, - parse_llm_response, call_llm_with_gleaning, call_llm_with_validation, + parse_llm_response, + validate_output, ) -from motion.operations.utils import validate_output, RichLoopBar class MapOperation(BaseOperation): diff --git a/motion/operations/reduce.py b/motion/operations/reduce.py index 74427222..bd0bf34f 100644 --- a/motion/operations/reduce.py +++ b/motion/operations/reduce.py @@ -9,21 +9,26 @@ import math import random import time -from typing import Dict, List, Tuple, Optional +from collections import deque from concurrent.futures import ThreadPoolExecutor, as_completed +from threading import Lock +from typing import Dict, List, Optional, Tuple +import jinja2 import numpy as np from jinja2 import Template -from motion.operations.base import BaseOperation -from motion.operations.utils import call_llm, call_llm_with_gleaning, parse_llm_response -from motion.operations.utils import validate_output, rich_as_completed -from litellm import completion_cost -import jinja2 -from threading import Lock -from collections import deque -from litellm import embedding -from sklearn.metrics.pairwise import cosine_similarity +from litellm import completion_cost, embedding from sklearn.cluster import KMeans +from sklearn.metrics.pairwise import cosine_similarity + +from motion.operations.base import BaseOperation +from motion.operations.utils import ( + call_llm, + call_llm_with_gleaning, + parse_llm_response, + rich_as_completed, + validate_output, +) class ReduceOperation(BaseOperation): diff --git a/motion/operations/resolve.py b/motion/operations/resolve.py index a54d2aca..70d6e7eb 100644 --- a/motion/operations/resolve.py +++ b/motion/operations/resolve.py @@ -2,15 +2,23 @@ The `ResolveOperation` class is a subclass of `BaseOperation` that performs a resolution operation on a dataset. It uses a combination of blocking techniques and LLM-based comparisons to efficiently identify and resolve duplicate or related entries within the dataset. """ -from typing import Dict, List, Any, Tuple from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Dict, List, Tuple + +import jinja2 from jinja2 import Template -from motion.operations.base import BaseOperation -from motion.operations.utils import call_llm, parse_llm_response, embedding -from motion.operations.utils import validate_output, rich_as_completed, RichLoopBar from litellm import completion_cost from sklearn.metrics.pairwise import cosine_similarity -import jinja2 + +from motion.operations.base import BaseOperation +from motion.operations.utils import ( + RichLoopBar, + call_llm, + embedding, + parse_llm_response, + rich_as_completed, + validate_output, +) def compare_pair( @@ -288,9 +296,9 @@ def should_compare(pair): for future in as_completed(future_to_pair): pair = future_to_pair[future] - is_match, cost = future.result() + is_match_result, cost = future.result() pair_costs += cost - if is_match: + if is_match_result: merge_clusters(pair[0], pair[1]) pbar.update(i) diff --git a/motion/operations/split.py b/motion/operations/split.py index 6fea8a7f..97f34ac1 100644 --- a/motion/operations/split.py +++ b/motion/operations/split.py @@ -1,12 +1,14 @@ +import math +import uuid from concurrent.futures import ThreadPoolExecutor from typing import Dict, List, Tuple -import uuid -from motion.operations.utils import call_llm, parse_llm_response + import tiktoken -from motion.operations.base import BaseOperation -import math -from litellm import completion_cost from jinja2 import Template +from litellm import completion_cost + +from motion.operations.base import BaseOperation +from motion.operations.utils import call_llm, parse_llm_response class SplitOperation(BaseOperation): diff --git a/motion/operations/unnest.py b/motion/operations/unnest.py index c3cee085..f8954c53 100644 --- a/motion/operations/unnest.py +++ b/motion/operations/unnest.py @@ -1,5 +1,6 @@ import copy from typing import Dict, List, Tuple + from motion.operations.base import BaseOperation diff --git a/motion/operations/utils.py b/motion/operations/utils.py index 728f6c95..9457bdbd 100644 --- a/motion/operations/utils.py +++ b/motion/operations/utils.py @@ -1,15 +1,16 @@ +import functools +import hashlib import json import threading -from typing import Callable, Dict, List, Any, Optional, Tuple, Iterable, Union -from litellm import completion, completion_cost +from concurrent.futures import as_completed +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union + from dotenv import load_dotenv +from frozendict import frozendict +from jinja2 import Template +from litellm import completion, completion_cost from rich.console import Console -import hashlib -import functools -from concurrent.futures import as_completed from tqdm import tqdm -from jinja2 import Template -from frozendict import frozendict load_dotenv() # litellm.set_verbose = True diff --git a/motion/optimizers/join_optimizer.py b/motion/optimizers/join_optimizer.py index c264fed1..34852f8a 100644 --- a/motion/optimizers/join_optimizer.py +++ b/motion/optimizers/join_optimizer.py @@ -1,13 +1,15 @@ -from concurrent.futures import ThreadPoolExecutor import json import random import re -from typing import List, Dict, Any, Optional, Tuple +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Dict, List, Optional, Tuple + import numpy as np -from litellm import embedding, completion_cost +from litellm import completion_cost, embedding from rich.console import Console -from motion.operations.resolve import compare_pair as compare_pair_resolve + from motion.operations.equijoin import compare_pair as compare_pair_equijoin +from motion.operations.resolve import compare_pair as compare_pair_resolve class JoinOptimizer: @@ -50,7 +52,7 @@ def _analyze_map_prompt_categorization(self, map_prompt: str) -> bool: }, { "role": "user", - "content": f"""Analyze the following map operation prompt and determine if it is explicitly categorical, + "content": f"""Analyze the following map operation prompt and determine if it is explicitly categorical, meaning it details a specific set of possible outputs: {map_prompt} @@ -282,7 +284,7 @@ def synthesize_resolution_prompt( reduce_key: List[str], output_schema: Dict[str, str], ) -> str: - system_prompt = f"""You are an AI assistant tasked with creating a resolution prompt for LLM-assisted entity resolution. + system_prompt = f"""You are an AI assistant tasked with creating a resolution prompt for LLM-assisted entity resolution. Your task is to create a prompt that will be used to merge multiple duplicate keys into a single, consolidated key. The key(s) being resolved (known as the reduce_key) are {', '.join(reduce_key)}. The duplicate keys will be provided in a list called 'matched_entries' in a Jinja2 template. @@ -312,7 +314,7 @@ def synthesize_resolution_prompt( {{% endfor %}} - Create a single, consolidated key that combines the information from all duplicate entries. + Create a single, consolidated key that combines the information from all duplicate entries. When merging, follow these guidelines: 1. [Provide specific merging instructions relevant to the data type] 2. [Provide conflict resolution guidelines] diff --git a/motion/optimizers/map_optimizer/config_generators.py b/motion/optimizers/map_optimizer/config_generators.py index 3635a0c2..bf71b836 100644 --- a/motion/optimizers/map_optimizer/config_generators.py +++ b/motion/optimizers/map_optimizer/config_generators.py @@ -1,11 +1,13 @@ import copy -import random import json -from typing import Dict, Any, List -from motion.optimizers.utils import extract_jinja_variables, LLMClient -from motion.optimizers.map_optimizer.utils import generate_and_validate_prompt +import random +from typing import Any, Dict, List + from rich.console import Console +from motion.optimizers.map_optimizer.utils import generate_and_validate_prompt +from motion.optimizers.utils import LLMClient, extract_jinja_variables + class ConfigGenerator: def __init__( @@ -78,7 +80,7 @@ def _get_split_config( Determine the split key and subprompt for processing chunks of the input data. The split key should be a key in the input data that contains a string to be split. - The subprompt should be designed to process individual chunks of the split data. + The subprompt should be designed to process individual chunks of the split data. Note that the subprompt's output schema will be: {json.dumps(output_schema, indent=2)}. Important: diff --git a/motion/optimizers/map_optimizer/evaluator.py b/motion/optimizers/map_optimizer/evaluator.py index ee9f6f55..7edd5b80 100644 --- a/motion/optimizers/map_optimizer/evaluator.py +++ b/motion/optimizers/map_optimizer/evaluator.py @@ -1,7 +1,7 @@ import json import time from concurrent.futures import ThreadPoolExecutor -from typing import Dict, Any, List, Tuple, Optional, Union, Callable +from typing import Any, Callable, Dict, List, Optional, Tuple, Union from rich.console import Console diff --git a/motion/optimizers/map_optimizer/operation_creators.py b/motion/optimizers/map_optimizer/operation_creators.py index 4a7260a9..a97ac511 100644 --- a/motion/optimizers/map_optimizer/operation_creators.py +++ b/motion/optimizers/map_optimizer/operation_creators.py @@ -1,4 +1,4 @@ -from typing import Dict, Any, List +from typing import Any, Dict, List class OperationCreator: diff --git a/motion/optimizers/map_optimizer/optimizer.py b/motion/optimizers/map_optimizer/optimizer.py index ef31a550..69dcadf5 100644 --- a/motion/optimizers/map_optimizer/optimizer.py +++ b/motion/optimizers/map_optimizer/optimizer.py @@ -1,18 +1,19 @@ +import concurrent +import copy import json import time -from typing import Any, Dict, List, Callable, Tuple import uuid -from motion.optimizers.utils import LLMClient +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Any, Callable, Dict, List, Tuple + from rich.console import Console -import copy from rich.table import Table -from concurrent.futures import ThreadPoolExecutor, as_completed -import concurrent -from motion.optimizers.map_optimizer.plan_generators import PlanGenerator from motion.optimizers.map_optimizer.evaluator import Evaluator +from motion.optimizers.map_optimizer.plan_generators import PlanGenerator from motion.optimizers.map_optimizer.prompt_generators import PromptGenerator from motion.optimizers.map_optimizer.utils import select_evaluation_samples +from motion.optimizers.utils import LLMClient class MapOptimizer: diff --git a/motion/optimizers/map_optimizer/plan_generators.py b/motion/optimizers/map_optimizer/plan_generators.py index 382ca7dd..515a5f16 100644 --- a/motion/optimizers/map_optimizer/plan_generators.py +++ b/motion/optimizers/map_optimizer/plan_generators.py @@ -1,17 +1,16 @@ import copy import json import random -from typing import Callable, Dict, Any, List from concurrent.futures import ThreadPoolExecutor, as_completed -from motion.optimizers.reduce_optimizer import ReduceOptimizer +from typing import Any, Callable, Dict, List from rich.console import Console -from motion.optimizers.utils import LLMClient -from motion.optimizers.map_optimizer.operation_creators import OperationCreator -from motion.optimizers.utils import extract_jinja_variables from motion.optimizers.map_optimizer.config_generators import ConfigGenerator +from motion.optimizers.map_optimizer.operation_creators import OperationCreator from motion.optimizers.map_optimizer.prompt_generators import PromptGenerator +from motion.optimizers.reduce_optimizer import ReduceOptimizer +from motion.optimizers.utils import LLMClient, extract_jinja_variables class PlanGenerator: @@ -481,7 +480,7 @@ def _assess_output_quality( 2. "Partially Satisfactory": The output met some of the validation prompt requirements but not all for the given chunk. 3. "Mostly Satisfactory": The output met most of the validation prompt requirements but has some room for improvement for the given chunk. 4. "Satisfactory": The output fully met the validation prompt requirements for the given chunk. - + Remember, only consider the main chunk content when evaluating the output, not any information surrounding the chunk. """ diff --git a/motion/optimizers/map_optimizer/prompt_generators.py b/motion/optimizers/map_optimizer/prompt_generators.py index c9a354e3..385ee5c9 100644 --- a/motion/optimizers/map_optimizer/prompt_generators.py +++ b/motion/optimizers/map_optimizer/prompt_generators.py @@ -1,11 +1,11 @@ -import random import json -from typing import Dict, Any, List, Tuple +import random +from typing import Any, Dict, List, Tuple from rich.console import Console -from motion.optimizers.utils import LLMClient from motion.optimizers.map_optimizer.utils import generate_and_validate_prompt +from motion.optimizers.utils import LLMClient class PromptGenerator: @@ -84,7 +84,7 @@ def _get_improved_prompt( Use the following feedback to improve the current prompt: {json.dumps(assessment['improvements'], indent=2)} - Improve the current prompt to better handle the input data and produce more accurate results. + Improve the current prompt to better handle the input data and produce more accurate results. Note: The new prompt should only include the variables present in the current prompt verbatim. Do not introduce any new variables. Provide your response in the following format: @@ -174,14 +174,14 @@ def _get_combine_prompt( base_prompt = f"""Original prompt (that operates on the full input, not the individual chunks): {op_config['prompt']} - + Output schema: {json.dumps(op_config['output']['schema'], indent=2)} Sample inputs from processing various chunks: {sample_inputs} - Modify the original prompt to be a prompt that will combine these chunk results to accomplish the original task. + Modify the original prompt to be a prompt that will combine these chunk results to accomplish the original task. Guidelines for your prompt template: - The only variable you are allowed to use is the values variable, which contains all chunk results. Each value is a dictionary with the keys {', '.join(schema_keys)} @@ -219,7 +219,7 @@ def _get_combine_prompt( Original task prompt: {op_config['prompt']} - + Output schema: {json.dumps(op_config['output']['schema'], indent=2)} @@ -229,7 +229,7 @@ def _get_combine_prompt( Prompt to combine results of subtasks: {combine_prompt} - Does the order of combining chunk results matter? Answer with 'yes' if order matters (non-commutative) or 'no' if order doesn't matter (commutative). + Does the order of combining chunk results matter? Answer with 'yes' if order matters (non-commutative) or 'no' if order doesn't matter (commutative). Explain your reasoning briefly. For example: diff --git a/motion/optimizers/map_optimizer/utils.py b/motion/optimizers/map_optimizer/utils.py index 29699775..e692235d 100644 --- a/motion/optimizers/map_optimizer/utils.py +++ b/motion/optimizers/map_optimizer/utils.py @@ -1,11 +1,12 @@ -import random import json -from typing import List, Dict, Any +import random +from typing import Any, Dict, List + +import jinja2 +from rich.console import Console from motion.operations import get_operation from motion.optimizers.utils import LLMClient -from rich.console import Console -import jinja2 def select_evaluation_samples( diff --git a/motion/optimizers/reduce_optimizer.py b/motion/optimizers/reduce_optimizer.py index d89f51f8..eacc41ff 100644 --- a/motion/optimizers/reduce_optimizer.py +++ b/motion/optimizers/reduce_optimizer.py @@ -1,13 +1,15 @@ import json import random -from typing import Any, Dict, List, Callable, Tuple, Union +from collections import Counter +from concurrent.futures import ThreadPoolExecutor, as_completed +from statistics import mean, median +from typing import Any, Callable, Dict, List, Tuple, Union + +from rich.console import Console + from motion.operations.base import BaseOperation from motion.optimizers.join_optimizer import JoinOptimizer -from rich.console import Console from motion.optimizers.utils import LLMClient -from collections import Counter -from statistics import mean, median -from concurrent.futures import ThreadPoolExecutor, as_completed class ReduceOptimizer: @@ -427,7 +429,7 @@ def _get_decomposition_details( 1. A sub-group key to use for the first reduce operation 2. A prompt for the first reduce operation 3. A prompt for the second (final) reduce operation - + For the reduce operation prompts, you should only minimally modify the original prompt. The prompts should be Jinja templates, and the only variables they can access are the `reduce_key` and `values` variables. Provide your suggestions in the following format: @@ -511,7 +513,7 @@ def _determine_value_sampling( Reduce Operation Prompt: {op_config['prompt']} - + Sample Input Data (first 2 items): {json.dumps(sample_input[:2], indent=2)} @@ -597,7 +599,7 @@ def _determine_value_sampling( Reduce Operation Prompt: {op_config['prompt']} - The query text should be a Jinja template with access to the `reduce_key` variable. + The query text should be a Jinja template with access to the `reduce_key` variable. Based on the reduce operation prompt, what would be an appropriate query text for selecting relevant samples? """ @@ -658,7 +660,7 @@ def _is_commutative( For example, sum and product operations are commutative, while subtraction and division are not. Based on the reduce operation prompt and the sample input data, determine if this operation is likely to be commutative. - Answer with 'yes' if order matters (non-commutative) or 'no' if order doesn't matter (commutative). + Answer with 'yes' if order matters (non-commutative) or 'no' if order doesn't matter (commutative). Explain your reasoning briefly. For example: @@ -1221,7 +1223,7 @@ def generate_single_prompt(): prompt = f""" Original Reduce Operation Prompt: {original_prompt} - + Sample Input: {json.dumps(input_example, indent=2)} @@ -1483,10 +1485,10 @@ def _synthesize_merge_prompt( prompt = f"""Reduce Operation Prompt (runs on the first batch of inputs): {plan["prompt"]} - + Fold Prompt (runs on the second and subsequent batches of inputs): {plan["fold_prompt"]} - + Sample output of the fold operation (an input to the merge operation): {json.dumps(random_output, indent=2)} diff --git a/motion/optimizers/utils.py b/motion/optimizers/utils.py index 7aceeb0e..f4ee2dea 100644 --- a/motion/optimizers/utils.py +++ b/motion/optimizers/utils.py @@ -1,7 +1,8 @@ -from typing import Dict, List, Any -from litellm import completion, completion_cost -from jinja2 import Environment, meta import re +from typing import Any, Dict, List + +from jinja2 import Environment, meta +from litellm import completion, completion_cost def extract_jinja_variables(template_string: str) -> List[str]: diff --git a/motion/runner.py b/motion/runner.py index 94a28c09..6ab170ad 100644 --- a/motion/runner.py +++ b/motion/runner.py @@ -1,12 +1,14 @@ +import json +import os from typing import Dict, List, Optional, Tuple + from dotenv import load_dotenv -import os +from rich import print as rprint from rich.console import Console from rich.progress import Progress, SpinnerColumn, TextColumn -from rich import print as rprint + from motion.operations import get_operation from motion.utils import load_config -import json load_dotenv() diff --git a/motion/utils.py b/motion/utils.py index a0c68045..c7fc40d4 100644 --- a/motion/utils.py +++ b/motion/utils.py @@ -1,5 +1,6 @@ +from typing import Any, Dict + import yaml -from typing import Dict, Any def load_config(config_path: str) -> Dict[str, Any]: diff --git a/poetry.lock b/poetry.lock index 909b27d7..12d11f91 100644 --- a/poetry.lock +++ b/poetry.lock @@ -184,6 +184,17 @@ files = [ {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, ] +[[package]] +name = "cfgv" +version = "3.4.0" +description = "Validate configuration and produce human readable error messages." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, + {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, +] + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -308,6 +319,17 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "distlib" +version = "0.3.8" +description = "Distribution utilities" +optional = false +python-versions = "*" +files = [ + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, +] + [[package]] name = "distro" version = "1.9.0" @@ -606,6 +628,20 @@ testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gr torch = ["safetensors[torch]", "torch"] typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] +[[package]] +name = "identify" +version = "2.6.0" +description = "File identification library for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "identify-2.6.0-py2.py3-none-any.whl", hash = "sha256:e79ae4406387a9d300332b5fd366d8994f1525e8414984e1a59e058b2eda2dd0"}, + {file = "identify-2.6.0.tar.gz", hash = "sha256:cb171c685bdc31bcc4c1734698736a7d5b6c8bf2e0c15117f4d469c8640ae5cf"}, +] + +[package.extras] +license = ["ukkonen"] + [[package]] name = "idna" version = "3.7" @@ -930,6 +966,75 @@ files = [ {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, ] +[[package]] +name = "mypy" +version = "1.11.1" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a32fc80b63de4b5b3e65f4be82b4cfa362a46702672aa6a0f443b4689af7008c"}, + {file = "mypy-1.11.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1952f5ea8a5a959b05ed5f16452fddadbaae48b5d39235ab4c3fc444d5fd411"}, + {file = "mypy-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1e30dc3bfa4e157e53c1d17a0dad20f89dc433393e7702b813c10e200843b03"}, + {file = "mypy-1.11.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2c63350af88f43a66d3dfeeeb8d77af34a4f07d760b9eb3a8697f0386c7590b4"}, + {file = "mypy-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:a831671bad47186603872a3abc19634f3011d7f83b083762c942442d51c58d58"}, + {file = "mypy-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7b6343d338390bb946d449677726edf60102a1c96079b4f002dedff375953fc5"}, + {file = "mypy-1.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e4fe9f4e5e521b458d8feb52547f4bade7ef8c93238dfb5bbc790d9ff2d770ca"}, + {file = "mypy-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:886c9dbecc87b9516eff294541bf7f3655722bf22bb898ee06985cd7269898de"}, + {file = "mypy-1.11.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca4a60e1dd9fd0193ae0067eaeeb962f2d79e0d9f0f66223a0682f26ffcc809"}, + {file = "mypy-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:0bd53faf56de9643336aeea1c925012837432b5faf1701ccca7fde70166ccf72"}, + {file = "mypy-1.11.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f39918a50f74dc5969807dcfaecafa804fa7f90c9d60506835036cc1bc891dc8"}, + {file = "mypy-1.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bc71d1fb27a428139dd78621953effe0d208aed9857cb08d002280b0422003a"}, + {file = "mypy-1.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b868d3bcff720dd7217c383474008ddabaf048fad8d78ed948bb4b624870a417"}, + {file = "mypy-1.11.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a707ec1527ffcdd1c784d0924bf5cb15cd7f22683b919668a04d2b9c34549d2e"}, + {file = "mypy-1.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:64f4a90e3ea07f590c5bcf9029035cf0efeae5ba8be511a8caada1a4893f5525"}, + {file = "mypy-1.11.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:749fd3213916f1751fff995fccf20c6195cae941dc968f3aaadf9bb4e430e5a2"}, + {file = "mypy-1.11.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b639dce63a0b19085213ec5fdd8cffd1d81988f47a2dec7100e93564f3e8fb3b"}, + {file = "mypy-1.11.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c956b49c5d865394d62941b109728c5c596a415e9c5b2be663dd26a1ff07bc0"}, + {file = "mypy-1.11.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45df906e8b6804ef4b666af29a87ad9f5921aad091c79cc38e12198e220beabd"}, + {file = "mypy-1.11.1-cp38-cp38-win_amd64.whl", hash = "sha256:d44be7551689d9d47b7abc27c71257adfdb53f03880841a5db15ddb22dc63edb"}, + {file = "mypy-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2684d3f693073ab89d76da8e3921883019ea8a3ec20fa5d8ecca6a2db4c54bbe"}, + {file = "mypy-1.11.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79c07eb282cb457473add5052b63925e5cc97dfab9812ee65a7c7ab5e3cb551c"}, + {file = "mypy-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11965c2f571ded6239977b14deebd3f4c3abd9a92398712d6da3a772974fad69"}, + {file = "mypy-1.11.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a2b43895a0f8154df6519706d9bca8280cda52d3d9d1514b2d9c3e26792a0b74"}, + {file = "mypy-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:1a81cf05975fd61aec5ae16501a091cfb9f605dc3e3c878c0da32f250b74760b"}, + {file = "mypy-1.11.1-py3-none-any.whl", hash = "sha256:0624bdb940255d2dd24e829d99a13cfeb72e4e9031f9492148f410ed30bcab54"}, + {file = "mypy-1.11.1.tar.gz", hash = "sha256:f404a0b069709f18bbdb702eb3dcfe51910602995de00bd39cea3050b5772d08"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.6.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "nodeenv" +version = "1.9.1" +description = "Node.js virtual environment builder" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, + {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, +] + [[package]] name = "openai" version = "1.37.0" @@ -964,6 +1069,22 @@ files = [ {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, ] +[[package]] +name = "platformdirs" +version = "4.2.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] + [[package]] name = "pluggy" version = "1.5.0" @@ -979,6 +1100,24 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "pre-commit" +version = "3.8.0" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +optional = false +python-versions = ">=3.9" +files = [ + {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, + {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, +] + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + [[package]] name = "pydantic" version = "2.8.2" @@ -1747,6 +1886,26 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "virtualenv" +version = "20.26.3" +description = "Virtual Python Environment builder" +optional = false +python-versions = ">=3.7" +files = [ + {file = "virtualenv-20.26.3-py3-none-any.whl", hash = "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589"}, + {file = "virtualenv-20.26.3.tar.gz", hash = "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a"}, +] + +[package.dependencies] +distlib = ">=0.3.7,<1" +filelock = ">=3.12.2,<4" +platformdirs = ">=3.9.1,<5" + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] + [[package]] name = "yarl" version = "1.9.4" @@ -1868,4 +2027,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "0020e11294aa9784af40e7b5fa672ad74b6bc290a7f21941dc0c6fd189610154" +content-hash = "994602c281a3cd70a9226d32f6ac838f0a09c40d0ba01c12673803fa156a24df" diff --git a/pyproject.toml b/pyproject.toml index 8bafb5b0..125f11b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,27 @@ frozendict = "^2.4.4" pytest = "^8.3.2" python-dotenv = "^1.0.1" ruff = "^0.6.1" +mypy = "^1.11.1" +pre-commit = "^3.8.0" + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "--basetemp=/tmp/pytest" +filterwarnings = [ + "ignore::DeprecationWarning", + "ignore::UserWarning", + "ignore::RuntimeWarning" +] + +[tool.mypy] +files = "motion" +mypy_path = "motion" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true +exclude = ['motion/tests*'] +ignore_missing_imports = true +show_error_codes = true [build-system] requires = ["poetry-core"]