pyproject.toml

[tool.poetry]
name = "llm-finetuning-hub"
version = "0.1.0"
description = "LLM Finetuning resource hub + toolkit"
authors = ["Benjamin Ye <benjamin.ye@georgian.io>"]
license = "Apache 2.0"
readme = "README.md"
packages = [{include = "llm_finetuning_hub"}]

[tool.poetry.dependencies]
python = ">=3.8, <=3.12"
transformers = "^4.37.2"
datasets = "^2.17.0"
peft = "^0.8.2"
pandas = "^2.2.0"
numpy = "^1.26.4"
ipdb = "^0.13.13"
evaluate = "^0.4.1"
scikit-learn = "^1.4.0"
scipy = "^1.12.0"
wandb = "^0.16.3"
einops = "^0.7.0"
bitsandbytes = "^0.42.0"
nltk = "^3.8.1"
accelerate = "^0.27.0"
trl = "^0.7.10"
rouge-score = "^0.1.2"
absl-py = "^2.1.0"
py7zr = "^0.20.8"
tiktoken = "^0.6.0"
ninja = "^1.11.1.1"
packaging = "^23.2"
sentencepiece = "^0.1.99"
protobuf = "^4.25.2"
ai21 = "^2.0.3"
openai = "^1.12.0"
ujson = "^5.9.0"
pyyaml = "^6.0.1"
ijson = "^3.2.3"
rich = "^13.7.0"
sqids = "^0.4.1"
pydantic = "^2.6.1"


[tool.poetry.group.dev.dependencies]
black = "^24.1.1"

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"