update setup and requirements

Dockerfile

@@ -1,4 +1,4 @@
-FROM nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04
+FROM nvidia/cuda:12.1.0-cudnn8-devel-ubuntu22.04
 
 ENV DEBIAN_FRONTEND=noninteractive \
     TZ=UTC
@@ -54,7 +54,7 @@ RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \
     && rm -f Miniconda3-latest-Linux-x86_64.sh
 ENV PATH /app/miniconda/bin:$PATH
 
-RUN conda create -p /app/env -y python=3.9
+RUN conda create -p /app/env -y python=3.10
 
 SHELL ["conda", "run","--no-capture-output", "-p","/app/env", "/bin/bash", "-c"]
 

requirements.txt

@@ -26,13 +26,13 @@ invisible-watermark==0.2.0
 packaging==23.1
 # latest versions
 tensorboard
-peft
-trl
-tiktoken
-transformers
-accelerate
+peft==0.5.0
+trl==0.7.2
+tiktoken==0.5.1
+transformers==4.34.1
+accelerate==0.24.0
 diffusers==0.21.4
-bitsandbytes
+bitsandbytes==0.41.0
 # extras
 rouge_score==0.1.2
 py7zr==0.20.6

src/autotrain/backend.py

@@ -49,7 +49,7 @@ def _tabular_munge_data(params, username):
     if params.valid_split is not None:
         valid_data_path = f"{params.data_path}/{params.valid_split}.csv"
     else:
-        valid_data_path = []
+        valid_data_path = None
     if os.path.exists(train_data_path):
         dset = AutoTrainDataset(
             train_data=[train_data_path],
@@ -58,7 +58,7 @@ def _tabular_munge_data(params, username):
             project_name=params.project_name,
             username=username,
             column_mapping={"id": params.col_map_id, "label": col_map_label},
-            valid_data=valid_data_path,
+            valid_data=[valid_data_path] if valid_data_path is not None else None,
             percent_valid=None,  # TODO: add to UI
         )
         dset.prepare()
@@ -72,7 +72,7 @@ def _llm_munge_data(params, username):
     if params.valid_split is not None:
         valid_data_path = f"{params.data_path}/{params.valid_split}.csv"
     else:
-        valid_data_path = []
+        valid_data_path = None
     if os.path.exists(train_data_path):
         dset = AutoTrainDataset(
             train_data=[train_data_path],
@@ -81,7 +81,7 @@ def _llm_munge_data(params, username):
             project_name=params.project_name,
             username=username,
             column_mapping={"text": params.text_column},
-            valid_data=valid_data_path,
+            valid_data=[valid_data_path] if valid_data_path is not None else None,
             percent_valid=None,  # TODO: add to UI
         )
         dset.prepare()
@@ -95,7 +95,7 @@ def _seq2seq_munge_data(params, username):
     if params.valid_split is not None:
         valid_data_path = f"{params.data_path}/{params.valid_split}.csv"
     else:
-        valid_data_path = []
+        valid_data_path = None
     if os.path.exists(train_data_path):
         dset = AutoTrainDataset(
             train_data=[train_data_path],
@@ -104,7 +104,7 @@ def _seq2seq_munge_data(params, username):
             project_name=params.project_name,
             username=username,
             column_mapping={"text": params.text_column, "label": params.target_column},
-            valid_data=valid_data_path,
+            valid_data=[valid_data_path] if valid_data_path is not None else None,
             percent_valid=None,  # TODO: add to UI
         )
         dset.prepare()

src/autotrain/cli/run_setup.py

@@ -34,31 +34,6 @@ def __init__(self, update_torch: bool, colab: bool = False):
         self.colab = colab
 
     def run(self):
-        # install latest transformers
-        cmd = "pip uninstall -y transformers && pip install git+https://github.com/huggingface/transformers.git"
-        pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        logger.info("Installing latest transformers@main")
-        _, _ = pipe.communicate()
-        logger.info("Successfully installed latest transformers")
-
-        cmd = "pip uninstall -y peft && pip install git+https://github.com/huggingface/peft.git"
-        pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        logger.info("Installing latest peft@main")
-        _, _ = pipe.communicate()
-        logger.info("Successfully installed latest peft")
-
-        cmd = "pip uninstall -y diffusers && pip install diffusers==0.21.4"
-        pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        logger.info("Installing latest diffusers@main")
-        _, _ = pipe.communicate()
-        logger.info("Successfully installed latest diffusers")
-
-        cmd = "pip uninstall -y trl && pip install git+https://github.com/huggingface/trl.git"
-        pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        logger.info("Installing latest trl@main")
-        _, _ = pipe.communicate()
-        logger.info("Successfully installed latest trl")
-
         if self.colab:
             cmd = "pip install -U xformers==0.0.22"
         else:

src/autotrain/trainers/seq2seq/__main__.py

@@ -1,6 +1,7 @@
 import argparse
 import json
 import os
+import sys
 from functools import partial
 
 import pandas as pd
@@ -98,7 +99,7 @@ def train(config):
             config=model_config,
             token=config.token,
             quantization_config=bnb_config,
-            torch_dtype=torch.float16,
+            torch_dtype=torch.float16 if config.fp16 else None,
             device_map={"": Accelerator().process_index} if torch.cuda.is_available() else None,
             trust_remote_code=True,
         )
@@ -194,6 +195,14 @@ def train(config):
         tokenizer=tokenizer,
     )
 
+    model.config.use_cache = False
+
+    if torch.__version__ >= "2" and sys.platform != "win32":
+        model = torch.compile(model)
+
+    for name, module in trainer.model.named_modules():
+        if "norm" in name:
+            module = module.to(torch.float32)
     trainer.train()
 
     logger.info("Finished training, saving model...")