py

docs/source/faq.mdx

@@ -16,6 +16,14 @@ You can safely remove the dataset from the Hub after training is complete.
 If uploaded, the dataset will be stored in your Hugging Face account as a private repository and will only be accessible by you
 and the training process. It is not used once the training is complete.
 
+## My training space paused for no reason mid-training
+
+AutoTrain Training Spaces will pause itself after training is done (or failed). This is done to save resources and costs.
+If your training failed, you can still see the space logs and find out what went wrong. Note: you won't be able to retrive the logs if you restart the space.
+
+Another reason for the space to pause is if the space is space's sleep time kicking in. If you have a long running training job, you must set the sleep time to a much higher value.
+The space will anyways pause itself after the training is done thus saving you costs.
+
 ## I get error `Your installed package nvidia-ml-py is corrupted. Skip patch functions`
 
 This error can be safely ignored. It is a warning from the `nvitop` library and does not affect the functionality of AutoTrain.

src/autotrain/cli/run_dreambooth.py

@@ -5,7 +5,6 @@
 from autotrain import logger
 from autotrain.cli import BaseAutoTrainCommand
 from autotrain.cli.utils import common_args
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.dreambooth.params import DreamBoothTrainingParams
 from autotrain.trainers.dreambooth.utils import VALID_IMAGE_EXTENSIONS, XL_MODELS
@@ -388,7 +387,6 @@ def __init__(self, args):
     def run(self):
         logger.info("Running DreamBooth Training")
         params = DreamBoothTrainingParams(**vars(self.args))
-        params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-        project = AutoTrainProject(params=params, backend=self.args.backend)
+        project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
         job_id = project.create()
         logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_extractive_qa.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.extractive_question_answering.params import ExtractiveQuestionAnsweringParams
 
@@ -101,7 +100,6 @@ def run(self):
         logger.info("Running Extractive Question Answering")
         if self.args.train:
             params = ExtractiveQuestionAnsweringParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_image_classification.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.image_classification.params import ImageClassificationParams
 
@@ -109,7 +108,6 @@ def run(self):
         logger.info("Running Image Classification")
         if self.args.train:
             params = ImageClassificationParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_image_regression.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.image_regression.params import ImageRegressionParams
 
@@ -109,7 +108,6 @@ def run(self):
         logger.info("Running Image Regression")
         if self.args.train:
             params = ImageRegressionParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_llm.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.clm.params import LLMTrainingParams
 
@@ -137,7 +136,6 @@ def run(self):
         logger.info("Running LLM")
         if self.args.train:
             params = LLMTrainingParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_object_detection.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.object_detection.params import ObjectDetectionParams
 
@@ -109,7 +108,6 @@ def run(self):
         logger.info("Running Object Detection")
         if self.args.train:
             params = ObjectDetectionParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_sent_tranformers.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.sent_transformers.params import SentenceTransformersParams
 
@@ -109,7 +108,6 @@ def run(self):
         logger.info("Running Sentence Transformers...")
         if self.args.train:
             params = SentenceTransformersParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_seq2seq.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.seq2seq.params import Seq2SeqParams
 
@@ -93,7 +92,6 @@ def run(self):
         logger.info("Running Seq2Seq Classification")
         if self.args.train:
             params = Seq2SeqParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_tabular.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.tabular.params import TabularParams
 
@@ -102,7 +101,6 @@ def run(self):
         logger.info("Running Tabular Training")
         if self.args.train:
             params = TabularParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_text_classification.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.text_classification.params import TextClassificationParams
 
@@ -102,7 +101,6 @@ def run(self):
         logger.info("Running Text Classification")
         if self.args.train:
             params = TextClassificationParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_text_regression.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.text_regression.params import TextRegressionParams
 
@@ -102,7 +101,6 @@ def run(self):
         logger.info("Running Text Regression")
         if self.args.train:
             params = TextRegressionParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_token_classification.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.token_classification.params import TokenClassificationParams
 
@@ -102,7 +101,6 @@ def run(self):
         logger.info("Running Token Classification")
         if self.args.train:
             params = TokenClassificationParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/cli/run_vlm.py

@@ -2,7 +2,6 @@
 
 from autotrain import logger
 from autotrain.cli.utils import get_field_info
-from autotrain.process import AutoTrainDataProcessor
 from autotrain.project import AutoTrainProject
 from autotrain.trainers.vlm.params import VLMTrainingParams
 
@@ -107,7 +106,6 @@ def run(self):
         logger.info("Running Image Regression")
         if self.args.train:
             params = VLMTrainingParams(**vars(self.args))
-            params = AutoTrainDataProcessor(params, local=self.args.backend.startswith("local"))
-            project = AutoTrainProject(params=params, backend=self.args.backend)
+            project = AutoTrainProject(params=params, backend=self.args.backend, process=True)
             job_id = project.create()
             logger.info(f"Job ID: {job_id}")

src/autotrain/project.py

@@ -499,7 +499,7 @@ class AutoTrainProject:
         - spaces-a10g-largex2
         - spaces-a10g-largex4
     process : bool
-        Flag to indicate if the params and dataset should be processed. If your data format is not AutoTrain-readable, set it to True. Set it to True when in doubt.
+        Flag to indicate if the params and dataset should be processed. If your data format is not AutoTrain-readable, set it to True. Set it to True when in doubt. Defaults to False.
 
     Methods
     -------