From 495f4b37ea020e3e5c616a673e529333be2e82d3 Mon Sep 17 00:00:00 2001 From: kerrlabajo Date: Thu, 16 May 2024 12:17:40 +0800 Subject: [PATCH 1/5] Update to accept user-based sub-directories after `DATASET_NAME` or `..` --- docker/yolov5-training/configure_dataset.py | 22 ++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/docker/yolov5-training/configure_dataset.py b/docker/yolov5-training/configure_dataset.py index 5ee5e65..e674fd9 100644 --- a/docker/yolov5-training/configure_dataset.py +++ b/docker/yolov5-training/configure_dataset.py @@ -15,15 +15,27 @@ # Define the file path FILE_PATH = args.dataset_config_path -# Define the new paths -NEW_PATH = f"/opt/ml/input/data/{DATASET_NAME}" -NEW_TRAIN = f"{NEW_PATH}/images/train" -NEW_VAL = f"{NEW_PATH}/images/train" - # Open and load the YAML file with open(FILE_PATH, 'r') as file: data = yaml.safe_load(file) +# Check if DATASET_NAME is in the train and val paths +if DATASET_NAME in data['train'] and DATASET_NAME in data['val']: + # Extract subdirectories after the dataset name in the original paths + train_subdirs = data['train'].split(DATASET_NAME, 1)[1] + val_subdirs = data['val'].split(DATASET_NAME, 1)[1] +elif data['train'].startswith('..') and data['val'].startswith('..'): + # Remove the '..' from the original paths + train_subdirs = data['train'][2:] + val_subdirs = data['val'][2:] +else: + raise ValueError("Invalid format for train or val paths") + +# Define the new paths +NEW_PATH = f"/opt/ml/input/data/{DATASET_NAME}" +NEW_TRAIN = f"{NEW_PATH}{train_subdirs}" +NEW_VAL = f"{NEW_PATH}{val_subdirs}" + # Modify the values data['path'] = NEW_PATH data['train'] = NEW_TRAIN From 9c8c1b06b6e3a4b6f505e57be00e82be193b0efc Mon Sep 17 00:00:00 2001 From: kerrlabajo Date: Thu, 16 May 2024 12:49:15 +0800 Subject: [PATCH 2/5] Fix unreplaced backslash into frontslash instead --- docker/yolov5-training/configure_dataset.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/yolov5-training/configure_dataset.py b/docker/yolov5-training/configure_dataset.py index e674fd9..03564db 100644 --- a/docker/yolov5-training/configure_dataset.py +++ b/docker/yolov5-training/configure_dataset.py @@ -33,8 +33,8 @@ # Define the new paths NEW_PATH = f"/opt/ml/input/data/{DATASET_NAME}" -NEW_TRAIN = f"{NEW_PATH}{train_subdirs}" -NEW_VAL = f"{NEW_PATH}{val_subdirs}" +NEW_TRAIN = f"{NEW_PATH}{train_subdirs}".replace('\\', '/') +NEW_VAL = f"{NEW_PATH}{val_subdirs}".replace('\\', '/') # Modify the values data['path'] = NEW_PATH From b7aa23a1a48694e7189a6bb19376e2405b87b579 Mon Sep 17 00:00:00 2001 From: kerrlabajo Date: Thu, 16 May 2024 12:58:39 +0800 Subject: [PATCH 3/5] Fix if `DATASET_NAME.yaml` does not exist, it will locate other to be renamed --- docker/yolov5-training/configure_dataset.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker/yolov5-training/configure_dataset.py b/docker/yolov5-training/configure_dataset.py index 03564db..c728155 100644 --- a/docker/yolov5-training/configure_dataset.py +++ b/docker/yolov5-training/configure_dataset.py @@ -1,6 +1,7 @@ import yaml import argparse import os +import glob # Define the argument parser parser = argparse.ArgumentParser(description='Configure dataset') @@ -15,6 +16,16 @@ # Define the file path FILE_PATH = args.dataset_config_path +# Check if DATASET_NAME.yaml exists +if not os.path.isfile(FILE_PATH): + # If not, find any .yaml file in the current directory + yaml_files = glob.glob('*.yaml') + if yaml_files: + # Rename the first .yaml file to DATASET_NAME.yaml + os.rename(yaml_files[0], FILE_PATH) + else: + raise FileNotFoundError("No .yaml file found to rename") + # Open and load the YAML file with open(FILE_PATH, 'r') as file: data = yaml.safe_load(file) From 608c7c0c598cbf705b9baa6b4b9b7bef698e3cf8 Mon Sep 17 00:00:00 2001 From: kerrlabajo Date: Thu, 16 May 2024 13:08:39 +0800 Subject: [PATCH 4/5] Fix unspecified base path before locating any `.yaml` --- docker/yolov5-training/configure_dataset.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/yolov5-training/configure_dataset.py b/docker/yolov5-training/configure_dataset.py index c728155..5a832a9 100644 --- a/docker/yolov5-training/configure_dataset.py +++ b/docker/yolov5-training/configure_dataset.py @@ -16,10 +16,13 @@ # Define the file path FILE_PATH = args.dataset_config_path +# Get the directory from the dataset_config_path +dir_path = os.path.dirname(args.dataset_config_path) + # Check if DATASET_NAME.yaml exists if not os.path.isfile(FILE_PATH): # If not, find any .yaml file in the current directory - yaml_files = glob.glob('*.yaml') + yaml_files = glob.glob(os.path.join(dir_path, '*.yaml')) if yaml_files: # Rename the first .yaml file to DATASET_NAME.yaml os.rename(yaml_files[0], FILE_PATH) From cd1172e9ffc7085599b395d463fc8ec526694f3f Mon Sep 17 00:00:00 2001 From: kerrlabajo Date: Thu, 16 May 2024 14:30:07 +0800 Subject: [PATCH 5/5] Check if ECR exists, otherwise create --- docker/scripts/pull_build_push.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/scripts/pull_build_push.sh b/docker/scripts/pull_build_push.sh index 1153be3..3c0dcb7 100644 --- a/docker/scripts/pull_build_push.sh +++ b/docker/scripts/pull_build_push.sh @@ -76,6 +76,9 @@ NEW_TAG="${VERSION}${TAG_BASE}" # Authenticate Docker to ECR aws ecr get-login-password --region ${AWS_REGION} | sudo docker login --username AWS --password-stdin ${ECR_URL} +# Check if the repository exists, if not create it +aws ecr describe-repositories --repository-names ${DOCKER_IMAGE} > /dev/null 2>&1 || aws ecr create-repository --repository-name ${DOCKER_IMAGE} > /dev/null 2>&1 + # Build and push the image sudo docker build -t ${ECR_URL}/${DOCKER_IMAGE}:${NEW_TAG} -f ../yolov5-training/Dockerfile ../yolov5-training sudo docker push ${ECR_URL}/${DOCKER_IMAGE}:${NEW_TAG}