From 187e52fc3db844645b469cc35122ccbc188357a1 Mon Sep 17 00:00:00 2001 From: Ajay Karpur Date: Tue, 11 May 2021 23:00:33 +0000 Subject: [PATCH] infra: sort imports in all .py files with isort (#2225) --- .../container-training/inference.py | 23 +- .../container-training/train.py | 32 ++- .../autogluon-tabular/utils/ag_utils.py | 10 +- .../docker/code/main.py | 10 +- .../docker/code/utils.py | 4 +- .../notebook/source_dir/train.py | 6 +- .../framework-container/package/setup.py | 5 +- .../src/custom_framework_training/training.py | 1 + .../notebook/source_dir/train.py | 6 +- .../docker/code/train.py | 6 +- .../aws-mask-rcnn.py | 6 +- .../resources/train.py | 7 +- .../container-optimized/resources/train.py | 7 +- .../container-script-mode/resources/train.py | 7 +- .../resources/predict.py | 30 +-- .../resources/serve.py | 2 +- .../container-serving/resources/predict.py | 32 +-- .../container-serving/resources/serve.py | 1 + .../container/resources/train.py | 7 +- .../tensorpack-mask-rcnn.py | 6 +- .../fairseq/distributed_train.py | 8 +- .../fairseq_translation/fairseq/predictor.py | 5 +- .../fairseq/sagemaker_translate.py | 14 +- .../fairseq_translation/fairseq/serve | 1 + .../fairseq_translation/fairseq/train | 10 +- .../fairseq/train_driver.py | 6 +- .../fastai_oxford_pets/source/pets.py | 9 +- .../dbpedia_processing.py | 22 +- .../abalone_processing.py | 22 +- .../preprocessor.py | 23 +- .../jax_bring_your_own/sagemaker_jax.py | 2 +- .../training_scripts/train_jax.py | 2 +- .../training_scripts/train_trax.py | 2 +- .../container/dockerd-entrypoint.py | 7 +- .../container/model_handler.py | 2 +- .../sklearn_preprocessor.py | 24 +- .../multi_model_pytorch/code/inference.py | 1 - .../multi_model_pytorch/code/train.py | 11 +- .../mxnet_mnist_byom/mnist.py | 6 +- .../container/cifar10/cifar10.py | 5 +- .../utils/utils_cifar.py | 4 +- .../container/decision_trees/predictor.py | 7 +- .../container/decision_trees/train | 3 +- .../container/cifar10/cifar10.py | 7 +- .../container/cifar10/resnet_model.py | 4 +- .../container/cifar10/train | 4 +- .../utils/generate_cifar10_tfrecords.py | 12 +- .../iris_dnn_classifier.py | 1 + .../unity_ml_agents/src/train.py | 11 +- .../sklearn_feature_selection.py | 24 +- .../container/decision_trees/predictor.py | 7 +- .../container/decision_trees/serve | 1 + .../container/decision_trees/train | 5 +- .../generate_tensorboard_command.py | 5 +- .../source/keras_cifar10.py | 8 +- .../mxnet_gluon_sentiment/sentiment.py | 4 +- .../frameworks/mxnet_onnx_ei/resnet152.py | 1 + .../getting_started/preprocessor.py | 1 + .../xgboost_abalone/abalone.py | 7 +- .../scikit_learn_iris/scikit_learn_iris.py | 3 +- .../mnist_tensor_analysis/src/mnist.py | 4 +- .../mnist_tensor_analysis/utils.py | 3 +- .../tensorflow_builtin_rule/src/mnist_byoc.py | 5 +- .../src/mnist_zerocodechange.py | 3 +- .../mnist.py | 7 +- .../utils.py | 1 + .../gluoncv_ssd_mobilenet/ssd_entry_point.py | 14 +- .../gluoncv_ssd_mobilenet/tools/concat_db.py | 3 +- .../gluoncv_ssd_mobilenet/tools/im2rec.py | 8 +- .../gluoncv_ssd_mobilenet/tools/imdb.py | 3 +- .../gluoncv_ssd_mobilenet/tools/pascal_voc.py | 6 +- .../tools/prepare_dataset.py | 6 +- .../imageclassification_caltech/inference.py | 9 +- .../pytorch_torchvision/code/resnet18.py | 2 +- .../code/vgg19_bn_compiled.py | 2 +- .../code/vgg19_bn_uncompiled.py | 2 +- .../tensorflow_distributed_mnist/mnist.py | 7 +- .../tensorflow_distributed_mnist/utils.py | 1 + .../cifar10/cifar10.py | 7 +- .../cifar10/resnet_model.py | 4 +- .../tensorflow_bring_your_own/cifar10/train | 4 +- .../utils/generate_cifar10_tfrecords.py | 12 +- .../xgboost/predictor.py | 7 +- .../xgboost_bring_your_own/xgboost/serve | 1 + .../xgboost_bring_your_own/xgboost/train | 4 +- .../postprocessor/docker/code/main.py | 55 ++-- .../postprocessor/docker/code/predictor.py | 21 +- .../postprocessor/docker/code/utils.py | 4 +- .../preprocessor/docker/code/main.py | 52 ++-- .../preprocessor/docker/code/predictor.py | 24 +- .../preprocessor/docker/code/utils.py | 4 +- .../generate-training-data.py | 2 +- .../load-ddb-data.py | 1 + end_to_end/create_dataset.py | 5 +- end_to_end/demo_helpers.py | 3 +- end_to_end/deploy_model.py | 4 +- end_to_end/xgboost_starter_script.py | 9 +- frameworks/mxnet/code/inference.py | 7 +- frameworks/mxnet/code/test_inference.py | 8 +- frameworks/mxnet/code/test_train.py | 8 +- frameworks/mxnet/code/train.py | 9 +- frameworks/pytorch/code/test_inference.py | 8 +- frameworks/pytorch/code/test_train.py | 8 +- frameworks/pytorch/code/train.py | 3 +- frameworks/tensorflow/code/test_train.py | 8 +- frameworks/tensorflow/code/train.py | 9 +- frameworks/utils/mnist.py | 7 +- .../container/news-classifier/predictor.py | 24 +- .../container/news-classifier/serve | 1 + .../container/news-classifier/train | 20 +- .../ActiveLearning/create_validation_set.py | 4 +- .../src/ActiveLearning/helper.py | 2 +- .../ActiveLearning/perform_active_learning.py | 8 +- .../ActiveLearning/prepare_for_inference.py | 5 +- .../ActiveLearning/prepare_for_training.py | 7 +- .../src/Bootstrap/add_record_id.py | 4 +- .../src/Bootstrap/copy_input_manifest.py | 4 +- .../src/Labeling/prepare_for_labeling.py | 4 +- .../src/MetaData/get_counts.py | 4 +- .../src/Output/export_final.py | 4 +- .../src/Output/export_partial.py | 6 +- .../src/dependency/python/s3_helper.py | 8 +- .../src/dependency/python/string_helper.py | 3 +- .../src/tests/test_add_record_id.py | 3 +- .../src/tests/test_copy_input_manifest.py | 4 +- .../src/tests/test_create_validation_set.py | 3 +- .../src/tests/test_get_counts.py | 3 +- .../src/tests/test_perform_active_learning.py | 5 +- .../src/tests/test_prepare_for_inference.py | 4 +- .../src/tests/test_prepare_for_labeling.py | 3 +- .../src/tests/test_prepare_for_training.py | 5 +- .../src/tests/test_simple_active_learning.py | 2 +- .../ground_truth_od.py | 3 +- .../glue_cw_worker_metrics_transform.py | 10 +- .../glue_reporting_transform_script.py | 26 +- .../src/lambda_src/api_batch_create/main.py | 7 +- .../api_batch_metadata_post/main.py | 4 +- .../src/lambda_src/api_batch_show/main.py | 5 +- .../src/lambda_src/api_workforce_show/main.py | 3 +- .../labeling_job_state_change/main.py | 8 +- .../src/lambda_src/main.py | 29 +-- .../src/lambda_src/shared/api_helpers.py | 7 +- .../src/lambda_src/shared/db.py | 12 +- .../src/lambda_src/shared/log.py | 2 +- .../src/lambda_src/shared/s3_accessor.py | 1 - .../step_functions_batch_error/main.py | 5 +- .../main.py | 7 +- .../step_functions_transformation/main.py | 1 - .../main.py | 11 +- .../main.py | 2 +- .../main.py | 1 - .../lambda_test/api_batch_create/main_test.py | 7 +- .../lambda_test/api_batch_show/main_test.py | 4 +- .../labeling_job_state_change/main_test.py | 2 +- .../step_function_batch_error/main_test.py | 2 +- .../main_test.py | 4 +- ...ntic_segmentation_mask_index_correction.py | 2 +- .../plotting_funcs.py | 4 +- .../quality_metrics_cli.py | 9 +- .../chainer_cifar10/s3_util.py | 5 +- .../src/chainer_cifar_vgg_single_machine.py | 11 +- .../keras_bring_your_own/main.py | 3 +- .../trainer/environment.py | 5 +- .../keras_bring_your_own/trainer/start.py | 14 +- .../cifar10.py | 1 - .../cifar10_utils.py | 5 +- .../random_tuner.py | 6 +- hyperparameter_tuning/mxnet_mnist/mnist.py | 1 - hyperparameter_tuning/pytorch_mnist/mnist.py | 3 +- .../rapids_bring_your_own/code/HPOConfig.py | 10 +- .../rapids_bring_your_own/code/MLWorkflow.py | 8 +- .../rapids_bring_your_own/code/serve.py | 24 +- .../rapids_bring_your_own/code/train.py | 2 +- .../code/workflows/MLWorkflowMultiCPU.py | 34 ++- .../code/workflows/MLWorkflowMultiGPU.py | 38 ++- .../code/workflows/MLWorkflowSingleCPU.py | 29 +-- .../code/workflows/MLWorkflowSingleGPU.py | 27 +- .../rapids_bring_your_own/helper_functions.py | 9 +- .../tensorflow2_mnist/code/test_train.py | 9 +- .../tensorflow2_mnist/code/train.py | 11 +- .../tensorflow_mnist/mnist.py | 1 + .../tensorflow_mnist/utils.py | 1 + .../ipinsights_login/generate_data.py | 7 +- .../generate_example_data.py | 3 +- .../tools/concat_db.py | 3 +- .../tools/im2rec.py | 8 +- .../tools/imdb.py | 3 +- .../tools/pascal_voc.py | 6 +- .../tools/prepare_dataset.py | 6 +- .../ntm_synthetic/generate_example_data.py | 2 +- .../tools/birdsOnEdge.py | 19 +- .../object_detection_birds/tools/im2rec.py | 8 +- .../tools/concat_db.py | 3 +- .../tools/im2rec.py | 8 +- .../tools/imdb.py | 3 +- .../tools/pascal_voc.py | 6 +- .../tools/prepare_dataset.py | 6 +- .../create_vocab_proto.py | 13 +- .../seq2seq_translation_en-de/record_pb2.py | 2 +- .../xgboost_abalone/abalone.py | 7 +- .../xgboost_abalone/inference.py | 1 - .../gluon_recommender_system/recommender.py | 7 +- .../training_pytorch/pytorch_train.py | 9 +- .../training_tensorflow/tensorflow_train.py | 3 +- .../bandits_recsys_movielens_testbed/misc.py | 4 +- .../src/env.py | 1 + .../src/io_utils.py | 7 +- .../src/train.py | 4 +- .../src/vw_agent.py | 13 +- .../src/vw_utils.py | 2 +- .../sim_app/sim_app_utils.py | 4 +- .../sim_app/statlog_sim_app.py | 11 +- .../src/eval-cfa-vw.py | 7 +- .../src/io_utils.py | 7 +- .../src/train-vw.py | 11 +- .../src/vw_model.py | 9 +- .../src/vw_utils.py | 2 +- reinforcement_learning/common/docker_utils.py | 4 +- reinforcement_learning/common/env_utils.py | 5 +- reinforcement_learning/common/misc.py | 4 +- .../common/sagemaker_rl/coach_launcher.py | 29 ++- .../common/sagemaker_rl/mpi_launcher.py | 16 +- .../common/sagemaker_rl/onnx_utils.py | 2 +- .../clients/ddb/experiment_db_client.py | 1 + .../clients/ddb/join_db_client.py | 1 + .../orchestrator/resource_manager.py | 9 +- .../orchestrator/utils/cloudwatch_logger.py | 2 +- .../workflow/manager/experiment_manager.py | 27 +- .../workflow/manager/join_manager.py | 13 +- .../workflow/manager/model_manager.py | 22 +- .../common/sagemaker_rl/ray_launcher.py | 8 +- .../sagemaker_rl/sage_cluster_communicator.py | 5 +- .../sagemaker_rl/stable_baselines_launcher.py | 11 +- .../common/sagemaker_rl/tf_serving_utils.py | 3 +- .../common/tests/unit/test_ray_launcher.py | 4 +- .../src/evaluate-coach.py | 3 +- .../src/preset-cartpole-ddqnbcq-env.py | 19 +- .../src/preset-cartpole-ddqnbcq.py | 17 +- .../src/train-coach.py | 3 +- .../src/deploy-mxnet-coach.py | 7 +- .../rl_cartpole_coach/src/evaluate-coach.py | 3 +- .../src/preset-cartpole-clippedppo.py | 6 +- .../src/preset-cartpole-dqn.py | 6 +- .../rl_cartpole_ray/src/evaluate-ray.py | 9 +- .../src/train-rl-cartpole-ray.py | 1 - .../copy_to_sagemaker_container.py | 1 + .../sim_app_bundler.py | 4 +- .../lib/model_validator/response_handlers.py | 3 +- .../src/lib/model_validator/validator.py | 9 +- .../src/lib/serve | 6 +- .../markov/agent_ctrl/bot_cars_agent_ctrl.py | 32 ++- .../markov/agent_ctrl/obstacles_agent_ctrl.py | 18 +- .../markov/agent_ctrl/rollout_agent_ctrl.py | 65 +++-- .../markov/agent_ctrl/training_agent_ctrl.py | 2 +- .../src/markov/agent_ctrl/utils.py | 18 +- .../markov/agents/rollout_agent_factory.py | 6 +- .../markov/agents/training_agent_factory.py | 2 +- .../src/markov/agents/utils.py | 7 +- .../src/markov/architecture/contrib.py | 2 +- .../architecture/custom_architectures.py | 2 +- .../markov/architecture/embedder_factory.py | 6 +- .../src/markov/auth/refreshed_session.py | 5 +- .../src/markov/boto/deepracer_boto_client.py | 12 +- .../src/markov/boto/s3/constants.py | 1 - .../src/markov/boto/s3/files/checkpoint.py | 30 +-- .../deepracer_checkpoint_json.py | 26 +- .../checkpoint_files/rl_coach_checkpoint.py | 34 +-- .../checkpoint_files/rl_coach_sync_file.py | 16 +- .../checkpoint_files/tensorflow_model.py | 26 +- .../markov/boto/s3/files/hyperparameters.py | 8 +- .../src/markov/boto/s3/files/ip_config.py | 21 +- .../src/markov/boto/s3/files/metrics.py | 8 +- .../markov/boto/s3/files/model_metadata.py | 14 +- .../markov/boto/s3/files/reward_function.py | 10 +- .../markov/boto/s3/files/simtrace_video.py | 6 +- .../files/virtual_event_best_sector_time.py | 26 +- .../src/markov/boto/s3/files/yaml_file.py | 41 ++- .../src/markov/boto/s3/s3_client.py | 13 +- .../src/markov/boto/sqs/sqs_client.py | 9 +- .../src/markov/camera_utils.py | 7 +- .../src/markov/cameras/abs_camera.py | 4 +- .../src/markov/cameras/camera_factory.py | 3 +- .../src/markov/cameras/camera_manager.py | 4 +- .../src/markov/cameras/constants.py | 2 + .../src/markov/cameras/frustum.py | 7 +- .../src/markov/cameras/frustum_manager.py | 3 +- .../cameras/handlers/follow_car_camera.py | 4 +- .../src/markov/cameras/handlers/top_camera.py | 12 +- .../src/markov/cameras/utils.py | 3 +- .../src/markov/deepracer_memory.py | 27 +- .../visual/light_randomizer.py | 20 +- .../visual/model_visual_randomizer.py | 23 +- .../environments/deepracer_racetrack_env.py | 22 +- .../src/markov/evaluation_worker.py | 73 +++--- .../deepracer_categorical.py | 3 +- .../observation/observation_binary_filter.py | 8 +- .../src/markov/gazebo_tracker/abs_tracker.py | 3 +- .../markov/gazebo_tracker/tracker_manager.py | 14 +- .../trackers/get_link_state_tracker.py | 12 +- .../trackers/get_model_state_tracker.py | 12 +- .../trackers/set_model_state_tracker.py | 10 +- .../trackers/set_visual_color_tracker.py | 10 +- .../set_visual_transparency_tracker.py | 11 +- .../src/markov/gazebo_utils/model_updater.py | 26 +- .../log_handler/deepracer_exceptions.py | 240 ++++++++++-------- .../markov/log_handler/exception_handler.py | 23 +- .../src/markov/memories/deepracer_memory.py | 18 +- .../src/markov/metrics/constants.py | 7 +- .../src/markov/metrics/s3_metrics.py | 45 ++-- .../multi_agent_coach/action_space_configs.py | 7 +- .../multi_agent_coach/agents/sac_agent.py | 20 +- .../tensorflow_components/heads/sac_head.py | 5 +- .../tensorflow_components/heads/sac_q_head.py | 5 +- .../multi_agent_environment.py | 10 +- .../multi_agent_graph_manager.py | 39 ++- .../multi_agent_level_manager.py | 8 +- .../src/markov/multi_agent_coach/spaces.py | 7 +- .../src/markov/reset/abstract_reset_rule.py | 1 + .../src/markov/reset/reset_rules_manager.py | 2 +- .../markov/reset/rules/crash_reset_rule.py | 4 +- .../rules/episode_complete_reset_rule.py | 2 +- .../reset/rules/immobilized_reset_rule.py | 3 +- .../reset/rules/off_track_reset_rule.py | 4 +- .../src/markov/reset/rules/race_time_rule.py | 3 +- .../markov/reset/rules/reverse_reset_rule.py | 3 +- .../src/markov/reset/utils.py | 4 +- .../src/markov/rollout_constants.py | 1 + .../src/markov/rollout_utils.py | 22 +- .../src/markov/rollout_worker.py | 92 ++++--- .../src/markov/rospy_wrappers.py | 12 +- .../src/markov/s3_boto_data_store.py | 28 +- .../src/markov/sagemaker_graph_manager.py | 27 +- .../src/markov/samples/sample_collector.py | 4 +- .../src/markov/sensors/composite_sensor.py | 3 +- .../src/markov/sensors/sensor_interface.py | 4 +- .../src/markov/sensors/sensors_rollout.py | 28 +- .../src/markov/sensors/sensors_training.py | 10 +- .../src/markov/sensors/utils.py | 9 +- .../src/markov/state_machine/fsm.py | 2 +- .../src/markov/track_geom/constants.py | 2 +- .../track_geom/spline/abstract_spline.py | 3 +- .../track_geom/spline/lane_change_spline.py | 9 +- .../markov/track_geom/spline/track_spline.py | 6 +- .../src/markov/track_geom/track_data.py | 19 +- .../src/markov/track_geom/utils.py | 5 +- .../src/markov/training_worker.py | 58 ++--- .../src/markov/utils.py | 60 ++--- .../src/markov/validation_worker.py | 29 +-- .../src/markov/virtual_event/utils.py | 15 +- .../virtual_event/virtual_event_manager.py | 108 ++++---- .../src/markov/virtual_event_worker.py | 12 +- .../src/markov/visual_effects/abs_effect.py | 1 + .../markov/visual_effects/effect_manager.py | 3 +- .../visual_effects/effects/blink_effect.py | 19 +- .../rl_game_server_autopilot/ap-server/app.py | 13 +- .../sagemaker/src/gameserver_env.py | 7 +- .../sagemaker/src/train_gameserver_ppo.py | 2 +- .../src/deploy-mxnet-coach.py | 7 +- .../src/eplus/envs/__init__.py | 6 +- .../src/eplus/envs/data_center_env.py | 11 +- .../src/eplus/envs/large_office_env.py | 11 +- .../src/eplus/envs/pyEp.py | 5 +- .../src/eplus/envs/pyEpError.py | 10 +- .../src/eplus/envs/socket_builder.py | 2 +- .../src/evaluate-baseline.py | 2 +- .../src/evaluate-coach.py | 3 +- .../src/preset-energy-plus-clipped-ppo.py | 8 +- .../source/custom/callbacks.py | 7 +- .../source/eplus/envs/medium_office_env.py | 20 +- .../source/hvac_ray_launcher.py | 7 +- .../source/ray_experiment_builder.py | 8 +- .../rl_hvac_ray_energyplus/source/rollout.py | 6 +- .../source/train-sagemaker-hvac.py | 9 +- .../src/evaluate-coach.py | 3 +- .../src/knapsack_baseline.py | 5 +- .../src/knapsack_env.py | 16 +- .../src/preset-knapsack-clippedppo.py | 6 +- .../src/train-coach.py | 3 +- .../src/deploy-mxnet-coach.py | 7 +- .../src/evaluate-coach.py | 3 +- .../src/preset-cartpole-clippedppo.py | 6 +- .../src/preset-cartpole-dqn.py | 6 +- .../src/deploy-mxnet-coach.py | 7 +- .../src/evaluate-coach.py | 3 +- .../src/patient_envs.py | 2 +- ...set-mountain-car-continuous-clipped-ppo.py | 6 +- .../src/preset-mountain-car-dqn.py | 8 +- .../src/environment.py | 12 +- .../tensorflow_resnet/compressor/__init__.py | 5 +- .../compressor/core/__init__.py | 4 +- .../compressor/core/module.py | 12 +- .../compressor/layers/__init__.py | 4 +- .../compressor/layers/activation.py | 5 +- .../compressor/layers/batch_norm.py | 8 +- .../compressor/layers/conv.py | 11 +- .../compressor/layers/dense.py | 8 +- .../compressor/layers/ops.py | 19 +- .../compressor/layers/pooling.py | 7 +- .../compressor/layers/renset_blocks.py | 7 +- .../tensorflow_resnet/compressor/resnet.py | 22 +- .../compressor/tf_interface.py | 3 +- .../src/tensorflow_resnet/compressor/train.py | 5 +- .../src/tensorflow_resnet/dataset/cifar10.py | 1 + .../dataset/cifar10_download_and_extract.py | 6 +- .../tensorflow_resnet/network_compression.py | 26 +- .../src/train-ray.py | 5 +- .../src/config.py | 2 +- .../src/evaluate-coach.py | 3 +- .../src/portfolio_env.py | 8 +- .../preset-portfolio-management-clippedppo.py | 12 +- .../src/utils.py | 5 +- .../src/autoscalesim.py | 4 +- .../src/evaluate-coach.py | 3 +- .../src/gymhelper.py | 2 +- .../src/preset-autoscale-a3c.py | 14 +- .../src/preset-autoscale-ppo.py | 6 +- .../src/train-coach.py | 5 +- .../src/news_vendor_env.py | 4 +- .../src/train_bin_packing.py | 3 +- .../src/train_news_vendor.py | 1 - .../src/train_vehicle_routing_problem.py | 3 +- .../src/vrp_env.py | 2 +- .../rl_roboschool_ray/src/evaluate-ray.py | 9 +- .../rl_roboschool_ray/src/train-hopper.py | 3 +- .../rl_roboschool_ray/src/train-humanoid.py | 3 +- .../rl_roboschool_ray/src/train-reacher.py | 3 +- .../src/data.py | 5 +- .../src/evaluate-coach.py | 3 +- .../src/preset-stock-trading-ddqn.py | 6 +- .../src/trading_env.py | 8 +- .../common/docker_utils.py | 4 +- .../common/misc.py | 4 +- .../common/sagemaker_rl/coach_launcher.py | 29 ++- .../common/sagemaker_rl/mpi_launcher.py | 16 +- .../common/sagemaker_rl/onnx_utils.py | 2 +- .../common/sagemaker_rl/ray_launcher.py | 3 +- .../sagemaker_rl/sage_cluster_communicator.py | 5 +- .../sagemaker_rl/stable_baselines_launcher.py | 11 +- .../src/deploy-coach.py | 7 +- .../src/preset.py | 4 +- .../src/tic_tac_toe.py | 7 +- .../tic_tac_toe_game.py | 5 +- .../src/TSP_baseline.py | 1 - .../src/TSP_env.py | 1 - .../src/TSP_view_2D.py | 2 +- .../src/VRP_abstract_env.py | 3 +- .../src/VRP_baseline_easy.py | 3 +- .../src/VRP_baseline_utils.py | 5 +- .../src/VRP_env.py | 1 - .../src/VRP_view_2D.py | 2 +- .../src/evaluate-coach.py | 3 +- .../src/preset-tsp-easy.py | 10 +- .../src/preset-tsp-medium.py | 10 +- .../src/preset-vrp-easy.py | 10 +- .../rl_unity_ray/src/evaluate-unity.py | 17 +- .../rl_unity_ray/src/train-unity.py | 9 +- .../docker/tf_keras_resnet_byoc.py | 6 +- .../train_tf.py | 6 +- .../mnist_tensor_analysis/src/mnist.py | 4 +- .../mnist_tensor_analysis/utils.py | 3 +- .../src/autoencoder_mnist.py | 5 +- .../entry_point/data.py | 4 +- .../entry_point/train.py | 11 +- .../utils/attention_head_view.py | 8 +- .../utils/neuron_view.py | 10 +- .../entry_point/custom_hook.py | 2 +- .../entry_point/train.py | 12 +- .../mxnet_gluon_spot_training.py | 5 +- .../scripts/pytorch_mnist.py | 12 +- .../custom_rule/check_accuracy.py | 4 +- .../model_alexnet.py | 8 +- .../model_resnet.py | 8 +- .../src/train.py | 19 +- .../entry_point/distributed_launch.py | 1 - .../src/tf_keras_resnet_byoc.py | 6 +- .../src/keras_fit_zero_code_change.py | 4 +- .../src/mnist_zerocodechange.py | 3 +- .../src/tf-resnet50-cifar10.py | 2 +- .../src/tf_keras_resnet_byoc.py | 2 +- .../src/tf_keras_resnet_zerocodechange.py | 2 +- .../sentiment-distributed.py | 6 +- .../demo/generate_cifar10_tfrecords.py | 4 +- .../demo/generate_large_tfrecords.py | 4 +- .../entry_point/tf-TensorLike.py | 6 +- .../entry_point/train_tf.py | 6 +- .../mnist.py | 9 +- .../code/data_util.py | 3 +- .../code/inference.py | 1 + .../track-an-airflow-workflow/code/train.py | 10 +- .../container/local_test/test_container.py | 3 +- .../local_test/test_container.py | 3 +- .../train.py | 3 +- .../local_test/test_container.py | 3 +- .../container_kms/train.py | 9 +- .../execution-role/iam_helpers.py | 5 +- .../pipelines/customer_churn/evaluate.py | 7 +- .../pipelines/customer_churn/pipeline.py | 32 +-- .../chainer_cifar10/s3_util.py | 5 +- .../src/chainer_cifar_vgg_distributed.py | 13 +- .../src/chainer_cifar_vgg_single_machine.py | 11 +- .../chainer_mnist_single_machine.py | 7 +- .../chainer_sentiment_analysis/dataset.py | 5 +- .../chainer_sentiment_analysis/s3_util.py | 5 +- .../chainer_sentiment_analysis/src/nets.py | 3 +- .../src/nlp_utils.py | 3 +- .../src/sentiment_analysis.py | 15 +- sagemaker-python-sdk/dgl_gcmc/data.py | 10 +- sagemaker-python-sdk/dgl_gcmc/model.py | 6 +- sagemaker-python-sdk/dgl_gcmc/train.py | 19 +- sagemaker-python-sdk/dgl_gcmc/utils.py | 9 +- sagemaker-python-sdk/dgl_gcn/mxnet_gcn.py | 21 +- sagemaker-python-sdk/dgl_gcn/pytorch_gcn.py | 15 +- sagemaker-python-sdk/dgl_gcn_tox21/main.py | 8 +- .../dgl_kge/dataloader/KGDataset.py | 6 +- .../dgl_kge/dataloader/sampler.py | 10 +- sagemaker-python-sdk/dgl_kge/eval.py | 17 +- .../dgl_kge/models/general_models.py | 35 +-- .../dgl_kge/models/mxnet/score_fun.py | 4 +- .../dgl_kge/models/mxnet/tensor_models.py | 5 +- .../dgl_kge/models/pytorch/score_fun.py | 2 +- .../dgl_kge/models/pytorch/tensor_models.py | 2 +- sagemaker-python-sdk/dgl_kge/train.py | 16 +- sagemaker-python-sdk/dgl_kge/train_mxnet.py | 11 +- sagemaker-python-sdk/dgl_kge/train_pytorch.py | 11 +- .../generate_tensorboard_command.py | 5 +- .../source_dir/cifar10_keras_main.py | 8 +- .../managed_spot_training_mxnet/mnist.py | 6 +- .../iris_dnn_classifier.py | 3 +- .../mxnet_gluon_mnist/mnist.py | 9 +- .../mxnet_gluon_sentiment/sentiment.py | 4 +- .../source/train_faster_rcnn.py | 19 +- .../source/train_mask_rcnn.py | 23 +- .../mxnet_horovod_mnist/code/horovod_mnist.py | 4 +- .../mxnet_horovod_mnist/code/test_train.py | 8 +- sagemaker-python-sdk/mxnet_mnist/mnist.py | 6 +- sagemaker-python-sdk/mxnet_mnist/mnist_ei.py | 6 +- .../mxnet_onnx_eia/resnet152.py | 1 + .../mxnet_onnx_export/mnist.py | 5 +- .../super_resolution.py | 6 +- .../pytorch_cnn_cifar10/source/cifar10.py | 3 +- .../pytorch_cnn_cifar10/utils_cifar.py | 4 +- .../pytorch_horovod_mnist/code/inference.py | 3 +- .../pytorch_horovod_mnist/code/mnist.py | 7 +- .../pytorch_horovod_mnist/code/model_def.py | 1 + .../pytorch_mnist/code/deploy_ei.py | 1 + sagemaker-python-sdk/pytorch_mnist/mnist.py | 3 +- .../sklearn_abalone_featurizer.py | 22 +- .../scikit_learn_iris/scikit_learn_iris.py | 5 +- .../source_dir/tensorboard_keras_cifar10.py | 20 +- .../train_model/train.py | 7 +- .../mnist.py | 7 +- .../utils.py | 1 + .../mnist_hvd.py | 11 +- .../pipemode.py | 2 +- .../mnist-2.py | 5 +- .../mnist.py | 15 +- .../sample_utils.py | 4 +- .../pytorch_mnist_batch_transform/mnist.py | 4 +- .../code/inference.py | 3 +- .../code/model_def.py | 4 +- .../code/train.py | 8 +- .../code/utilities.py | 5 +- .../generate_cifar10_tfrecords.py | 8 +- .../code/inference.py | 1 + .../code/inference.py | 3 +- .../dnn-classifier/train.py | 3 +- .../app/main.py | 4 +- .../introduction/preprocessor.py | 1 + .../tensorflow/monitoringjob_utils.py | 6 +- .../visualization/utils.py | 5 +- .../get_input.py | 3 +- .../mnist.py | 5 +- .../resnet18.py | 3 +- .../mnist.py | 7 +- .../utils.py | 1 + .../gluoncv_ssd_mobilenet/ssd_entry_point.py | 14 +- .../gluoncv_ssd_mobilenet/tools/concat_db.py | 3 +- .../gluoncv_ssd_mobilenet/tools/im2rec.py | 8 +- .../gluoncv_ssd_mobilenet/tools/imdb.py | 3 +- .../gluoncv_ssd_mobilenet/tools/pascal_voc.py | 6 +- .../tools/prepare_dataset.py | 6 +- .../gluoncv_yolo_darknet/tools/concat_db.py | 3 +- .../gluoncv_yolo_darknet/tools/im2rec.py | 8 +- .../gluoncv_yolo_darknet/tools/imdb.py | 3 +- .../gluoncv_yolo_darknet/tools/pascal_voc.py | 6 +- .../tools/prepare_dataset.py | 6 +- .../gluoncv_yolo_darknet/train_yolo.py | 54 ++-- .../imageclassification_caltech/inference.py | 9 +- .../mxnet_mnist/get_input.py | 3 +- .../mxnet_mnist/mnist.py | 5 +- .../pytorch_torchvision/code/resnet18.py | 2 +- .../code/vgg19_bn_compiled.py | 2 +- .../code/vgg19_bn_uncompiled.py | 2 +- .../tensorflow_distributed_mnist/mnist.py | 7 +- .../tensorflow_distributed_mnist/utils.py | 1 + .../container/program/bootstrap.py | 5 +- .../container/program/submit | 9 +- .../generate_example_data.py | 3 +- .../code/glue_etl.py | 7 +- .../code/query_training_status.py | 5 +- .../training_pipeline_pytorch_mnist/mnist.py | 3 +- .../train_pytorch_smdataparallel_maskrcnn.py | 15 +- .../data_parallel/mnist/code/inference.py | 1 + .../data_parallel/mnist/code/model_def.py | 2 +- .../train_pytorch_smdataparallel_mnist.py | 11 +- .../bert/bert_example/file_utils.py | 2 +- .../bert/bert_example/modeling.py | 12 +- .../bert_example/sagemaker_smp_pretrain.py | 66 +++-- .../bert/bert_example/schedulers.py | 3 +- .../model_parallel/bert/bert_example/utils.py | 5 +- .../model_parallel/mnist/utils/pt_mnist.py | 25 +- .../train_tensorflow_smdataparallel_mnist.py | 5 +- .../model_parallel/mnist/utils/tf2.py | 20 +- .../model_parallel/mnist/utils/tf2_hvd.py | 7 +- use-cases/computer_vision/deploy_model.py | 4 +- use-cases/computer_vision/split_data.py | 7 +- use-cases/customer_churn/preprocessing.py | 9 +- .../customer_churn/preprocessing_predw.py | 9 +- use-cases/predmaint/demo_helpers.py | 1 + .../data_wrangler_assets/preprocessing_3.py | 12 +- use-cases/retail_recommend/deploy.py | 4 +- use-cases/retail_recommend/preprocessing.py | 12 +- utils/datasets.py | 2 +- 623 files changed, 2849 insertions(+), 2883 deletions(-) diff --git a/advanced_functionality/autogluon-tabular/container-training/inference.py b/advanced_functionality/autogluon-tabular/container-training/inference.py index 88ea754483..5aa86bdbe1 100644 --- a/advanced_functionality/autogluon-tabular/container-training/inference.py +++ b/advanced_functionality/autogluon-tabular/container-training/inference.py @@ -1,27 +1,28 @@ -import sys -import os import argparse -import logging -import warnings -import time +import copy import json +import logging +import os import subprocess -import copy +import sys +import time +import warnings warnings.filterwarnings("ignore", category=FutureWarning) -import numpy as np -import pandas as pd import pickle +from collections import Counter from io import StringIO -from timeit import default_timer as timer from itertools import islice -from collections import Counter +from timeit import default_timer as timer + +import numpy as np +import pandas as pd with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) + from autogluon.tabular import TabularDataset, TabularPredictor from prettytable import PrettyTable - from autogluon.tabular import TabularPredictor, TabularDataset def make_str_table(df): diff --git a/advanced_functionality/autogluon-tabular/container-training/train.py b/advanced_functionality/autogluon-tabular/container-training/train.py index bc4e7dd9f1..0c19564996 100644 --- a/advanced_functionality/autogluon-tabular/container-training/train.py +++ b/advanced_functionality/autogluon-tabular/container-training/train.py @@ -1,23 +1,23 @@ -import ast import argparse +import ast +import glob +import json import logging -import warnings import os -import json -import glob +import pickle +import shutil import subprocess import sys -import boto3 -import pickle -import pandas as pd +import warnings from collections import Counter from timeit import default_timer as timer -import numpy as np -import seaborn as sns +import boto3 import matplotlib.pyplot as plt -import shutil import networkx as nx +import numpy as np +import pandas as pd +import seaborn as sns logging.basicConfig(level=logging.DEBUG) logging.info(subprocess.call("ls -lR /opt/ml/input".split())) @@ -29,12 +29,10 @@ with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) - from prettytable import PrettyTable import autogluon as ag - - from autogluon.tabular import TabularDataset, TabularPredictor - from autogluon.core.constants import BINARY, MULTICLASS, REGRESSION, SOFTCLASS + from autogluon.tabular import TabularDataset, TabularPredictor + from prettytable import PrettyTable # print(f'DEBUG AutoGluon version : {ag.__version__}') @@ -74,11 +72,11 @@ def format_for_print(df): def get_roc_auc(y_test_true, y_test_pred, labels, class_labels_internal, model_output_dir): - from sklearn.preprocessing import label_binarize - from sklearn.metrics import roc_curve, auc - from itertools import cycle + from sklearn.metrics import auc, roc_curve + from sklearn.preprocessing import label_binarize + y_test_true_binalized = label_binarize(y_test_true, classes=labels) if len(labels) == 2: diff --git a/advanced_functionality/autogluon-tabular/utils/ag_utils.py b/advanced_functionality/autogluon-tabular/utils/ag_utils.py index 461c62b6fe..1f9678bf2a 100644 --- a/advanced_functionality/autogluon-tabular/utils/ag_utils.py +++ b/advanced_functionality/autogluon-tabular/utils/ag_utils.py @@ -1,13 +1,11 @@ -from IPython.display import display, IFrame -from ipywidgets import interact, Image, VBox, HTML, GridspecLayout, Layout, widgets -import ipywidgets as ipyw - -import boto3 - import os import tarfile +import boto3 +import ipywidgets as ipyw import pandas as pd +from IPython.display import IFrame, display +from ipywidgets import HTML, GridspecLayout, Image, Layout, VBox, interact, widgets def search_training_jobs(job_tag_name, job_tag_value): diff --git a/advanced_functionality/custom-training-containers/basic-training-container/docker/code/main.py b/advanced_functionality/custom-training-containers/basic-training-container/docker/code/main.py index 03ac53dca1..5884a6ee14 100644 --- a/advanced_functionality/custom-training-containers/basic-training-container/docker/code/main.py +++ b/advanced_functionality/custom-training-containers/basic-training-container/docker/code/main.py @@ -1,16 +1,16 @@ from __future__ import absolute_import +import os import sys import time -import os -from utils import ExitSignalHandler from utils import ( - write_failure_file, - print_json_object, + ExitSignalHandler, load_json_object, - save_model_artifacts, print_files_in_path, + print_json_object, + save_model_artifacts, + write_failure_file, ) hyperparameters_file_path = "/opt/ml/input/config/hyperparameters.json" diff --git a/advanced_functionality/custom-training-containers/basic-training-container/docker/code/utils.py b/advanced_functionality/custom-training-containers/basic-training-container/docker/code/utils.py index 4a038130eb..3478a4441f 100644 --- a/advanced_functionality/custom-training-containers/basic-training-container/docker/code/utils.py +++ b/advanced_functionality/custom-training-containers/basic-training-container/docker/code/utils.py @@ -1,7 +1,7 @@ -import signal -import pprint import json import os +import pprint +import signal from os import path diff --git a/advanced_functionality/custom-training-containers/framework-container/notebook/source_dir/train.py b/advanced_functionality/custom-training-containers/framework-container/notebook/source_dir/train.py index 9019a4e9a5..a3f00fc282 100644 --- a/advanced_functionality/custom-training-containers/framework-container/notebook/source_dir/train.py +++ b/advanced_functionality/custom-training-containers/framework-container/notebook/source_dir/train.py @@ -1,11 +1,11 @@ from __future__ import absolute_import +import argparse +import os import sys import time -import os -import argparse -from utils import save_model_artifacts, print_files_in_path +from utils import print_files_in_path, save_model_artifacts def train(hp1, hp2, hp3, train_channel, validation_channel): diff --git a/advanced_functionality/custom-training-containers/framework-container/package/setup.py b/advanced_functionality/custom-training-containers/framework-container/package/setup.py index 864b9807ac..5f7a2fcef7 100644 --- a/advanced_functionality/custom-training-containers/framework-container/package/setup.py +++ b/advanced_functionality/custom-training-containers/framework-container/package/setup.py @@ -1,9 +1,8 @@ from __future__ import absolute_import -from glob import glob import os -from os.path import basename -from os.path import splitext +from glob import glob +from os.path import basename, splitext from setuptools import find_packages, setup diff --git a/advanced_functionality/custom-training-containers/framework-container/package/src/custom_framework_training/training.py b/advanced_functionality/custom-training-containers/framework-container/package/src/custom_framework_training/training.py index 5ea22d9cd6..963e91d1c6 100644 --- a/advanced_functionality/custom-training-containers/framework-container/package/src/custom_framework_training/training.py +++ b/advanced_functionality/custom-training-containers/framework-container/package/src/custom_framework_training/training.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import logging + from sagemaker_training import entry_point, environment logger = logging.getLogger(__name__) diff --git a/advanced_functionality/custom-training-containers/script-mode-container-2/notebook/source_dir/train.py b/advanced_functionality/custom-training-containers/script-mode-container-2/notebook/source_dir/train.py index 9019a4e9a5..a3f00fc282 100644 --- a/advanced_functionality/custom-training-containers/script-mode-container-2/notebook/source_dir/train.py +++ b/advanced_functionality/custom-training-containers/script-mode-container-2/notebook/source_dir/train.py @@ -1,11 +1,11 @@ from __future__ import absolute_import +import argparse +import os import sys import time -import os -import argparse -from utils import save_model_artifacts, print_files_in_path +from utils import print_files_in_path, save_model_artifacts def train(hp1, hp2, hp3, train_channel, validation_channel): diff --git a/advanced_functionality/custom-training-containers/script-mode-container/docker/code/train.py b/advanced_functionality/custom-training-containers/script-mode-container/docker/code/train.py index 9019a4e9a5..a3f00fc282 100644 --- a/advanced_functionality/custom-training-containers/script-mode-container/docker/code/train.py +++ b/advanced_functionality/custom-training-containers/script-mode-container/docker/code/train.py @@ -1,11 +1,11 @@ from __future__ import absolute_import +import argparse +import os import sys import time -import os -import argparse -from utils import save_model_artifacts, print_files_in_path +from utils import print_files_in_path, save_model_artifacts def train(hp1, hp2, hp3, train_channel, validation_channel): diff --git a/advanced_functionality/distributed_tensorflow_mask_rcnn/aws-mask-rcnn.py b/advanced_functionality/distributed_tensorflow_mask_rcnn/aws-mask-rcnn.py index 4425efa713..4f5eb7c08b 100644 --- a/advanced_functionality/distributed_tensorflow_mask_rcnn/aws-mask-rcnn.py +++ b/advanced_functionality/distributed_tensorflow_mask_rcnn/aws-mask-rcnn.py @@ -15,15 +15,15 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import glob import json import os import shutil +import signal +import socket import subprocess import sys import time -import signal -import socket -import glob def copy_files(src, dest): diff --git a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-optimized-script-mode/resources/train.py b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-optimized-script-mode/resources/train.py index 01b300c4c2..95341b0f91 100644 --- a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-optimized-script-mode/resources/train.py +++ b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-optimized-script-mode/resources/train.py @@ -1,13 +1,12 @@ +import glob import json import os import shutil +import signal +import socket import subprocess import sys import time -import signal -import socket -import glob - from contextlib import contextmanager diff --git a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-optimized/resources/train.py b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-optimized/resources/train.py index 01b300c4c2..95341b0f91 100644 --- a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-optimized/resources/train.py +++ b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-optimized/resources/train.py @@ -1,13 +1,12 @@ +import glob import json import os import shutil +import signal +import socket import subprocess import sys import time -import signal -import socket -import glob - from contextlib import contextmanager diff --git a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-script-mode/resources/train.py b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-script-mode/resources/train.py index c1e28370dd..661e941384 100644 --- a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-script-mode/resources/train.py +++ b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-script-mode/resources/train.py @@ -1,13 +1,12 @@ +import glob import json import os import shutil +import signal +import socket import subprocess import sys import time -import signal -import socket -import glob - from contextlib import contextmanager diff --git a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/resources/predict.py b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/resources/predict.py index 9ee136b2b0..bdc5454929 100644 --- a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/resources/predict.py +++ b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/resources/predict.py @@ -1,20 +1,21 @@ -import os, glob +import glob +import os import sys sys.path.append("/mask-rcnn-tensorflow/MaskRCNN") -from model.generalized_rcnn import ResNetFPNModel -from config import finalize_configs, config as cfg -from dataset import DetectionDataset +from itertools import groupby +from threading import Lock +import cv2 +import numpy as np +from config import config as cfg +from config import finalize_configs +from dataset import DetectionDataset +from model.generalized_rcnn import ResNetFPNModel from tensorpack.predict.base import OfflinePredictor -from tensorpack.tfutils.sessinit import get_model_loader from tensorpack.predict.config import PredictConfig - -import numpy as np -import cv2 -from itertools import groupby -from threading import Lock +from tensorpack.tfutils.sessinit import get_model_loader class MaskRCNNService: @@ -25,7 +26,7 @@ class MaskRCNNService: # class method to load trained model and create an offline predictor @classmethod def get_predictor(cls): - """ load trained model""" + """load trained model""" with cls.lock: # check if model is already loaded @@ -172,11 +173,10 @@ def binary_mask_to_rle(cls, binary_mask): # create predictor MaskRCNNService.get_predictor() -import json -from flask import Flask -from flask import request -from flask import Response import base64 +import json + +from flask import Flask, Response, request app = Flask(__name__) diff --git a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/resources/serve.py b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/resources/serve.py index 1210f7312e..22c3c98a82 100644 --- a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/resources/serve.py +++ b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving-optimized/resources/serve.py @@ -12,12 +12,12 @@ # timeout MODEL_SERVER_TIMEOUT 70 seconds from __future__ import print_function + import os import signal import subprocess import sys - model_server_timeout = os.environ.get("MODEL_SERVER_TIMEOUT", 70) model_server_workers = int(os.environ.get("MODEL_SERVER_WORKERS", 1)) diff --git a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving/resources/predict.py b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving/resources/predict.py index bda7e0d12a..d0b978cc2b 100644 --- a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving/resources/predict.py +++ b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving/resources/predict.py @@ -1,21 +1,22 @@ -import os, glob +import glob +import os import sys sys.path.append("/tensorpack/examples/FasterRCNN") -from modeling.generalized_rcnn import ResNetFPNModel -from config import finalize_configs, config as cfg -from eval import predict_image, DetectionResult -from dataset import register_coco +from itertools import groupby +from threading import Lock +import cv2 +import numpy as np +from config import config as cfg +from config import finalize_configs +from dataset import register_coco +from eval import DetectionResult, predict_image +from modeling.generalized_rcnn import ResNetFPNModel from tensorpack.predict.base import OfflinePredictor -from tensorpack.tfutils.sessinit import get_model_loader from tensorpack.predict.config import PredictConfig - -import numpy as np -import cv2 -from itertools import groupby -from threading import Lock +from tensorpack.tfutils.sessinit import get_model_loader class MaskRCNNService: @@ -26,7 +27,7 @@ class MaskRCNNService: # class method to load trained model and create an offline predictor @classmethod def get_predictor(cls): - """ load trained model""" + """load trained model""" with cls.lock: # check if model is already loaded @@ -139,11 +140,10 @@ def binary_mask_to_rle(cls, binary_mask): # create predictor MaskRCNNService.get_predictor() -import json -from flask import Flask -from flask import request -from flask import Response import base64 +import json + +from flask import Flask, Response, request app = Flask(__name__) diff --git a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving/resources/serve.py b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving/resources/serve.py index 728809c204..22c3c98a82 100644 --- a/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving/resources/serve.py +++ b/advanced_functionality/distributed_tensorflow_mask_rcnn/container-serving/resources/serve.py @@ -12,6 +12,7 @@ # timeout MODEL_SERVER_TIMEOUT 70 seconds from __future__ import print_function + import os import signal import subprocess diff --git a/advanced_functionality/distributed_tensorflow_mask_rcnn/container/resources/train.py b/advanced_functionality/distributed_tensorflow_mask_rcnn/container/resources/train.py index c1e28370dd..661e941384 100644 --- a/advanced_functionality/distributed_tensorflow_mask_rcnn/container/resources/train.py +++ b/advanced_functionality/distributed_tensorflow_mask_rcnn/container/resources/train.py @@ -1,13 +1,12 @@ +import glob import json import os import shutil +import signal +import socket import subprocess import sys import time -import signal -import socket -import glob - from contextlib import contextmanager diff --git a/advanced_functionality/distributed_tensorflow_mask_rcnn/tensorpack-mask-rcnn.py b/advanced_functionality/distributed_tensorflow_mask_rcnn/tensorpack-mask-rcnn.py index fc91d843a9..5b94d7142c 100644 --- a/advanced_functionality/distributed_tensorflow_mask_rcnn/tensorpack-mask-rcnn.py +++ b/advanced_functionality/distributed_tensorflow_mask_rcnn/tensorpack-mask-rcnn.py @@ -15,15 +15,15 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ +import glob import json import os import shutil +import signal +import socket import subprocess import sys import time -import signal -import socket -import glob def copy_files(src, dest): diff --git a/advanced_functionality/fairseq_translation/fairseq/distributed_train.py b/advanced_functionality/fairseq_translation/fairseq/distributed_train.py index f22cc91f9b..3b53025984 100644 --- a/advanced_functionality/fairseq_translation/fairseq/distributed_train.py +++ b/advanced_functionality/fairseq_translation/fairseq/distributed_train.py @@ -6,17 +6,15 @@ # the root directory of this source tree. An additional grant of patent rights # can be found in the PATENTS file in the same directory. +import json import os import socket import subprocess -import json -from train_driver import main as single_process_main +import torch from fairseq import distributed_utils, options - from multiprocessing_train import ErrorHandler - -import torch +from train_driver import main as single_process_main def run(args, error_queue): diff --git a/advanced_functionality/fairseq_translation/fairseq/predictor.py b/advanced_functionality/fairseq_translation/fairseq/predictor.py index e1f353c240..7b1089b85a 100644 --- a/advanced_functionality/fairseq_translation/fairseq/predictor.py +++ b/advanced_functionality/fairseq_translation/fairseq/predictor.py @@ -1,8 +1,7 @@ -from sagemaker_translate import model_fn, input_fn, output_fn, predict_fn - -import flask import os +import flask +from sagemaker_translate import input_fn, model_fn, output_fn, predict_fn prefix = "/opt/ml/" model_path = os.path.join(prefix, "model") diff --git a/advanced_functionality/fairseq_translation/fairseq/sagemaker_translate.py b/advanced_functionality/fairseq_translation/fairseq/sagemaker_translate.py index 7c83301f33..b56e0d5cd5 100644 --- a/advanced_functionality/fairseq_translation/fairseq/sagemaker_translate.py +++ b/advanced_functionality/fairseq_translation/fairseq/sagemaker_translate.py @@ -9,20 +9,18 @@ Translate raw text with a trained model. Batches data on-the-fly. """ -from collections import namedtuple -import numpy as np -import sys -import os -import logging -import json import copy +import json +import logging +import os +import sys +from collections import namedtuple +import numpy as np import torch - from fairseq import data, options, tasks, tokenizer, utils from fairseq.sequence_generator import SequenceGenerator - Batch = namedtuple("Batch", "srcs tokens lengths") Translation = namedtuple("Translation", "src_str hypos pos_scores alignments") diff --git a/advanced_functionality/fairseq_translation/fairseq/serve b/advanced_functionality/fairseq_translation/fairseq/serve index 328b986b43..0b6552a1a7 100644 --- a/advanced_functionality/fairseq_translation/fairseq/serve +++ b/advanced_functionality/fairseq_translation/fairseq/serve @@ -14,6 +14,7 @@ # timeout MODEL_SERVER_TIMEOUT 60 seconds from __future__ import print_function + import multiprocessing import os import signal diff --git a/advanced_functionality/fairseq_translation/fairseq/train b/advanced_functionality/fairseq_translation/fairseq/train index 63511dec2f..9b4cc5150b 100644 --- a/advanced_functionality/fairseq_translation/fairseq/train +++ b/advanced_functionality/fairseq_translation/fairseq/train @@ -1,22 +1,18 @@ #!/opt/conda/bin/python from __future__ import print_function -import sys - -import os +import copy import json +import os import pickle +import shutil import sys import traceback -import copy -import shutil from glob import glob from fairseq import distributed_utils, options - from train_driver import main - # These are the paths to where SageMaker mounts interesting things in your container. prefix = '/opt/ml/' diff --git a/advanced_functionality/fairseq_translation/fairseq/train_driver.py b/advanced_functionality/fairseq_translation/fairseq/train_driver.py index a9f55b833f..0f903043dd 100644 --- a/advanced_functionality/fairseq_translation/fairseq/train_driver.py +++ b/advanced_functionality/fairseq_translation/fairseq/train_driver.py @@ -14,14 +14,14 @@ import collections import itertools -import os import math -import torch +import os +import torch from fairseq import distributed_utils, options, progress_bar, tasks, utils from fairseq.data import iterators -from fairseq.trainer import Trainer from fairseq.meters import AverageMeter, StopwatchMeter +from fairseq.trainer import Trainer def main(args): diff --git a/advanced_functionality/fastai_oxford_pets/source/pets.py b/advanced_functionality/fastai_oxford_pets/source/pets.py index 4b542306b3..da47763ef1 100644 --- a/advanced_functionality/fastai_oxford_pets/source/pets.py +++ b/advanced_functionality/fastai_oxford_pets/source/pets.py @@ -1,13 +1,12 @@ import argparse +import glob +import io import logging -import sagemaker_containers -import requests - import os -import io -import glob import time +import requests +import sagemaker_containers from fastai.vision import * logger = logging.getLogger(__name__) diff --git a/advanced_functionality/inference_pipeline_sparkml_blazingtext_dbpedia/dbpedia_processing.py b/advanced_functionality/inference_pipeline_sparkml_blazingtext_dbpedia/dbpedia_processing.py index 13a336a754..6fb5096e72 100644 --- a/advanced_functionality/inference_pipeline_sparkml_blazingtext_dbpedia/dbpedia_processing.py +++ b/advanced_functionality/inference_pipeline_sparkml_blazingtext_dbpedia/dbpedia_processing.py @@ -1,23 +1,19 @@ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals -import time -import sys +import csv import os import shutil -import csv -import boto3 - -from awsglue.utils import getResolvedOptions +import sys +import time +import boto3 import pyspark -from pyspark.sql import SparkSession -from pyspark.sql.types import IntegerType, StructField, StructType, StringType +from awsglue.utils import getResolvedOptions +from mleap.pyspark.spark_support import SimpleSparkSerializer from pyspark.ml.feature import Tokenizer +from pyspark.sql import SparkSession from pyspark.sql.functions import * -from mleap.pyspark.spark_support import SimpleSparkSerializer - -from awsglue.utils import getResolvedOptions +from pyspark.sql.types import IntegerType, StringType, StructField, StructType def csv_line(data): diff --git a/advanced_functionality/inference_pipeline_sparkml_xgboost_abalone/abalone_processing.py b/advanced_functionality/inference_pipeline_sparkml_xgboost_abalone/abalone_processing.py index 0c7aac9090..535cdbc1ff 100644 --- a/advanced_functionality/inference_pipeline_sparkml_xgboost_abalone/abalone_processing.py +++ b/advanced_functionality/inference_pipeline_sparkml_xgboost_abalone/abalone_processing.py @@ -1,22 +1,20 @@ -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import print_function, unicode_literals -import time -import sys +import csv import os import shutil -import csv -import boto3 - -from awsglue.utils import getResolvedOptions +import sys +import time +import boto3 import pyspark -from pyspark.sql import SparkSession +from awsglue.utils import getResolvedOptions +from mleap.pyspark.spark_support import SimpleSparkSerializer from pyspark.ml import Pipeline -from pyspark.sql.types import StructField, StructType, StringType, DoubleType -from pyspark.ml.feature import StringIndexer, VectorIndexer, OneHotEncoder, VectorAssembler +from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler, VectorIndexer +from pyspark.sql import SparkSession from pyspark.sql.functions import * -from mleap.pyspark.spark_support import SimpleSparkSerializer +from pyspark.sql.types import DoubleType, StringType, StructField, StructType def csv_line(data): diff --git a/advanced_functionality/inference_pipeline_sparkml_xgboost_car_evaluation/preprocessor.py b/advanced_functionality/inference_pipeline_sparkml_xgboost_car_evaluation/preprocessor.py index 870e9cd227..4f955002a3 100644 --- a/advanced_functionality/inference_pipeline_sparkml_xgboost_car_evaluation/preprocessor.py +++ b/advanced_functionality/inference_pipeline_sparkml_xgboost_car_evaluation/preprocessor.py @@ -1,27 +1,26 @@ from __future__ import print_function -import time -import sys +import csv import os import shutil -import csv -import boto3 - -from awsglue.utils import getResolvedOptions +import sys +import time +import boto3 import pyspark -from pyspark.sql import SparkSession +from awsglue.utils import getResolvedOptions +from mleap.pyspark.spark_support import SimpleSparkSerializer from pyspark.ml import Pipeline +from pyspark.ml.evaluation import MulticlassClassificationEvaluator from pyspark.ml.feature import ( - StringIndexer, - VectorIndexer, + IndexToString, OneHotEncoder, + StringIndexer, VectorAssembler, - IndexToString, + VectorIndexer, ) -from pyspark.ml.evaluation import MulticlassClassificationEvaluator +from pyspark.sql import SparkSession from pyspark.sql.functions import * -from mleap.pyspark.spark_support import SimpleSparkSerializer def toCSVLine(data): diff --git a/advanced_functionality/jax_bring_your_own/sagemaker_jax.py b/advanced_functionality/jax_bring_your_own/sagemaker_jax.py index 6eaa5b31a2..6b8da276b3 100644 --- a/advanced_functionality/jax_bring_your_own/sagemaker_jax.py +++ b/advanced_functionality/jax_bring_your_own/sagemaker_jax.py @@ -14,8 +14,8 @@ Custom Framework Estimator for JAX """ from sagemaker.estimator import Framework -from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT from sagemaker.tensorflow.model import TensorFlowModel +from sagemaker.vpc_utils import VPC_CONFIG_DEFAULT class JaxEstimator(Framework): diff --git a/advanced_functionality/jax_bring_your_own/training_scripts/train_jax.py b/advanced_functionality/jax_bring_your_own/training_scripts/train_jax.py index 4156f20a69..b8b213c3ef 100644 --- a/advanced_functionality/jax_bring_your_own/training_scripts/train_jax.py +++ b/advanced_functionality/jax_bring_your_own/training_scripts/train_jax.py @@ -13,9 +13,9 @@ """ Train JAX model and serialize as TF SavedModel """ -import time import argparse import functools +import time import jax import jax.numpy as jnp diff --git a/advanced_functionality/jax_bring_your_own/training_scripts/train_trax.py b/advanced_functionality/jax_bring_your_own/training_scripts/train_trax.py index d6b84daeaa..81eae29ac7 100644 --- a/advanced_functionality/jax_bring_your_own/training_scripts/train_trax.py +++ b/advanced_functionality/jax_bring_your_own/training_scripts/train_trax.py @@ -15,8 +15,8 @@ """ import argparse -import trax import tensorflow as tf +import trax from trax import layers as tl from trax.supervised import training diff --git a/advanced_functionality/multi_model_bring_your_own/container/dockerd-entrypoint.py b/advanced_functionality/multi_model_bring_your_own/container/dockerd-entrypoint.py index 540733c1e7..e54a6ac074 100644 --- a/advanced_functionality/multi_model_bring_your_own/container/dockerd-entrypoint.py +++ b/advanced_functionality/multi_model_bring_your_own/container/dockerd-entrypoint.py @@ -1,9 +1,10 @@ +import os +import shlex import subprocess import sys -import shlex -import os -from retrying import retry from subprocess import CalledProcessError + +from retrying import retry from sagemaker_inference import model_server diff --git a/advanced_functionality/multi_model_bring_your_own/container/model_handler.py b/advanced_functionality/multi_model_bring_your_own/container/model_handler.py index a10d3a2dd3..bdae697ec0 100644 --- a/advanced_functionality/multi_model_bring_your_own/container/model_handler.py +++ b/advanced_functionality/multi_model_bring_your_own/container/model_handler.py @@ -1,12 +1,12 @@ """ ModelHandler defines an example model handler for load and inference requests for MXNet CPU models """ -from collections import namedtuple import glob import json import logging import os import re +from collections import namedtuple import mxnet as mx import numpy as np diff --git a/advanced_functionality/multi_model_linear_learner_home_value/sklearn_preprocessor.py b/advanced_functionality/multi_model_linear_learner_home_value/sklearn_preprocessor.py index d85be55fdd..7e098139ec 100644 --- a/advanced_functionality/multi_model_linear_learner_home_value/sklearn_preprocessor.py +++ b/advanced_functionality/multi_model_linear_learner_home_value/sklearn_preprocessor.py @@ -1,23 +1,16 @@ from __future__ import print_function -import time -import sys -from io import StringIO -import os -import shutil - import argparse import csv import json +import os +import shutil +import sys +import time +from io import StringIO + import numpy as np import pandas as pd - -from sklearn.compose import ColumnTransformer -from sklearn.externals import joblib -from sklearn.impute import SimpleImputer -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import Binarizer, StandardScaler, OneHotEncoder - from sagemaker_containers.beta.framework import ( content_types, encoders, @@ -26,6 +19,11 @@ transformer, worker, ) +from sklearn.compose import ColumnTransformer +from sklearn.externals import joblib +from sklearn.impute import SimpleImputer +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import Binarizer, OneHotEncoder, StandardScaler # Since we get a headerless CSV file we specify the column names here. feature_columns_names = [ diff --git a/advanced_functionality/multi_model_pytorch/code/inference.py b/advanced_functionality/multi_model_pytorch/code/inference.py index 1af56366fc..3313ab8b85 100644 --- a/advanced_functionality/multi_model_pytorch/code/inference.py +++ b/advanced_functionality/multi_model_pytorch/code/inference.py @@ -11,7 +11,6 @@ # Local Dependencies: from model import MNISTNet - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) diff --git a/advanced_functionality/multi_model_pytorch/code/train.py b/advanced_functionality/multi_model_pytorch/code/train.py index 1b98d64fe8..12a759afa8 100644 --- a/advanced_functionality/multi_model_pytorch/code/train.py +++ b/advanced_functionality/multi_model_pytorch/code/train.py @@ -1,6 +1,5 @@ # Python Built-Ins: import argparse -from distutils.dir_util import copy_tree import gzip import json import logging @@ -8,22 +7,22 @@ import shutil import subprocess import sys +from distutils.dir_util import copy_tree from tempfile import TemporaryDirectory # External Dependencies: import numpy as np -from packaging import version as pkgversion -from sagemaker_pytorch_serving_container import handler_service as default_handler_service import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim -from torch.utils.data import Dataset, DataLoader +from model import MNISTNet +from packaging import version as pkgversion +from sagemaker_pytorch_serving_container import handler_service as default_handler_service +from torch.utils.data import DataLoader, Dataset # Local Dependencies: from inference import * -from model import MNISTNet - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/advanced_functionality/mxnet_mnist_byom/mnist.py b/advanced_functionality/mxnet_mnist_byom/mnist.py index 172fb18121..a45e9c519d 100644 --- a/advanced_functionality/mxnet_mnist_byom/mnist.py +++ b/advanced_functionality/mxnet_mnist_byom/mnist.py @@ -1,10 +1,10 @@ +import gzip import logging +import os +import struct -import gzip import mxnet as mx import numpy as np -import os -import struct def find_file(root_path, file_name): diff --git a/advanced_functionality/pytorch_extending_our_containers/container/cifar10/cifar10.py b/advanced_functionality/pytorch_extending_our_containers/container/cifar10/cifar10.py index c711351a6a..86fe31cf86 100644 --- a/advanced_functionality/pytorch_extending_our_containers/container/cifar10/cifar10.py +++ b/advanced_functionality/pytorch_extending_our_containers/container/cifar10/cifar10.py @@ -10,15 +10,15 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import ast import argparse +import ast import logging - import os import torch import torch.distributed as dist import torch.nn as nn +import torch.nn.functional as F import torch.nn.parallel import torch.optim import torch.utils.data @@ -26,7 +26,6 @@ import torchvision import torchvision.models import torchvision.transforms as transforms -import torch.nn.functional as F logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/advanced_functionality/pytorch_extending_our_containers/utils/utils_cifar.py b/advanced_functionality/pytorch_extending_our_containers/utils/utils_cifar.py index 0344945c52..03540da788 100644 --- a/advanced_functionality/pytorch_extending_our_containers/utils/utils_cifar.py +++ b/advanced_functionality/pytorch_extending_our_containers/utils/utils_cifar.py @@ -10,11 +10,11 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import matplotlib.pyplot as plt +import numpy as np import torch import torchvision import torchvision.transforms as transforms -import matplotlib.pyplot as plt -import numpy as np classes = ("plane", "car", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck") diff --git a/advanced_functionality/scikit_bring_your_own/container/decision_trees/predictor.py b/advanced_functionality/scikit_bring_your_own/container/decision_trees/predictor.py index 3df7e29036..d779e47763 100644 --- a/advanced_functionality/scikit_bring_your_own/container/decision_trees/predictor.py +++ b/advanced_functionality/scikit_bring_your_own/container/decision_trees/predictor.py @@ -3,16 +3,15 @@ from __future__ import print_function -import os +import io import json +import os import pickle -import io -import sys import signal +import sys import traceback import flask - import pandas as pd prefix = "/opt/ml/" diff --git a/advanced_functionality/scikit_bring_your_own/container/decision_trees/train b/advanced_functionality/scikit_bring_your_own/container/decision_trees/train index c3b1de15f6..8654139ed8 100755 --- a/advanced_functionality/scikit_bring_your_own/container/decision_trees/train +++ b/advanced_functionality/scikit_bring_your_own/container/decision_trees/train @@ -6,14 +6,13 @@ from __future__ import print_function -import os import json +import os import pickle import sys import traceback import pandas as pd - from sklearn import tree # These are the paths to where SageMaker mounts interesting things in your container. diff --git a/advanced_functionality/tensorflow_bring_your_own/container/cifar10/cifar10.py b/advanced_functionality/tensorflow_bring_your_own/container/cifar10/cifar10.py index e0e851b476..a79d83db14 100644 --- a/advanced_functionality/tensorflow_bring_your_own/container/cifar10/cifar10.py +++ b/advanced_functionality/tensorflow_bring_your_own/container/cifar10/cifar10.py @@ -10,17 +10,14 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import argparse import functools import os -import tensorflow as tf - import resnet_model +import tensorflow as tf INPUT_TENSOR_NAME = "inputs" SIGNATURE_NAME = "serving_default" diff --git a/advanced_functionality/tensorflow_bring_your_own/container/cifar10/resnet_model.py b/advanced_functionality/tensorflow_bring_your_own/container/cifar10/resnet_model.py index 6985fcf985..81f456d1dc 100644 --- a/advanced_functionality/tensorflow_bring_your_own/container/cifar10/resnet_model.py +++ b/advanced_functionality/tensorflow_bring_your_own/container/cifar10/resnet_model.py @@ -29,9 +29,7 @@ 'v1' variant in [1] is the use of batch normalization before every weight layer rather than after. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf diff --git a/advanced_functionality/tensorflow_bring_your_own/container/cifar10/train b/advanced_functionality/tensorflow_bring_your_own/container/cifar10/train index 553cd7217f..1c73ca6719 100755 --- a/advanced_functionality/tensorflow_bring_your_own/container/cifar10/train +++ b/advanced_functionality/tensorflow_bring_your_own/container/cifar10/train @@ -19,10 +19,10 @@ from __future__ import print_function -import os import json -import sys +import os import subprocess +import sys import traceback # These are the paths to where SageMaker mounts interesting things in your container. diff --git a/advanced_functionality/tensorflow_bring_your_own/utils/generate_cifar10_tfrecords.py b/advanced_functionality/tensorflow_bring_your_own/utils/generate_cifar10_tfrecords.py index d466f4e587..3476c3bea2 100644 --- a/advanced_functionality/tensorflow_bring_your_own/utils/generate_cifar10_tfrecords.py +++ b/advanced_functionality/tensorflow_bring_your_own/utils/generate_cifar10_tfrecords.py @@ -16,9 +16,7 @@ python version of the CIFAR-10 dataset downloaded from https://www.cs.toronto.edu/~kriz/cifar.html. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import argparse import os @@ -26,12 +24,12 @@ import sys import tarfile +import tensorflow as tf +from IPython.display import display +from ipywidgets import FloatProgress from six.moves import cPickle as pickle -from six.moves import urllib from six.moves import xrange # pylint: disable=redefined-builtin -from ipywidgets import FloatProgress -from IPython.display import display -import tensorflow as tf +from six.moves import urllib CIFAR_FILENAME = "cifar-10-python.tar.gz" CIFAR_DOWNLOAD_URL = "https://www.cs.toronto.edu/~kriz/" + CIFAR_FILENAME diff --git a/advanced_functionality/tensorflow_iris_byom/iris_dnn_classifier.py b/advanced_functionality/tensorflow_iris_byom/iris_dnn_classifier.py index 89027d23a7..d686f1e8fb 100644 --- a/advanced_functionality/tensorflow_iris_byom/iris_dnn_classifier.py +++ b/advanced_functionality/tensorflow_iris_byom/iris_dnn_classifier.py @@ -1,4 +1,5 @@ import os + import numpy as np import tensorflow as tf diff --git a/advanced_functionality/unity_ml_agents/src/train.py b/advanced_functionality/unity_ml_agents/src/train.py index c4ba7f7e3f..7efbdb0c84 100644 --- a/advanced_functionality/unity_ml_agents/src/train.py +++ b/advanced_functionality/unity_ml_agents/src/train.py @@ -1,12 +1,13 @@ import argparse -import numpy as np +import json import os -import tensorflow as tf -from mlagents_envs.environment import UnityEnvironment -import mlagents import subprocess + +import mlagents +import numpy as np +import tensorflow as tf import yaml -import json +from mlagents_envs.environment import UnityEnvironment def parse_args(): diff --git a/autopilot/custom-feature-selection/sklearn_feature_selection.py b/autopilot/custom-feature-selection/sklearn_feature_selection.py index 7926e69d97..e63633f754 100644 --- a/autopilot/custom-feature-selection/sklearn_feature_selection.py +++ b/autopilot/custom-feature-selection/sklearn_feature_selection.py @@ -1,23 +1,16 @@ from __future__ import print_function -import time -import sys -from io import StringIO -import os -import shutil - import argparse import csv import json +import os +import shutil +import sys +import time +from io import StringIO + import numpy as np import pandas as pd - -from sklearn.externals import joblib -from sklearn.impute import SimpleImputer -from sklearn.pipeline import Pipeline -from sklearn.svm import SVR -from sklearn.feature_selection import f_regression, mutual_info_regression, SelectKBest, RFE - from sagemaker_containers.beta.framework import ( content_types, encoders, @@ -26,6 +19,11 @@ transformer, worker, ) +from sklearn.externals import joblib +from sklearn.feature_selection import RFE, SelectKBest, f_regression, mutual_info_regression +from sklearn.impute import SimpleImputer +from sklearn.pipeline import Pipeline +from sklearn.svm import SVR label_column = "y" INPUT_FEATURES_SIZE = 100 diff --git a/aws_marketplace/creating_marketplace_products/container/decision_trees/predictor.py b/aws_marketplace/creating_marketplace_products/container/decision_trees/predictor.py index b68c85cf2c..725f626679 100644 --- a/aws_marketplace/creating_marketplace_products/container/decision_trees/predictor.py +++ b/aws_marketplace/creating_marketplace_products/container/decision_trees/predictor.py @@ -3,17 +3,16 @@ from __future__ import print_function -import os import json +import os import pickle -import StringIO -import sys import signal +import sys import traceback import flask - import pandas as pd +import StringIO prefix = "/opt/ml/" model_path = os.path.join(prefix, "model") diff --git a/aws_marketplace/creating_marketplace_products/container/decision_trees/serve b/aws_marketplace/creating_marketplace_products/container/decision_trees/serve index 6747fbde09..9eb568c778 100755 --- a/aws_marketplace/creating_marketplace_products/container/decision_trees/serve +++ b/aws_marketplace/creating_marketplace_products/container/decision_trees/serve @@ -14,6 +14,7 @@ # timeout MODEL_SERVER_TIMEOUT 60 seconds from __future__ import print_function + import multiprocessing import os import signal diff --git a/aws_marketplace/creating_marketplace_products/container/decision_trees/train b/aws_marketplace/creating_marketplace_products/container/decision_trees/train index d21246f0b7..461c7e4ece 100755 --- a/aws_marketplace/creating_marketplace_products/container/decision_trees/train +++ b/aws_marketplace/creating_marketplace_products/container/decision_trees/train @@ -6,17 +6,16 @@ from __future__ import print_function -import os import json +import os import pickle import sys import traceback import pandas as pd - +from numpy import mean from sklearn import tree from sklearn.model_selection import cross_val_score -from numpy import mean # These are the paths to where SageMaker mounts interesting things in your container. diff --git a/aws_sagemaker_studio/frameworks/keras_pipe_mode_horovod/generate_tensorboard_command.py b/aws_sagemaker_studio/frameworks/keras_pipe_mode_horovod/generate_tensorboard_command.py index 1482a9519b..d1c1074db4 100644 --- a/aws_sagemaker_studio/frameworks/keras_pipe_mode_horovod/generate_tensorboard_command.py +++ b/aws_sagemaker_studio/frameworks/keras_pipe_mode_horovod/generate_tensorboard_command.py @@ -10,9 +10,10 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -import boto3 -from datetime import datetime, timedelta import re +from datetime import datetime, timedelta + +import boto3 client = boto3.client("sagemaker") running_jobs = client.list_training_jobs(CreationTimeAfter=datetime.utcnow() - timedelta(hours=1)) diff --git a/aws_sagemaker_studio/frameworks/keras_pipe_mode_horovod/source/keras_cifar10.py b/aws_sagemaker_studio/frameworks/keras_pipe_mode_horovod/source/keras_cifar10.py index a7f768ad07..9534c2fbc4 100644 --- a/aws_sagemaker_studio/frameworks/keras_pipe_mode_horovod/source/keras_cifar10.py +++ b/aws_sagemaker_studio/frameworks/keras_pipe_mode_horovod/source/keras_cifar10.py @@ -15,24 +15,24 @@ import argparse import json import logging -import re import os +import re import keras import tensorflow as tf from keras import backend as K -from keras.callbacks import TensorBoard, ModelCheckpoint +from keras.callbacks import ModelCheckpoint, TensorBoard from keras.layers import ( Activation, + BatchNormalization, Conv2D, Dense, Dropout, Flatten, MaxPooling2D, - BatchNormalization, ) from keras.models import Sequential -from keras.optimizers import Adam, SGD, RMSprop +from keras.optimizers import SGD, Adam, RMSprop logging.getLogger().setLevel(logging.INFO) tf.logging.set_verbosity(tf.logging.INFO) diff --git a/aws_sagemaker_studio/frameworks/mxnet_gluon_sentiment/sentiment.py b/aws_sagemaker_studio/frameworks/mxnet_gluon_sentiment/sentiment.py index 686f312bd8..01de986498 100644 --- a/aws_sagemaker_studio/frameworks/mxnet_gluon_sentiment/sentiment.py +++ b/aws_sagemaker_studio/frameworks/mxnet_gluon_sentiment/sentiment.py @@ -12,8 +12,8 @@ import mxnet as mx import numpy as np -from mxnet import gluon, autograd, nd -from mxnet.io import DataIter, DataBatch, DataDesc +from mxnet import autograd, gluon, nd +from mxnet.io import DataBatch, DataDesc, DataIter logging.basicConfig(level=logging.DEBUG) diff --git a/aws_sagemaker_studio/frameworks/mxnet_onnx_ei/resnet152.py b/aws_sagemaker_studio/frameworks/mxnet_onnx_ei/resnet152.py index d5f35c17c8..2faafae329 100644 --- a/aws_sagemaker_studio/frameworks/mxnet_onnx_ei/resnet152.py +++ b/aws_sagemaker_studio/frameworks/mxnet_onnx_ei/resnet152.py @@ -2,6 +2,7 @@ import json import logging + import mxnet as mx import mxnet.contrib.onnx as onnx_mxnet import numpy as np diff --git a/aws_sagemaker_studio/getting_started/preprocessor.py b/aws_sagemaker_studio/getting_started/preprocessor.py index ed4f249f56..200d8ca9ef 100644 --- a/aws_sagemaker_studio/getting_started/preprocessor.py +++ b/aws_sagemaker_studio/getting_started/preprocessor.py @@ -1,6 +1,7 @@ import json import random + # sample preprocess_handler (to be implemented by customer) # This is a trivial example, where we demonstrate an echo preprocessor for json data # for others though, we are generating random data (real customers would not do that obviously/hopefully) diff --git a/aws_sagemaker_studio/introduction_to_amazon_algorithms/xgboost_abalone/abalone.py b/aws_sagemaker_studio/introduction_to_amazon_algorithms/xgboost_abalone/abalone.py index a2c8847c18..42a59b880e 100644 --- a/aws_sagemaker_studio/introduction_to_amazon_algorithms/xgboost_abalone/abalone.py +++ b/aws_sagemaker_studio/introduction_to_amazon_algorithms/xgboost_abalone/abalone.py @@ -16,14 +16,13 @@ import json import logging import os -import pandas as pd import pickle as pkl +import pandas as pd +import xgboost as xgb from sagemaker_containers import entry_point -from sagemaker_xgboost_container.data_utils import get_dmatrix from sagemaker_xgboost_container import distributed - -import xgboost as xgb +from sagemaker_xgboost_container.data_utils import get_dmatrix def _xgb_train(params, dtrain, evals, num_boost_round, model_dir, is_master): diff --git a/aws_sagemaker_studio/sagemaker-python-sdk/scikit_learn_iris/scikit_learn_iris.py b/aws_sagemaker_studio/sagemaker-python-sdk/scikit_learn_iris/scikit_learn_iris.py index 333db63e76..ceda952730 100644 --- a/aws_sagemaker_studio/sagemaker-python-sdk/scikit_learn_iris/scikit_learn_iris.py +++ b/aws_sagemaker_studio/sagemaker-python-sdk/scikit_learn_iris/scikit_learn_iris.py @@ -15,12 +15,11 @@ import argparse import os -import pandas as pd +import pandas as pd from sklearn import tree from sklearn.externals import joblib - if __name__ == "__main__": parser = argparse.ArgumentParser() diff --git a/aws_sagemaker_studio/sagemaker_debugger/mnist_tensor_analysis/src/mnist.py b/aws_sagemaker_studio/sagemaker_debugger/mnist_tensor_analysis/src/mnist.py index 877a565895..bffbcce401 100644 --- a/aws_sagemaker_studio/sagemaker_debugger/mnist_tensor_analysis/src/mnist.py +++ b/aws_sagemaker_studio/sagemaker_debugger/mnist_tensor_analysis/src/mnist.py @@ -1,5 +1,7 @@ # Standard Library import argparse +import os +import time # Third Party import mxnet as mx @@ -7,8 +9,6 @@ from mxnet import autograd, gluon, init from mxnet.gluon import nn from mxnet.gluon.data.vision import datasets, transforms -import os -import time def parse_args(): diff --git a/aws_sagemaker_studio/sagemaker_debugger/mnist_tensor_analysis/utils.py b/aws_sagemaker_studio/sagemaker_debugger/mnist_tensor_analysis/utils.py index 887a58d43c..adf661beec 100644 --- a/aws_sagemaker_studio/sagemaker_debugger/mnist_tensor_analysis/utils.py +++ b/aws_sagemaker_studio/sagemaker_debugger/mnist_tensor_analysis/utils.py @@ -1,6 +1,7 @@ -import numpy as np import math + import matplotlib.pyplot as plt +import numpy as np from matplotlib.animation import FuncAnimation plt.rcParams.update({"font.size": 8}) diff --git a/aws_sagemaker_studio/sagemaker_debugger/tensorflow_builtin_rule/src/mnist_byoc.py b/aws_sagemaker_studio/sagemaker_debugger/tensorflow_builtin_rule/src/mnist_byoc.py index fa527718d8..aad5ce027f 100644 --- a/aws_sagemaker_studio/sagemaker_debugger/tensorflow_builtin_rule/src/mnist_byoc.py +++ b/aws_sagemaker_studio/sagemaker_debugger/tensorflow_builtin_rule/src/mnist_byoc.py @@ -11,14 +11,13 @@ # Standard Library import argparse +import logging import random # Third Party import numpy as np -import tensorflow as tf import smdebug.tensorflow as smd - -import logging +import tensorflow as tf logging.getLogger().setLevel(logging.INFO) diff --git a/aws_sagemaker_studio/sagemaker_debugger/tensorflow_builtin_rule/src/mnist_zerocodechange.py b/aws_sagemaker_studio/sagemaker_debugger/tensorflow_builtin_rule/src/mnist_zerocodechange.py index e091dea52b..1b09760cad 100644 --- a/aws_sagemaker_studio/sagemaker_debugger/tensorflow_builtin_rule/src/mnist_zerocodechange.py +++ b/aws_sagemaker_studio/sagemaker_debugger/tensorflow_builtin_rule/src/mnist_zerocodechange.py @@ -8,14 +8,13 @@ # Standard Library import argparse +import logging import random # Third Party import numpy as np import tensorflow as tf -import logging - logging.getLogger().setLevel(logging.INFO) parser = argparse.ArgumentParser() diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/mnist.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/mnist.py index fd32806c33..2622565a6c 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/mnist.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/mnist.py @@ -1,4 +1,5 @@ import os + import tensorflow as tf from tensorflow.python.estimator.model_fn import ModeKeys as Modes @@ -110,9 +111,10 @@ def _input_fn(training_dir, training_filename, batch_size=100): def neo_preprocess(payload, content_type): + import io import logging + import numpy as np - import io logging.info("Invoking user-defined pre-processing function") @@ -132,9 +134,10 @@ def neo_preprocess(payload, content_type): ### NOTE: this function cannot use MXNet def neo_postprocess(result): + import json import logging + import numpy as np - import json logging.info("Invoking user-defined post-processing function") diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/utils.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/utils.py index cf8af1b2b1..c883d56685 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/utils.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/utils.py @@ -1,5 +1,6 @@ """Converts MNIST data to TFRecords file format with Example protos.""" import os + import tensorflow as tf diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/ssd_entry_point.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/ssd_entry_point.py index a47d035468..bfb27dc713 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/ssd_entry_point.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/ssd_entry_point.py @@ -1,9 +1,10 @@ import io -import os -import PIL.Image import json import logging +import os + import numpy as np +import PIL.Image logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -12,14 +13,13 @@ # Training methods # # ------------------------------------------------------------ # +import argparse import glob import time -import argparse import warnings + import mxnet as mx -from mxnet import nd -from mxnet import gluon -from mxnet import autograd +from mxnet import autograd, gluon, nd def parse_args(): @@ -74,7 +74,7 @@ def get_dataloader(net, data_shape, batch_size, num_workers, ctx): """Get dataloader.""" from gluoncv import data as gdata - from gluoncv.data.batchify import Tuple, Stack, Pad + from gluoncv.data.batchify import Pad, Stack, Tuple from gluoncv.data.transforms.presets.ssd import SSDDefaultTrainTransform width, height = data_shape, data_shape diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/concat_db.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/concat_db.py index 9b11085b6a..51ff2adcd3 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/concat_db.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/concat_db.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -from imdb import Imdb import random +from imdb import Imdb + class ConcatDB(Imdb): """ diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/im2rec.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/im2rec.py index d6ad10aceb..ecb8657259 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/im2rec.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/im2rec.py @@ -18,18 +18,20 @@ # under the License. from __future__ import print_function + import os import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "../python")) -import mxnet as mx -import random import argparse -import cv2 +import random import time import traceback +import cv2 +import mxnet as mx + try: import multiprocessing except ImportError: diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/imdb.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/imdb.py index b10d32aec3..c9414326a4 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/imdb.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/imdb.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -import numpy as np import os.path as osp +import numpy as np + class Imdb(object): """ diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/pascal_voc.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/pascal_voc.py index ae3d2a054a..d2b1826fef 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/pascal_voc.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/pascal_voc.py @@ -16,11 +16,13 @@ # under the License. from __future__ import print_function + import os -import numpy as np -from imdb import Imdb import xml.etree.ElementTree as ET + import cv2 +import numpy as np +from imdb import Imdb class PascalVoc(Imdb): diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/prepare_dataset.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/prepare_dataset.py index 6d6ad352cb..00f892b078 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/prepare_dataset.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/prepare_dataset.py @@ -19,14 +19,16 @@ # under the License. from __future__ import print_function -import sys, os + import argparse +import os import subprocess +import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "..")) -from pascal_voc import PascalVoc from concat_db import ConcatDB +from pascal_voc import PascalVoc def load_pascal(image_set, year, devkit_path, shuffle=False): diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/imageclassification_caltech/inference.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/imageclassification_caltech/inference.py index ae88931164..41f3cbcdda 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/imageclassification_caltech/inference.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/imageclassification_caltech/inference.py @@ -1,12 +1,13 @@ -import numpy as np +import io import json +import logging +import os + import mxnet as mx # Please make sure to import neomx import neomx # noqa: F401 -import io -import os -import logging +import numpy as np # Change the context to mx.gpu() if deploying to a GPU endpoint ctx = mx.cpu() diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py index c59619a91c..8948ed6ee7 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py @@ -4,9 +4,9 @@ import os import pickle +import neopytorch import numpy as np import torch -import neopytorch import torchvision.transforms as transforms from PIL import Image # Training container doesn't have this package diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py index 06e9f288cd..040a0bd6f1 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py @@ -4,9 +4,9 @@ import os import pickle +import neopytorch import numpy as np import torch -import neopytorch import torchvision.transforms as transforms from PIL import Image # Training container doesn't have this package diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py index 50eecb0233..de6a0819da 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py @@ -4,9 +4,9 @@ import os import pickle +import neopytorch import numpy as np import torch -import neopytorch import torchvision.transforms as transforms from PIL import Image # Training container doesn't have this package diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/mnist.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/mnist.py index 11f4d395ce..c5096c6eb8 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/mnist.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/mnist.py @@ -1,4 +1,5 @@ import os + import tensorflow as tf from tensorflow.python.estimator.model_fn import ModeKeys as Modes @@ -111,9 +112,10 @@ def _input_fn(training_dir, training_filename, batch_size=100): def neo_preprocess(payload, content_type): + import io import logging + import numpy as np - import io logging.info("Invoking user-defined pre-processing function") @@ -133,9 +135,10 @@ def neo_preprocess(payload, content_type): ### NOTE: this function cannot use MXNet def neo_postprocess(result): + import json import logging + import numpy as np - import json logging.info("Invoking user-defined post-processing function") diff --git a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/utils.py b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/utils.py index cf8af1b2b1..c883d56685 100644 --- a/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/utils.py +++ b/aws_sagemaker_studio/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/utils.py @@ -1,5 +1,6 @@ """Converts MNIST data to TFRecords file format with Example protos.""" import os + import tensorflow as tf diff --git a/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/cifar10.py b/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/cifar10.py index e0e851b476..a79d83db14 100644 --- a/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/cifar10.py +++ b/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/cifar10.py @@ -10,17 +10,14 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import argparse import functools import os -import tensorflow as tf - import resnet_model +import tensorflow as tf INPUT_TENSOR_NAME = "inputs" SIGNATURE_NAME = "serving_default" diff --git a/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/resnet_model.py b/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/resnet_model.py index 6985fcf985..81f456d1dc 100644 --- a/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/resnet_model.py +++ b/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/resnet_model.py @@ -29,9 +29,7 @@ 'v1' variant in [1] is the use of batch normalization before every weight layer rather than after. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import tensorflow as tf diff --git a/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/train b/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/train index 553cd7217f..1c73ca6719 100755 --- a/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/train +++ b/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/cifar10/train @@ -19,10 +19,10 @@ from __future__ import print_function -import os import json -import sys +import os import subprocess +import sys import traceback # These are the paths to where SageMaker mounts interesting things in your container. diff --git a/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/utils/generate_cifar10_tfrecords.py b/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/utils/generate_cifar10_tfrecords.py index d466f4e587..3476c3bea2 100644 --- a/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/utils/generate_cifar10_tfrecords.py +++ b/aws_sagemaker_studio/sagemaker_studio_image_build/tensorflow_bring_your_own/utils/generate_cifar10_tfrecords.py @@ -16,9 +16,7 @@ python version of the CIFAR-10 dataset downloaded from https://www.cs.toronto.edu/~kriz/cifar.html. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import argparse import os @@ -26,12 +24,12 @@ import sys import tarfile +import tensorflow as tf +from IPython.display import display +from ipywidgets import FloatProgress from six.moves import cPickle as pickle -from six.moves import urllib from six.moves import xrange # pylint: disable=redefined-builtin -from ipywidgets import FloatProgress -from IPython.display import display -import tensorflow as tf +from six.moves import urllib CIFAR_FILENAME = "cifar-10-python.tar.gz" CIFAR_DOWNLOAD_URL = "https://www.cs.toronto.edu/~kriz/" + CIFAR_FILENAME diff --git a/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/predictor.py b/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/predictor.py index 847c624f07..d2bb6e86d2 100644 --- a/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/predictor.py +++ b/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/predictor.py @@ -3,16 +3,15 @@ from __future__ import print_function -import os import json +import os import pickle -from io import StringIO -import sys import signal +import sys import traceback +from io import StringIO import flask - import pandas as pd import xgboost diff --git a/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/serve b/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/serve index 1269361563..604434529a 100644 --- a/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/serve +++ b/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/serve @@ -14,6 +14,7 @@ # timeout MODEL_SERVER_TIMEOUT 60 seconds from __future__ import print_function + import multiprocessing import os import signal diff --git a/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/train b/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/train index cc187d9c97..2af6234353 100644 --- a/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/train +++ b/aws_sagemaker_studio/sagemaker_studio_image_build/xgboost_bring_your_own/xgboost/train @@ -6,15 +6,15 @@ from __future__ import print_function -import os import json +import os import pickle import sys import traceback import pandas as pd -import xgboost as xgb import smdebug.xgboost as smd +import xgboost as xgb print("Libraries imported") diff --git a/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/main.py b/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/main.py index f11b6ed930..9ecba72293 100644 --- a/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/main.py +++ b/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/main.py @@ -1,44 +1,26 @@ -from __future__ import absolute_import -from __future__ import print_function -from string import Template +from __future__ import absolute_import, print_function -import sys -import time -import os +import argparse +import csv +import decimal +import json import multiprocessing +import os +import shutil import signal import subprocess -from urllib.parse import urlparse - -from utils import ExitSignalHandler -from utils import ( - write_failure_file, - print_json_object, - load_json_object, - save_model_artifacts, - print_files_in_path, -) - +import sys +import time import traceback from io import StringIO -import os -import shutil +from string import Template +from urllib.parse import urlparse -import argparse -import csv -import json +import boto3 import numpy as np import pandas as pd - -from joblib import dump, load -from sklearn.feature_extraction.text import CountVectorizer -from sklearn.preprocessing import LabelEncoder - -import boto3 -import decimal - from botocore.exceptions import ClientError - +from joblib import dump, load from sagemaker_containers.beta.framework import ( content_types, encoders, @@ -47,6 +29,17 @@ transformer, worker, ) +from sklearn.feature_extraction.text import CountVectorizer +from sklearn.preprocessing import LabelEncoder + +from utils import ( + ExitSignalHandler, + load_json_object, + print_files_in_path, + print_json_object, + save_model_artifacts, + write_failure_file, +) cpu_count = multiprocessing.cpu_count() diff --git a/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/predictor.py b/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/predictor.py index 34f44a581f..71482345bf 100644 --- a/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/predictor.py +++ b/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/predictor.py @@ -3,21 +3,22 @@ from __future__ import print_function -import os -import sys -import stat +import csv +import glob import json +import os +import random import shutil +import stat +import sys +from io import StringIO + +import boto3 import flask -from flask import Flask, jsonify, request, Response -import glob -import pandas as pd import numpy as np -import random -import csv -from io import StringIO +import pandas as pd +from flask import Flask, Response, jsonify, request from joblib import dump, load -import boto3 from sagemaker_containers.beta.framework import ( content_types, encoders, diff --git a/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/utils.py b/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/utils.py index 4a038130eb..3478a4441f 100644 --- a/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/utils.py +++ b/contrib/inference_pipeline_custom_containers/containers/postprocessor/docker/code/utils.py @@ -1,7 +1,7 @@ -import signal -import pprint import json import os +import pprint +import signal from os import path diff --git a/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/main.py b/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/main.py index 03bdb4a119..49543eec76 100644 --- a/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/main.py +++ b/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/main.py @@ -1,44 +1,25 @@ -from __future__ import absolute_import -from __future__ import print_function -from string import Template +from __future__ import absolute_import, print_function -import sys -import time +import argparse +import csv +import decimal +import json +import multiprocessing import os import shutil -import multiprocessing import signal import subprocess - -from utils import ExitSignalHandler -from utils import ( - write_failure_file, - print_json_object, - load_json_object, - save_model_artifacts, - print_files_in_path, -) - - +import sys +import time import traceback from io import StringIO -import os +from string import Template -import argparse -import csv -import json +import boto3 import numpy as np import pandas as pd - -from joblib import dump, load -from sklearn.feature_extraction.text import CountVectorizer -from sklearn.preprocessing import LabelEncoder - -import boto3 -import decimal - from botocore.exceptions import ClientError - +from joblib import dump, load from sagemaker_containers.beta.framework import ( content_types, encoders, @@ -47,6 +28,17 @@ transformer, worker, ) +from sklearn.feature_extraction.text import CountVectorizer +from sklearn.preprocessing import LabelEncoder + +from utils import ( + ExitSignalHandler, + load_json_object, + print_files_in_path, + print_json_object, + save_model_artifacts, + write_failure_file, +) hyperparameters_file_path = "/opt/ml/input/config/hyperparameters.json" inputdataconfig_file_path = "/opt/ml/input/config/inputdataconfig.json" diff --git a/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/predictor.py b/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/predictor.py index f9eb81c012..afc8e8f118 100644 --- a/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/predictor.py +++ b/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/predictor.py @@ -3,18 +3,19 @@ from __future__ import print_function -import os -import sys -import stat +import csv +import glob import json +import os import shutil +import stat +import sys +from io import StringIO + import flask -from flask import Flask, jsonify, request, make_response, Response -import glob -import pandas as pd import numpy as np -import csv -from io import StringIO +import pandas as pd +from flask import Flask, Response, jsonify, make_response, request from joblib import dump, load from sagemaker_containers.beta.framework import ( content_types, @@ -24,12 +25,13 @@ transformer, worker, ) + from utils import ( - write_failure_file, - print_json_object, load_json_object, - save_model_artifacts, print_files_in_path, + print_json_object, + save_model_artifacts, + write_failure_file, ) model_artifacts_path = "/opt/ml/model/" diff --git a/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/utils.py b/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/utils.py index 4a038130eb..3478a4441f 100644 --- a/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/utils.py +++ b/contrib/inference_pipeline_custom_containers/containers/preprocessor/docker/code/utils.py @@ -1,7 +1,7 @@ -import signal -import pprint import json import os +import pprint +import signal from os import path diff --git a/contrib/inference_pipeline_custom_containers/generate-training-data.py b/contrib/inference_pipeline_custom_containers/generate-training-data.py index 44be93d0c3..4e05b77588 100644 --- a/contrib/inference_pipeline_custom_containers/generate-training-data.py +++ b/contrib/inference_pipeline_custom_containers/generate-training-data.py @@ -1,6 +1,6 @@ import argparse -import random import csv +import random parser = argparse.ArgumentParser(description="Generate sample data") parser.add_argument("--samples", type=int, default=10000, help="Number of samples to generate") diff --git a/contrib/inference_pipeline_custom_containers/load-ddb-data.py b/contrib/inference_pipeline_custom_containers/load-ddb-data.py index 4a932a4a40..fb3666b644 100644 --- a/contrib/inference_pipeline_custom_containers/load-ddb-data.py +++ b/contrib/inference_pipeline_custom_containers/load-ddb-data.py @@ -1,5 +1,6 @@ import argparse import csv + import boto3 parser = argparse.ArgumentParser(description="Load DynamoDB data") diff --git a/end_to_end/create_dataset.py b/end_to_end/create_dataset.py index f8c8b0d51a..0d8e0f5cf1 100644 --- a/end_to_end/create_dataset.py +++ b/end_to_end/create_dataset.py @@ -1,8 +1,9 @@ +import argparse +import pathlib import time + import boto3 -import argparse import pandas as pd -import pathlib # Parse argument variables passed via the CreateDataset processing step parser = argparse.ArgumentParser() diff --git a/end_to_end/demo_helpers.py b/end_to_end/demo_helpers.py index 024f8aa9bd..f5c4be478c 100644 --- a/end_to_end/demo_helpers.py +++ b/end_to_end/demo_helpers.py @@ -1,6 +1,7 @@ -import boto3 import time +import boto3 + def delete_project_resources( sagemaker_boto_client, diff --git a/end_to_end/deploy_model.py b/end_to_end/deploy_model.py index 829672bbe5..480118f158 100644 --- a/end_to_end/deploy_model.py +++ b/end_to_end/deploy_model.py @@ -1,7 +1,7 @@ -import time -import boto3 import argparse +import time +import boto3 # Parse argument variables passed via the DeployModel processing step parser = argparse.ArgumentParser() diff --git a/end_to_end/xgboost_starter_script.py b/end_to_end/xgboost_starter_script.py index 26a1e6fbc0..9b54110524 100644 --- a/end_to_end/xgboost_starter_script.py +++ b/end_to_end/xgboost_starter_script.py @@ -1,10 +1,11 @@ +import argparse +import json import os -import sys import pickle -import xgboost as xgb -import argparse +import sys + import pandas as pd -import json +import xgboost as xgb if __name__ == "__main__": parser = argparse.ArgumentParser() diff --git a/frameworks/mxnet/code/inference.py b/frameworks/mxnet/code/inference.py index 87a9ca1b1f..09c7222ac6 100644 --- a/frameworks/mxnet/code/inference.py +++ b/frameworks/mxnet/code/inference.py @@ -15,14 +15,13 @@ from __future__ import print_function +import json import logging +import os -from mxnet import gluon import mxnet as mx - import numpy as np -import json -import os +from mxnet import gluon logging.basicConfig(level=logging.DEBUG) diff --git a/frameworks/mxnet/code/test_inference.py b/frameworks/mxnet/code/test_inference.py index e4af705149..12bd8d3f5b 100644 --- a/frameworks/mxnet/code/test_inference.py +++ b/frameworks/mxnet/code/test_inference.py @@ -13,16 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from inference import transform_fn, model_fn -import os import json +import os import shutil +import tarfile + import boto3 import botocore -import tarfile import numpy as np import sagemaker +from inference import model_fn, transform_fn + def fetch_model(model_data): """Untar the model.tar.gz object either from local file system diff --git a/frameworks/mxnet/code/test_train.py b/frameworks/mxnet/code/test_train.py index 980c421ba1..bba2094bb5 100644 --- a/frameworks/mxnet/code/test_train.py +++ b/frameworks/mxnet/code/test_train.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from train import train, parse_args - -import sys +import json import os +import sys + import boto3 -import json +from train import parse_args, train dirname = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(dirname, "config.json"), "r") as f: diff --git a/frameworks/mxnet/code/train.py b/frameworks/mxnet/code/train.py index f17435dc9a..3a6b4e70e0 100644 --- a/frameworks/mxnet/code/train.py +++ b/frameworks/mxnet/code/train.py @@ -17,17 +17,16 @@ import argparse import gzip +import json import logging import os +import time import mxnet as mx -from mxnet import gluon, autograd +import numpy as np +from mxnet import autograd, gluon from mxnet.gluon import nn from mxnet.gluon.data import Dataset -import numpy as np -import json -import time - logging.basicConfig(level=logging.DEBUG) diff --git a/frameworks/pytorch/code/test_inference.py b/frameworks/pytorch/code/test_inference.py index 5c7006b58e..45958f971f 100644 --- a/frameworks/pytorch/code/test_inference.py +++ b/frameworks/pytorch/code/test_inference.py @@ -1,13 +1,15 @@ -from inference import model_fn, input_fn, predict_fn, output_fn -import os import json +import os import shutil +import tarfile + import boto3 import botocore -import tarfile import numpy as np import sagemaker +from inference import input_fn, model_fn, output_fn, predict_fn + def fetch_model(model_data): """Untar the model.tar.gz object either from local file system diff --git a/frameworks/pytorch/code/test_train.py b/frameworks/pytorch/code/test_train.py index ca012cff1a..a850509b72 100644 --- a/frameworks/pytorch/code/test_train.py +++ b/frameworks/pytorch/code/test_train.py @@ -1,9 +1,9 @@ -from train import train, parse_args - -import sys +import json import os +import sys + import boto3 -import json +from train import parse_args, train dirname = os.path.dirname(os.path.abspath(__file__)) diff --git a/frameworks/pytorch/code/train.py b/frameworks/pytorch/code/train.py index 45772cc770..c1797d438b 100644 --- a/frameworks/pytorch/code/train.py +++ b/frameworks/pytorch/code/train.py @@ -10,8 +10,7 @@ import torch.nn as nn import torch.nn.functional as F import torch.optim as optim -from torch.utils.data import Dataset, DataLoader - +from torch.utils.data import DataLoader, Dataset logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/frameworks/tensorflow/code/test_train.py b/frameworks/tensorflow/code/test_train.py index 467adaad84..bb86fa408b 100644 --- a/frameworks/tensorflow/code/test_train.py +++ b/frameworks/tensorflow/code/test_train.py @@ -1,9 +1,9 @@ -from train import train, parse_args - -import sys +import json import os +import sys + import boto3 -import json +from train import parse_args, train dirname = os.path.dirname(os.path.abspath(__file__)) diff --git a/frameworks/tensorflow/code/train.py b/frameworks/tensorflow/code/train.py index 8f3d720465..4a3a2c5837 100644 --- a/frameworks/tensorflow/code/train.py +++ b/frameworks/tensorflow/code/train.py @@ -1,17 +1,16 @@ from __future__ import print_function import argparse +import gzip +import json import logging import os -import json -import gzip -import numpy as np import traceback +import numpy as np import tensorflow as tf -from tensorflow.keras.layers import Dense, Flatten, Conv2D from tensorflow.keras import Model - +from tensorflow.keras.layers import Conv2D, Dense, Flatten logging.basicConfig(level=logging.DEBUG) diff --git a/frameworks/utils/mnist.py b/frameworks/utils/mnist.py index dac0e2e9a5..572a5a67f6 100644 --- a/frameworks/utils/mnist.py +++ b/frameworks/utils/mnist.py @@ -1,9 +1,10 @@ -import numpy as np -from urllib import request import gzip +import json import os +from urllib import request + import boto3 -import json +import numpy as np dirname = os.path.dirname(os.path.abspath(__file__)) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/predictor.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/predictor.py index eccf83732f..dc70295e03 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/predictor.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/predictor.py @@ -16,26 +16,22 @@ from __future__ import print_function -import os import json +import logging +import os import pickle -import sys import signal +import sys import traceback -import json -import flask - -from io import StringIO, BytesIO +from io import BytesIO, StringIO -import tensorflow as tf -import pandas as pd +import boto3 +import flask import numpy as np -from tensorflow.python.keras.preprocessing.text import Tokenizer +import pandas as pd +import tensorflow as tf from tensorflow.python.keras.preprocessing.sequence import pad_sequences - -import logging - -import boto3 +from tensorflow.python.keras.preprocessing.text import Tokenizer MAX_LEN = 100 prefix = "/opt/ml/" @@ -121,7 +117,7 @@ def _dump_jsonlines_entry(prediction): @app.route("/invocations", methods=["POST"]) def transformation(): - """Do an inference on a single news headline. """ + """Do an inference on a single news headline.""" data = None if flask.request.content_type == "application/jsonlines": diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/serve b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/serve index 392f5e25ee..786e036a88 100755 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/serve +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/serve @@ -14,6 +14,7 @@ # timeout MODEL_SERVER_TIMEOUT 60 seconds from __future__ import print_function + import multiprocessing import os import signal diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/train b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/train index 392c5bbeab..0bb894b705 100755 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/train +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/container/news-classifier/train @@ -18,26 +18,24 @@ from __future__ import print_function -import os import json +import os import pickle +import re import sys import traceback -from sklearn.feature_extraction import stop_words - +import boto3 +import numpy as np import pandas as pd import tensorflow as tf -import re -import numpy as np - -import boto3 - -from tensorflow.python.estimator.export.export import build_raw_serving_input_receiver_fn +from sklearn.feature_extraction import stop_words +from tensorflow.keras.losses import CategoricalCrossentropy +from tensorflow.python.estimator.export.export import \ + build_raw_serving_input_receiver_fn from tensorflow.python.estimator.export.export_output import PredictOutput -from tensorflow.python.keras.preprocessing.text import Tokenizer from tensorflow.python.keras.preprocessing.sequence import pad_sequences -from tensorflow.keras.losses import CategoricalCrossentropy +from tensorflow.python.keras.preprocessing.text import Tokenizer # These are the paths to where SageMaker mounts interesting things in your container. diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/create_validation_set.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/create_validation_set.py index ffc6476fc2..c3e58d7128 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/create_validation_set.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/create_validation_set.py @@ -1,7 +1,7 @@ -from s3_helper import S3Ref, copy_with_query, create_ref_at_parent_key - import logging +from s3_helper import S3Ref, copy_with_query, create_ref_at_parent_key + logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/helper.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/helper.py index 5ff9558a60..47ec2f7d3b 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/helper.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/helper.py @@ -1,5 +1,5 @@ -from datetime import datetime import random +from datetime import datetime AUTOANNOTATION_THRESHOLD = 0.50 JOB_TYPE = "groundtruth/text-classification" diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/perform_active_learning.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/perform_active_learning.py index 7b26666016..c7f1d834b0 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/perform_active_learning.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/perform_active_learning.py @@ -1,12 +1,10 @@ import json - -from s3_helper import S3Ref, download, download_with_query, upload, create_ref_at_parent_key -from string_helper import generate_job_id_and_s3_path +import logging from io import StringIO from ActiveLearning.helper import SimpleActiveLearning - -import logging +from s3_helper import S3Ref, create_ref_at_parent_key, download, download_with_query, upload +from string_helper import generate_job_id_and_s3_path logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/prepare_for_inference.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/prepare_for_inference.py index f5e06cef52..91d78cfd54 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/prepare_for_inference.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/prepare_for_inference.py @@ -1,9 +1,8 @@ import json - +import logging from io import StringIO -from s3_helper import S3Ref, copy_with_query_and_transform, create_ref_at_parent_key -import logging +from s3_helper import S3Ref, copy_with_query_and_transform, create_ref_at_parent_key logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/prepare_for_training.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/prepare_for_training.py index 965ab1c0b6..a6585a6a7a 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/prepare_for_training.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/ActiveLearning/prepare_for_training.py @@ -1,17 +1,16 @@ import json +import logging from functools import partial - from io import StringIO + from s3_helper import ( S3Ref, copy_with_query_and_transform, - download_with_query, create_ref_at_parent_key, + download_with_query, ) from string_helper import generate_job_id_and_s3_path -import logging - logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Bootstrap/add_record_id.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Bootstrap/add_record_id.py index c9b0fb60d1..5604acfe17 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Bootstrap/add_record_id.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Bootstrap/add_record_id.py @@ -1,8 +1,8 @@ -from s3_helper import S3Ref, download, upload import json +import logging from io import StringIO -import logging +from s3_helper import S3Ref, download, upload logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Bootstrap/copy_input_manifest.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Bootstrap/copy_input_manifest.py index 18e4d253bc..8e41c35050 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Bootstrap/copy_input_manifest.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Bootstrap/copy_input_manifest.py @@ -1,7 +1,7 @@ -from s3_helper import S3Ref, get_content_size, copy - import logging +from s3_helper import S3Ref, copy, get_content_size + logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Labeling/prepare_for_labeling.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Labeling/prepare_for_labeling.py index 0c573fd2a6..4d626bdfc5 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Labeling/prepare_for_labeling.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Labeling/prepare_for_labeling.py @@ -1,8 +1,8 @@ +import logging + from s3_helper import S3Ref, copy_with_query from string_helper import generate_job_id_and_s3_path -import logging - logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/MetaData/get_counts.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/MetaData/get_counts.py index 866d52eda5..8be133b9dd 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/MetaData/get_counts.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/MetaData/get_counts.py @@ -1,7 +1,7 @@ +import logging from functools import partial -from s3_helper import S3Ref, get_count_with_query -import logging +from s3_helper import S3Ref, get_count_with_query logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Output/export_final.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Output/export_final.py index b02e6aee4d..2ff607c641 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Output/export_final.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Output/export_final.py @@ -1,7 +1,7 @@ -from s3_helper import S3Ref, copy - import logging +from s3_helper import S3Ref, copy + logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Output/export_partial.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Output/export_partial.py index a0bc4ba057..a1ae2774f9 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Output/export_partial.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/Output/export_partial.py @@ -1,10 +1,10 @@ import json -from collections import OrderedDict -from s3_helper import S3Ref, download, upload - import logging +from collections import OrderedDict from io import StringIO +from s3_helper import S3Ref, download, upload + logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/dependency/python/s3_helper.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/dependency/python/s3_helper.py index 6091c63b3b..8727fd12ee 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/dependency/python/s3_helper.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/dependency/python/s3_helper.py @@ -1,13 +1,11 @@ """ Utility file to help with s3 operations. """ +from io import BytesIO, StringIO, TextIOWrapper +from typing import Callable, NamedTuple from urllib.parse import urlparse -import boto3 - -from typing import NamedTuple -from typing import Callable -from io import BytesIO, StringIO, TextIOWrapper +import boto3 s3r = boto3.resource("s3") s3 = boto3.client("s3") diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/dependency/python/string_helper.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/dependency/python/string_helper.py index 83ab44ce5a..0373ac8246 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/dependency/python/string_helper.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/dependency/python/string_helper.py @@ -1,6 +1,5 @@ -import string import random - +import string from typing import Tuple diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_add_record_id.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_add_record_id.py index 9333ed4a9c..ec0b058f55 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_add_record_id.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_add_record_id.py @@ -1,7 +1,6 @@ import boto3 -from moto import mock_s3 - from Bootstrap.add_record_id import lambda_handler +from moto import mock_s3 @mock_s3 diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_copy_input_manifest.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_copy_input_manifest.py index d7b86366d2..ddd73e3c6d 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_copy_input_manifest.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_copy_input_manifest.py @@ -1,7 +1,7 @@ -import pytest import boto3 -from moto import mock_s3 +import pytest from Bootstrap.copy_input_manifest import lambda_handler +from moto import mock_s3 @mock_s3 diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_create_validation_set.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_create_validation_set.py index e2c99a8b15..fedf4c2cb8 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_create_validation_set.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_create_validation_set.py @@ -1,6 +1,5 @@ -from moto import mock_s3 - from ActiveLearning.create_validation_set import lambda_handler +from moto import mock_s3 @mock_s3 diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_get_counts.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_get_counts.py index 77a2548ec0..9086c65769 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_get_counts.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_get_counts.py @@ -1,7 +1,6 @@ import boto3 -from moto import mock_s3 - from MetaData.get_counts import lambda_handler +from moto import mock_s3 @mock_s3 diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_perform_active_learning.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_perform_active_learning.py index 2d8a858ed7..9f83be1f22 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_perform_active_learning.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_perform_active_learning.py @@ -1,7 +1,8 @@ -import boto3 -from moto import mock_s3 from io import StringIO + +import boto3 from ActiveLearning.perform_active_learning import lambda_handler +from moto import mock_s3 @mock_s3 diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_inference.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_inference.py index 0360b69ffa..65a0c10e29 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_inference.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_inference.py @@ -1,8 +1,8 @@ -from moto import mock_s3 -import boto3 from io import StringIO +import boto3 from ActiveLearning.prepare_for_inference import lambda_handler +from moto import mock_s3 @mock_s3 diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_labeling.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_labeling.py index 65796b51bb..31e89fb59f 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_labeling.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_labeling.py @@ -1,6 +1,5 @@ -from moto import mock_s3 - from Labeling.prepare_for_labeling import lambda_handler +from moto import mock_s3 @mock_s3 diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_training.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_training.py index fb83035afa..350a22689c 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_training.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_prepare_for_training.py @@ -1,8 +1,9 @@ -from moto import mock_s3 -import boto3 from io import StringIO + +import boto3 import string_helper from ActiveLearning.prepare_for_training import lambda_handler +from moto import mock_s3 @mock_s3 diff --git a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_simple_active_learning.py b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_simple_active_learning.py index b3858cef3c..dc21f954c0 100644 --- a/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_simple_active_learning.py +++ b/ground_truth_labeling_jobs/bring_your_own_model_for_sagemaker_labeling_workflows_with_active_learning/src/tests/test_simple_active_learning.py @@ -1,5 +1,5 @@ -from ActiveLearning.helper import SimpleActiveLearning import pytest +from ActiveLearning.helper import SimpleActiveLearning def test_compute_margin_high_confidence(): diff --git a/ground_truth_labeling_jobs/ground_truth_object_detection_tutorial/ground_truth_od.py b/ground_truth_labeling_jobs/ground_truth_object_detection_tutorial/ground_truth_od.py index 9160bc8261..d5b9b01ebf 100644 --- a/ground_truth_labeling_jobs/ground_truth_object_detection_tutorial/ground_truth_od.py +++ b/ground_truth_labeling_jobs/ground_truth_object_detection_tutorial/ground_truth_od.py @@ -4,6 +4,7 @@ """ import os + import imageio import matplotlib.pyplot as plt import numpy as np @@ -171,7 +172,7 @@ def plot_gt_bbs(self, ax, img_kwargs={}, box_kwargs={"edgecolor": "lime", "lw": self.plot_bbs(ax, self.gt_boxes, img_kwargs=img_kwargs, box_kwargs=box_kwargs) def compute_img_confidence(self): - """ Compute the mean bb confidence. """ + """Compute the mean bb confidence.""" if len(self.consolidated_boxes) > 0: return np.mean([box.confidence for box in self.consolidated_boxes]) else: diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/datalake/scripts/glue_cw_worker_metrics_transform.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/datalake/scripts/glue_cw_worker_metrics_transform.py index 7d53e5f116..acd73339a6 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/datalake/scripts/glue_cw_worker_metrics_transform.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/datalake/scripts/glue_cw_worker_metrics_transform.py @@ -1,17 +1,15 @@ """ Glue Script to tranfor worker metrics from Cloudwatch into Glue Catalog """ -from pyspark.sql.functions import from_json, col -from pyspark.sql.functions import year, month, dayofmonth, hour -from pyspark.sql.types import StructType, StructField, StringType, TimestampType import sys -from pyspark.context import SparkContext import pyspark.sql.functions as f - -from awsglue.utils import getResolvedOptions from awsglue.context import GlueContext from awsglue.job import Job +from awsglue.utils import getResolvedOptions +from pyspark.context import SparkContext +from pyspark.sql.functions import col, dayofmonth, from_json, hour, month, year +from pyspark.sql.types import StringType, StructField, StructType, TimestampType sc = SparkContext() glueContext = GlueContext(SparkContext.getOrCreate()) diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/datalake/scripts/glue_reporting_transform_script.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/datalake/scripts/glue_reporting_transform_script.py index 0a6e5cbf4c..0ed4b31b4a 100755 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/datalake/scripts/glue_reporting_transform_script.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/datalake/scripts/glue_reporting_transform_script.py @@ -1,18 +1,24 @@ -from pyspark.sql.functions import from_json, col, split, input_file_name -from pyspark.sql.functions import year, month, dayofmonth, hour -from pyspark.sql.types import StructType, StructField, StringType, TimestampType, ArrayType -from pyspark.sql.functions import udf import sys -from pyspark.sql.functions import unix_timestamp -from pyspark.context import SparkContext +import boto3 import pyspark.sql.functions as f - -from awsglue.utils import getResolvedOptions from awsglue.context import GlueContext from awsglue.job import Job - -import boto3 +from awsglue.utils import getResolvedOptions +from pyspark.context import SparkContext +from pyspark.sql.functions import ( + col, + dayofmonth, + from_json, + hour, + input_file_name, + month, + split, + udf, + unix_timestamp, + year, +) +from pyspark.sql.types import ArrayType, StringType, StructField, StructType, TimestampType sc = SparkContext() glueContext = GlueContext(SparkContext.getOrCreate()) diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_create/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_create/main.py index 680ee3e15f..24d238a46b 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_create/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_create/main.py @@ -5,15 +5,14 @@ and validating input. """ -import os -import uuid import json +import os import re +import uuid import boto3 import botocore - -from shared import db, log, label_arn +from shared import db, label_arn, log from shared.constants import SmgtJobType batch_step_function_arn = os.getenv("BATCH_CREATION_STEP_FUNCTION_ARN") diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_metadata_post/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_metadata_post/main.py index cfa10c664b..5446f0c92c 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_metadata_post/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_metadata_post/main.py @@ -7,8 +7,8 @@ import json -from shared import log, db -from shared.constants import BatchMetadataType, BatchStatus, BatchMetadataTableAttributes +from shared import db, log +from shared.constants import BatchMetadataTableAttributes, BatchMetadataType, BatchStatus from shared.lambda_context import get_boto_client diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_show/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_show/main.py index db5fde5db3..a2f862463d 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_show/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_batch_show/main.py @@ -6,11 +6,10 @@ import json import botocore - from shared import db -from shared.constants import BatchMetadataType, BatchStatus -from shared.log import logger, log_request_and_context from shared.api_helpers import input_batch_to_human_readable +from shared.constants import BatchMetadataType, BatchStatus +from shared.log import log_request_and_context, logger def get_all_batches(): diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_workforce_show/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_workforce_show/main.py index e9a20e69a6..3f0869edf3 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_workforce_show/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/api_workforce_show/main.py @@ -7,9 +7,8 @@ import os import boto3 - from shared.constants import SMGTJobCategory -from shared.log import logger, log_request_and_context +from shared.log import log_request_and_context, logger def get_member_definition_info(cognito, member_definition): diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/labeling_job_state_change/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/labeling_job_state_change/main.py index 08e3006030..d3144f11f0 100755 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/labeling_job_state_change/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/labeling_job_state_change/main.py @@ -6,14 +6,12 @@ validate fully, this will still write the job state back to the DB. """ import json +import os +from re import search import boto3 - from shared import db, log -from re import search -import os - -from shared.constants import BatchStatus, BatchMetadataTableAttributes, BatchMetadataType +from shared.constants import BatchMetadataTableAttributes, BatchMetadataType, BatchStatus from shared.db import get_batch_metadata_by_labeling_job_name sfn_client = boto3.client("stepfunctions") diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/main.py index e4573645e8..09c8cd7e3f 100755 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/main.py @@ -1,33 +1,28 @@ """Provides named imports for lambda deployment.""" -from api_batch_show.main import lambda_handler as api_batch_show_lambda_handler from api_batch_create.main import lambda_handler as api_batch_create_lambda_handler -from api_workforce_show.main import lambda_handler as api_workforce_show_lambda_handler from api_batch_metadata_post.main import lambda_handler as api_batch_post_metadata_lambda_handler - +from api_batch_show.main import lambda_handler as api_batch_show_lambda_handler +from api_workforce_show.main import lambda_handler as api_workforce_show_lambda_handler +from labeling_job_state_change.main import lambda_handler as labeling_job_state_change_handler +from step_functions_batch_error.main import ( + lambda_handler as step_functions_batch_error_lambda_handler, +) +from step_functions_copy_logs_and_send_batch_completed.main import ( + lambda_handler as step_functions_copy_logs_and_send_batch_completed_lambda_handler, +) +from step_functions_send_second_level_sns_and_check_response.main import ( + lambda_handler as step_functions_send_second_level_sns_and_check_response_lambda_handler, +) from step_functions_transformation.main import ( lambda_handler as step_functions_transformation_lambda_handler, ) from step_functions_trigger_labeling_job.main import ( lambda_handler as step_functions_trigger_labeling_job_lambda_handler, ) -from step_functions_send_second_level_sns_and_check_response.main import ( - lambda_handler as step_functions_send_second_level_sns_and_check_response_lambda_handler, -) from step_functions_wait_batch_completion.main import ( lambda_handler as step_functions_wait_batch_completion_lambda_handler, ) -from step_functions_copy_logs_and_send_batch_completed.main import ( - lambda_handler as step_functions_copy_logs_and_send_batch_completed_lambda_handler, -) -from step_functions_batch_error.main import ( - lambda_handler as step_functions_batch_error_lambda_handler, -) - -from labeling_job_state_change.main import ( - lambda_handler as labeling_job_state_change_handler, -) - from step_functions_wait_for_metadata_supply.main import ( lambda_handler as step_functions_wait_batch_metadata_input_lambda_handler, ) diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/api_helpers.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/api_helpers.py index eca0b46ef6..96bf063666 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/api_helpers.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/api_helpers.py @@ -5,12 +5,9 @@ import boto3 import botocore - from shared import db -from shared.constants import ( - BatchMetadataTableAttributes as Attributes, - BatchMetadataType, -) +from shared.constants import BatchMetadataTableAttributes as Attributes +from shared.constants import BatchMetadataType from shared.log import logger diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/db.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/db.py index cc75966108..b7e13937af 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/db.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/db.py @@ -1,16 +1,14 @@ import os +from decimal import Decimal import boto3 from boto3.dynamodb.conditions import Key -from .constants import ( - BatchMetadataTableAttributes as Attributes, - BatchCurrentStep, - BatchMetadataType, - BatchStatus, -) -from decimal import Decimal from shared import log +from .constants import BatchCurrentStep +from .constants import BatchMetadataTableAttributes as Attributes +from .constants import BatchMetadataType, BatchStatus + BATCH_EXECUTION_METADATA_TABLE_NAME = os.getenv("BATCH_EXECUTION_METADATA_TABLE_NAME", "") dynamodb = boto3.resource("dynamodb") diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/log.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/log.py index bba9a74864..8a17ac7417 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/log.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/log.py @@ -1,9 +1,9 @@ import json import logging -from pprint import pformat import sys import time import traceback +from pprint import pformat # Root-level loggers should only print when something bad happens logger = logging.getLogger() diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/s3_accessor.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/s3_accessor.py index aaaa204f04..1fd531360b 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/s3_accessor.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/shared/s3_accessor.py @@ -1,5 +1,4 @@ import boto3 - from shared import log s3 = boto3.resource("s3") diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_batch_error/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_batch_error/main.py index 6b69772444..434edb45bd 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_batch_error/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_batch_error/main.py @@ -6,14 +6,13 @@ 2) Publish job failed SNS. """ -import os import json +import os import boto3 import botocore - -from shared.log import log_request_and_context, logger from shared import db +from shared.log import log_request_and_context, logger def lambda_handler(event, context): diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_copy_logs_and_send_batch_completed/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_copy_logs_and_send_batch_completed/main.py index a807395fb9..689f2e8edd 100755 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_copy_logs_and_send_batch_completed/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_copy_logs_and_send_batch_completed/main.py @@ -4,16 +4,15 @@ Sends message to status SNS on completion. """ -import os import json +import os import boto3 import botocore - -from shared.constants import BatchStatus -from shared.log import log_request_and_context, logger from shared import db from shared.api_helpers import input_batch_to_human_readable +from shared.constants import BatchStatus +from shared.log import log_request_and_context, logger sns = boto3.resource("sns") diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_transformation/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_transformation/main.py index e0364e6239..caffc5590e 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_transformation/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_transformation/main.py @@ -6,7 +6,6 @@ """ import boto3 - from shared import db, log from shared.constants import BatchCurrentStep, BatchMetadataType, BatchStatus diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_trigger_labeling_job/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_trigger_labeling_job/main.py index 3efd3c2a4f..5657936781 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_trigger_labeling_job/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_trigger_labeling_job/main.py @@ -5,19 +5,18 @@ import json import os -from urllib.parse import urlparse from collections import namedtuple +from urllib.parse import urlparse import boto3 import botocore - -from shared import db, log, label_arn +from shared import db, label_arn, log from shared.constants import ( - BatchCurrentStep, - BatchStatus, - BatchMetadataType, SNS_DEDUPLICATION_KEY_NAME, + BatchCurrentStep, BatchMetadataTableAttributes, + BatchMetadataType, + BatchStatus, SmgtJobType, ) from shared.s3_accessor import fetch_s3, put_s3 diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_wait_batch_completion/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_wait_batch_completion/main.py index a7f81d5e21..48f743aef7 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_wait_batch_completion/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_wait_batch_completion/main.py @@ -5,8 +5,8 @@ import json from shared import db, log +from shared.constants import BatchMetadataTableAttributes, BatchStatus from shared.lambda_context import get_boto_client -from shared.constants import BatchStatus, BatchMetadataTableAttributes def lambda_handler(event, context): diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_wait_for_metadata_supply/main.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_wait_for_metadata_supply/main.py index f741275eda..8b5c66a12e 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_wait_for_metadata_supply/main.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_src/step_functions_wait_for_metadata_supply/main.py @@ -7,7 +7,6 @@ from urllib.parse import urlparse import boto3 - from shared import db, log sns = boto3.resource("sns") diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/api_batch_create/main_test.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/api_batch_create/main_test.py index 789087fe82..c842c4b367 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/api_batch_create/main_test.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/api_batch_create/main_test.py @@ -2,12 +2,11 @@ import os import unittest from unittest import TestCase -from unittest.mock import patch, Mock - -from botocore.exceptions import ClientError +from unittest.mock import Mock, patch from api_batch_create.main import lambda_handler -from test_shared.mock_objects import TestContext, InputTestData +from botocore.exceptions import ClientError +from test_shared.mock_objects import InputTestData, TestContext class TestCase(TestCase): diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/api_batch_show/main_test.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/api_batch_show/main_test.py index 85de2a1617..4a63290f3b 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/api_batch_show/main_test.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/api_batch_show/main_test.py @@ -1,9 +1,9 @@ import unittest from unittest import TestCase -from unittest.mock import patch, Mock +from unittest.mock import Mock, patch from api_batch_show.main import lambda_handler -from test_shared.mock_objects import TestContext, InputTestData, OutputTestData +from test_shared.mock_objects import InputTestData, OutputTestData, TestContext class TestCase(TestCase): diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/labeling_job_state_change/main_test.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/labeling_job_state_change/main_test.py index 371bd1afb3..1d333df4ec 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/labeling_job_state_change/main_test.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/labeling_job_state_change/main_test.py @@ -3,7 +3,7 @@ from unittest.mock import patch from labeling_job_state_change.main import lambda_handler -from test_shared.mock_objects import TestContext, OutputTestData +from test_shared.mock_objects import OutputTestData, TestContext class TestCase(TestCase): diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/step_function_batch_error/main_test.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/step_function_batch_error/main_test.py index df4c7e28c4..4156315307 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/step_function_batch_error/main_test.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/step_function_batch_error/main_test.py @@ -1,7 +1,7 @@ import os import unittest from unittest import TestCase -from unittest.mock import patch, Mock +from unittest.mock import Mock, patch from step_functions_batch_error.main import lambda_handler from test_shared.mock_objects import TestContext diff --git a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/step_functions_copy_logs_and_send_batch_completed/main_test.py b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/step_functions_copy_logs_and_send_batch_completed/main_test.py index 354622dc79..c2fd017d1e 100644 --- a/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/step_functions_copy_logs_and_send_batch_completed/main_test.py +++ b/ground_truth_labeling_jobs/multi_modal_parallel_sagemaker_labeling_workflows_with_step_functions/src/lambda_test/step_functions_copy_logs_and_send_batch_completed/main_test.py @@ -1,9 +1,9 @@ import os import unittest from unittest import TestCase -from unittest.mock import patch, Mock +from unittest.mock import Mock, patch -from test_shared.mock_objects import TestContext, OutputTestData +from test_shared.mock_objects import OutputTestData, TestContext class TestCase(TestCase): diff --git a/ground_truth_labeling_jobs/sagemaker_ground_truth_semantic_segmentation_mask_index_correction.py b/ground_truth_labeling_jobs/sagemaker_ground_truth_semantic_segmentation_mask_index_correction.py index e3f0c21b1b..a27e9932e7 100644 --- a/ground_truth_labeling_jobs/sagemaker_ground_truth_semantic_segmentation_mask_index_correction.py +++ b/ground_truth_labeling_jobs/sagemaker_ground_truth_semantic_segmentation_mask_index_correction.py @@ -31,9 +31,9 @@ import argparse import base64 +import json from collections import defaultdict from io import BytesIO -import json from pathlib import Path from urllib.parse import urlparse diff --git a/ground_truth_labeling_jobs/video_annotations_quality_assessment/plotting_funcs.py b/ground_truth_labeling_jobs/video_annotations_quality_assessment/plotting_funcs.py index 2405fe9e4a..7ee764b312 100644 --- a/ground_truth_labeling_jobs/video_annotations_quality_assessment/plotting_funcs.py +++ b/ground_truth_labeling_jobs/video_annotations_quality_assessment/plotting_funcs.py @@ -1,6 +1,6 @@ -from matplotlib import pyplot as plt -import pandas as pd import numpy as np +import pandas as pd +from matplotlib import pyplot as plt # from numba import njit diff --git a/ground_truth_labeling_jobs/video_annotations_quality_assessment/quality_metrics_cli.py b/ground_truth_labeling_jobs/video_annotations_quality_assessment/quality_metrics_cli.py index f0d26ca524..118bdb7871 100644 --- a/ground_truth_labeling_jobs/video_annotations_quality_assessment/quality_metrics_cli.py +++ b/ground_truth_labeling_jobs/video_annotations_quality_assessment/quality_metrics_cli.py @@ -1,12 +1,13 @@ -import os import json -import numpy as np +import os + import argh import boto3 +import numpy as np from argh import arg -from tqdm import tqdm -from scipy.spatial import distance from plotting_funcs import * +from scipy.spatial import distance +from tqdm import tqdm s3 = boto3.client("s3") diff --git a/hyperparameter_tuning/chainer_cifar10/s3_util.py b/hyperparameter_tuning/chainer_cifar10/s3_util.py index c838d84bb6..fc7391694a 100644 --- a/hyperparameter_tuning/chainer_cifar10/s3_util.py +++ b/hyperparameter_tuning/chainer_cifar10/s3_util.py @@ -11,10 +11,11 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import boto3 +import os import tarfile from urllib.parse import urlparse -import os + +import boto3 def retrieve_output_from_s3(s3_url, output_dir): diff --git a/hyperparameter_tuning/chainer_cifar10/src/chainer_cifar_vgg_single_machine.py b/hyperparameter_tuning/chainer_cifar10/src/chainer_cifar_vgg_single_machine.py index 42d37677ee..06e0183924 100644 --- a/hyperparameter_tuning/chainer_cifar10/src/chainer_cifar_vgg_single_machine.py +++ b/hyperparameter_tuning/chainer_cifar10/src/chainer_cifar_vgg_single_machine.py @@ -11,21 +11,18 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function import argparse import os -import numpy as np - import chainer import chainer.functions as F import chainer.links as L -from chainer import training -from chainer import serializers -from chainer.training import extensions - import net +import numpy as np +from chainer import serializers, training +from chainer.training import extensions if __name__ == "__main__": diff --git a/hyperparameter_tuning/keras_bring_your_own/main.py b/hyperparameter_tuning/keras_bring_your_own/main.py index 8ac6ac2ba0..af0351ceea 100644 --- a/hyperparameter_tuning/keras_bring_your_own/main.py +++ b/hyperparameter_tuning/keras_bring_your_own/main.py @@ -17,9 +17,8 @@ import tempfile import numpy as np -import tensorflow as tf - import sagemaker +import tensorflow as tf from sagemaker.estimator import Estimator NUM_CLASSES = 10 diff --git a/hyperparameter_tuning/keras_bring_your_own/trainer/environment.py b/hyperparameter_tuning/keras_bring_your_own/trainer/environment.py index de254b14fe..3378cb798e 100644 --- a/hyperparameter_tuning/keras_bring_your_own/trainer/environment.py +++ b/hyperparameter_tuning/keras_bring_your_own/trainer/environment.py @@ -12,15 +12,14 @@ # language governing permissions and limitations under the License. from __future__ import absolute_import -import yaml +import collections import logging import multiprocessing - import os import shlex import subprocess -import collections +import yaml logging.basicConfig() logger = logging.getLogger(__name__) diff --git a/hyperparameter_tuning/keras_bring_your_own/trainer/start.py b/hyperparameter_tuning/keras_bring_your_own/trainer/start.py index 3752236849..9a552a8422 100644 --- a/hyperparameter_tuning/keras_bring_your_own/trainer/start.py +++ b/hyperparameter_tuning/keras_bring_your_own/trainer/start.py @@ -10,18 +10,16 @@ # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from __future__ import absolute_import -from __future__ import print_function +from __future__ import absolute_import, print_function +import os + +import numpy as np import tensorflow as tf from tensorflow import keras -from tensorflow.keras.preprocessing.image import ImageDataGenerator +from tensorflow.keras.layers import Activation, Conv2D, Dense, Dropout, Flatten, MaxPooling2D from tensorflow.keras.models import Sequential -from tensorflow.keras.layers import Dense, Dropout, Activation, Flatten -from tensorflow.keras.layers import Conv2D, MaxPooling2D -import os -import numpy as np - +from tensorflow.keras.preprocessing.image import ImageDataGenerator from trainer.environment import create_trainer_environment NUM_CLASSES = 10 diff --git a/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/cifar10.py b/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/cifar10.py index 10eaf30242..910d713fe4 100644 --- a/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/cifar10.py +++ b/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/cifar10.py @@ -11,7 +11,6 @@ from mxnet import gluon from mxnet.gluon.model_zoo import vision as models - # ------------------------------------------------------------ # # Training methods # # ------------------------------------------------------------ # diff --git a/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/cifar10_utils.py b/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/cifar10_utils.py index 85b33078fa..23be837b45 100644 --- a/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/cifar10_utils.py +++ b/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/cifar10_utils.py @@ -1,8 +1,9 @@ -import numpy as np import os import zipfile -from skimage import io + +import numpy as np from mxnet.test_utils import download +from skimage import io def download_training_data(): diff --git a/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/random_tuner.py b/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/random_tuner.py index 7d0e22eaee..cd89b4ec1b 100644 --- a/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/random_tuner.py +++ b/hyperparameter_tuning/mxnet_gluon_cifar10_random_search/random_tuner.py @@ -1,10 +1,10 @@ import random +import re import time + import boto3 -import re -import pandas as pd import numpy as np - +import pandas as pd ################# # Hyperparameters diff --git a/hyperparameter_tuning/mxnet_mnist/mnist.py b/hyperparameter_tuning/mxnet_mnist/mnist.py index 4be15b7db7..72e385e196 100644 --- a/hyperparameter_tuning/mxnet_mnist/mnist.py +++ b/hyperparameter_tuning/mxnet_mnist/mnist.py @@ -7,7 +7,6 @@ import mxnet as mx import numpy as np - from sagemaker_mxnet_container.training_utils import scheduler_host diff --git a/hyperparameter_tuning/pytorch_mnist/mnist.py b/hyperparameter_tuning/pytorch_mnist/mnist.py index 3e84e91e93..df896758a3 100644 --- a/hyperparameter_tuning/pytorch_mnist/mnist.py +++ b/hyperparameter_tuning/pytorch_mnist/mnist.py @@ -2,8 +2,9 @@ import json import logging import os -import sagemaker_containers import sys + +import sagemaker_containers import torch import torch.distributed as dist import torch.nn as nn diff --git a/hyperparameter_tuning/rapids_bring_your_own/code/HPOConfig.py b/hyperparameter_tuning/rapids_bring_your_own/code/HPOConfig.py index 14245e5549..37f86d341e 100644 --- a/hyperparameter_tuning/rapids_bring_your_own/code/HPOConfig.py +++ b/hyperparameter_tuning/rapids_bring_your_own/code/HPOConfig.py @@ -14,19 +14,19 @@ # limitations under the License. # -import os import argparse import glob +import logging +import os import pprint -import HPODatasets -import logging +import HPODatasets hpo_log = logging.getLogger("hpo_log") class HPOConfig(object): - """ Cloud integrated RAPIDS HPO functionality with AWS SageMaker focus """ + """Cloud integrated RAPIDS HPO functionality with AWS SageMaker focus""" sagemaker_directory_structure = { "train_data": "/opt/ml/input/data/training", @@ -120,7 +120,7 @@ def parse_configuration(self): return dataset_type, model_type, compute_type, cv_folds def parse_hyper_parameter_inputs(self, input_args): - """ Parse hyperparmeters provided by the HPO orchestrator """ + """Parse hyperparmeters provided by the HPO orchestrator""" hpo_log.info("parsing model hyperparameters from command line arguments...log") # noqa parser = argparse.ArgumentParser() diff --git a/hyperparameter_tuning/rapids_bring_your_own/code/MLWorkflow.py b/hyperparameter_tuning/rapids_bring_your_own/code/MLWorkflow.py index 97fb1ef378..fd747c0c03 100644 --- a/hyperparameter_tuning/rapids_bring_your_own/code/MLWorkflow.py +++ b/hyperparameter_tuning/rapids_bring_your_own/code/MLWorkflow.py @@ -14,18 +14,16 @@ # limitations under the License. # -from abc import abstractmethod - import functools -import time - import logging +import time +from abc import abstractmethod hpo_log = logging.getLogger("hpo_log") def create_workflow(hpo_config): - """ Workflow Factory [instantiate MLWorkflow based on config] """ + """Workflow Factory [instantiate MLWorkflow based on config]""" if hpo_config.compute_type == "single-CPU": from workflows.MLWorkflowSingleCPU import MLWorkflowSingleCPU diff --git a/hyperparameter_tuning/rapids_bring_your_own/code/serve.py b/hyperparameter_tuning/rapids_bring_your_own/code/serve.py index f883cb443c..675a25af15 100644 --- a/hyperparameter_tuning/rapids_bring_your_own/code/serve.py +++ b/hyperparameter_tuning/rapids_bring_your_own/code/serve.py @@ -14,25 +14,23 @@ # limitations under the License. # -import os -import sys -import traceback -import joblib import glob import json +import logging +import os +import sys import time - -import xgboost -import numpy +import traceback +from functools import lru_cache import flask +import joblib +import numpy +import xgboost from flask import Flask, Response -import logging -from functools import lru_cache - try: - """ check for GPU via library imports """ + """check for GPU via library imports""" import cupy from cuml import ForestInference @@ -47,7 +45,7 @@ def serve(xgboost_threshold=0.5): - """ Flask Inference Server for SageMaker hosting of RAPIDS Models """ + """Flask Inference Server for SageMaker hosting of RAPIDS Models""" app = Flask(__name__) logging.basicConfig(level=logging.DEBUG) @@ -60,7 +58,7 @@ def serve(xgboost_threshold=0.5): @app.route("/ping", methods=["GET"]) def ping(): - """ SageMaker required method, ping heartbeat """ + """SageMaker required method, ping heartbeat""" return Response(response="\n", status=200) @lru_cache() diff --git a/hyperparameter_tuning/rapids_bring_your_own/code/train.py b/hyperparameter_tuning/rapids_bring_your_own/code/train.py index 59c3ccb5a0..ebb7d694f3 100644 --- a/hyperparameter_tuning/rapids_bring_your_own/code/train.py +++ b/hyperparameter_tuning/rapids_bring_your_own/code/train.py @@ -14,9 +14,9 @@ # limitations under the License. # +import logging import sys import traceback -import logging from HPOConfig import HPOConfig from MLWorkflow import create_workflow diff --git a/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowMultiCPU.py b/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowMultiCPU.py index 1220e7debc..ed01b92c13 100644 --- a/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowMultiCPU.py +++ b/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowMultiCPU.py @@ -14,30 +14,26 @@ # limitations under the License. # -import time +import logging import os +import time +import warnings import dask -from dask.distributed import LocalCluster, Client, wait - -import xgboost import joblib - +import xgboost +from dask.distributed import Client, LocalCluster, wait from dask_ml.model_selection import train_test_split +from MLWorkflow import MLWorkflow, timer_decorator from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score -import logging -import warnings - -from MLWorkflow import MLWorkflow, timer_decorator - hpo_log = logging.getLogger("hpo_log") warnings.filterwarnings("ignore") class MLWorkflowMultiCPU(MLWorkflow): - """ Multi-CPU Workflow """ + """Multi-CPU Workflow""" def __init__(self, hpo_config): hpo_log.info("Multi-CPU Workflow") @@ -53,7 +49,7 @@ def __init__(self, hpo_config): @timer_decorator def cluster_initialize(self): - """ Initialize dask CPU cluster """ + """Initialize dask CPU cluster""" cluster = None client = None @@ -74,7 +70,7 @@ def cluster_initialize(self): return cluster, client def ingest_data(self): - """ Ingest dataset, CSV and Parquet supported """ + """Ingest dataset, CSV and Parquet supported""" if self.dataset_cache is not None: hpo_log.info("> skipping ingestion, using cache") @@ -102,7 +98,7 @@ def ingest_data(self): return dataset def handle_missing_data(self, dataset): - """ Drop samples with missing data [ inplace ] """ + """Drop samples with missing data [ inplace ]""" dataset = dataset.dropna() return dataset @@ -140,7 +136,7 @@ def split_dataset(self, dataset, random_state): @timer_decorator def fit(self, X_train, y_train): - """ Fit decision tree model """ + """Fit decision tree model""" if "XGBoost" in self.hpo_config.model_type: hpo_log.info("> fit xgboost model") dtrain = xgboost.dask.DaskDMatrix(self.client, X_train, y_train) @@ -164,7 +160,7 @@ def fit(self, X_train, y_train): @timer_decorator def predict(self, trained_model, X_test, threshold=0.5): - """ Inference with the trained model on the unseen test data """ + """Inference with the trained model on the unseen test data""" hpo_log.info("> predict with trained model ") if "XGBoost" in self.hpo_config.model_type: @@ -179,7 +175,7 @@ def predict(self, trained_model, X_test, threshold=0.5): @timer_decorator def score(self, y_test, predictions): - """ Score predictions vs ground truth labels on test data """ + """Score predictions vs ground truth labels on test data""" hpo_log.info("> score predictions") score = accuracy_score( @@ -192,7 +188,7 @@ def score(self, y_test, predictions): return score def save_best_model(self, score, trained_model, filename="saved_model"): - """ Persist/save model that sets a new high score """ + """Persist/save model that sets a new high score""" if score > self.best_score: self.best_score = score @@ -220,7 +216,7 @@ async def cleanup(self, i_fold): self.cluster, self.client = self.cluster_initialize() def emit_final_score(self): - """ Emit score for parsing by the cloud HPO orchestrator """ + """Emit score for parsing by the cloud HPO orchestrator""" exec_time = time.perf_counter() - self.start_time hpo_log.info(f"total_time = {exec_time:.5f} s ") diff --git a/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowMultiGPU.py b/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowMultiGPU.py index bc46e8c5b5..0d782fe839 100644 --- a/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowMultiGPU.py +++ b/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowMultiGPU.py @@ -15,34 +15,30 @@ # -import time +import logging import os +import time +import warnings +import cupy import dask import dask_cudf -from dask_cuda import LocalCUDACluster -from dask.distributed import wait, Client - -import cupy -import xgboost import joblib - -from dask_ml.model_selection import train_test_split +import xgboost from cuml.dask.common.utils import persist_across_workers from cuml.dask.ensemble import RandomForestClassifier from cuml.metrics import accuracy_score - +from dask.distributed import Client, wait +from dask_cuda import LocalCUDACluster +from dask_ml.model_selection import train_test_split from MLWorkflow import MLWorkflow, timer_decorator -import logging -import warnings - hpo_log = logging.getLogger("hpo_log") warnings.filterwarnings("ignore") class MLWorkflowMultiGPU(MLWorkflow): - """ Multi-GPU Workflow """ + """Multi-GPU Workflow""" def __init__(self, hpo_config): hpo_log.info("Multi-GPU Workflow") @@ -58,7 +54,7 @@ def __init__(self, hpo_config): @timer_decorator def cluster_initialize(self): - """ Initialize dask GPU cluster""" + """Initialize dask GPU cluster""" cluster = None client = None @@ -80,7 +76,7 @@ def cluster_initialize(self): return cluster, client def ingest_data(self): - """ Ingest dataset, CSV and Parquet supported [ async/lazy ]""" + """Ingest dataset, CSV and Parquet supported [ async/lazy ]""" if self.dataset_cache is not None: hpo_log.info("> skipping ingestion, using cache") @@ -105,7 +101,7 @@ def ingest_data(self): return dataset def handle_missing_data(self, dataset): - """ Drop samples with missing data [ inplace ] """ + """Drop samples with missing data [ inplace ]""" dataset = dataset.dropna() return dataset @@ -144,7 +140,7 @@ def split_dataset(self, dataset, random_state): @timer_decorator def fit(self, X_train, y_train): - """ Fit decision tree model """ + """Fit decision tree model""" if "XGBoost" in self.hpo_config.model_type: hpo_log.info("> fit xgboost model") dtrain = xgboost.dask.DaskDMatrix(self.client, X_train, y_train) @@ -169,7 +165,7 @@ def fit(self, X_train, y_train): @timer_decorator def predict(self, trained_model, X_test, threshold=0.5): - """ Inference with the trained model on the unseen test data """ + """Inference with the trained model on the unseen test data""" hpo_log.info("> predict with trained model ") if "XGBoost" in self.hpo_config.model_type: @@ -185,7 +181,7 @@ def predict(self, trained_model, X_test, threshold=0.5): @timer_decorator def score(self, y_test, predictions): - """ Score predictions vs ground truth labels on test data """ + """Score predictions vs ground truth labels on test data""" hpo_log.info("> score predictions") y_test = y_test.compute() score = accuracy_score( @@ -198,7 +194,7 @@ def score(self, y_test, predictions): return score def save_best_model(self, score, trained_model, filename="saved_model"): - """ Persist/save model that sets a new high score """ + """Persist/save model that sets a new high score""" if score > self.best_score: self.best_score = score @@ -227,7 +223,7 @@ async def cleanup(self, i_fold): self.cluster, self.client = self.cluster_initialize() def emit_final_score(self): - """ Emit score for parsing by the cloud HPO orchestrator """ + """Emit score for parsing by the cloud HPO orchestrator""" exec_time = time.perf_counter() - self.start_time hpo_log.info(f"total_time = {exec_time:.5f} s ") diff --git a/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowSingleCPU.py b/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowSingleCPU.py index 68296ad973..8b9b811dec 100644 --- a/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowSingleCPU.py +++ b/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowSingleCPU.py @@ -14,26 +14,23 @@ # limitations under the License. # -import time +import logging import os +import time +import joblib import pandas import xgboost -import joblib - -from sklearn.model_selection import train_test_split +from MLWorkflow import MLWorkflow, timer_decorator from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score - -from MLWorkflow import MLWorkflow, timer_decorator - -import logging +from sklearn.model_selection import train_test_split hpo_log = logging.getLogger("hpo_log") class MLWorkflowSingleCPU(MLWorkflow): - """ Single-CPU Workflow """ + """Single-CPU Workflow""" def __init__(self, hpo_config): hpo_log.info("Single-CPU Workflow") @@ -47,7 +44,7 @@ def __init__(self, hpo_config): @timer_decorator def ingest_data(self): - """ Ingest dataset, CSV and Parquet supported """ + """Ingest dataset, CSV and Parquet supported""" if self.dataset_cache is not None: hpo_log.info("> skipping ingestion, using cache") @@ -84,7 +81,7 @@ def ingest_data(self): @timer_decorator def handle_missing_data(self, dataset): - """ Drop samples with missing data [ inplace ] """ + """Drop samples with missing data [ inplace ]""" dataset = dataset.dropna() return dataset @@ -114,7 +111,7 @@ def split_dataset(self, dataset, random_state): @timer_decorator def fit(self, X_train, y_train): - """ Fit decision tree model """ + """Fit decision tree model""" if "XGBoost" in self.hpo_config.model_type: hpo_log.info("> fit xgboost model") dtrain = xgboost.DMatrix(data=X_train, label=y_train) @@ -137,7 +134,7 @@ def fit(self, X_train, y_train): @timer_decorator def predict(self, trained_model, X_test, threshold=0.5): - """ Inference with the trained model on the unseen test data """ + """Inference with the trained model on the unseen test data""" hpo_log.info("> predict with trained model ") if "XGBoost" in self.hpo_config.model_type: @@ -151,7 +148,7 @@ def predict(self, trained_model, X_test, threshold=0.5): @timer_decorator def score(self, y_test, predictions): - """ Score predictions vs ground truth labels on test data """ + """Score predictions vs ground truth labels on test data""" dataset_dtype = self.hpo_config.dataset_dtype score = accuracy_score(y_test.astype(dataset_dtype), predictions.astype(dataset_dtype)) @@ -160,7 +157,7 @@ def score(self, y_test, predictions): return score def save_best_model(self, score, trained_model, filename="saved_model"): - """ Persist/save model that sets a new high score """ + """Persist/save model that sets a new high score""" if score > self.best_score: self.best_score = score @@ -175,7 +172,7 @@ def cleanup(self, i_fold): hpo_log.info("> end of fold \n") def emit_final_score(self): - """ Emit score for parsing by the cloud HPO orchestrator """ + """Emit score for parsing by the cloud HPO orchestrator""" exec_time = time.perf_counter() - self.start_time hpo_log.info(f"total_time = {exec_time:.5f} s ") diff --git a/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowSingleGPU.py b/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowSingleGPU.py index be547734a3..2fe1a36744 100644 --- a/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowSingleGPU.py +++ b/hyperparameter_tuning/rapids_bring_your_own/code/workflows/MLWorkflowSingleGPU.py @@ -15,26 +15,23 @@ # -import time +import logging import os +import time import cudf -import xgboost import joblib - -from cuml.preprocessing.model_selection import train_test_split +import xgboost from cuml.ensemble import RandomForestClassifier from cuml.metrics import accuracy_score - +from cuml.preprocessing.model_selection import train_test_split from MLWorkflow import MLWorkflow, timer_decorator -import logging - hpo_log = logging.getLogger("hpo_log") class MLWorkflowSingleGPU(MLWorkflow): - """ Single-GPU Workflow """ + """Single-GPU Workflow""" def __init__(self, hpo_config): hpo_log.info("Single-GPU Workflow \n") @@ -48,7 +45,7 @@ def __init__(self, hpo_config): @timer_decorator def ingest_data(self): - """ Ingest dataset, CSV and Parquet supported """ + """Ingest dataset, CSV and Parquet supported""" if self.dataset_cache is not None: hpo_log.info("skipping ingestion, using cache") @@ -77,7 +74,7 @@ def ingest_data(self): @timer_decorator def handle_missing_data(self, dataset): - """ Drop samples with missing data [ inplace ] """ + """Drop samples with missing data [ inplace ]""" dataset = dataset.dropna() return dataset @@ -105,7 +102,7 @@ def split_dataset(self, dataset, random_state): @timer_decorator def fit(self, X_train, y_train): - """ Fit decision tree model """ + """Fit decision tree model""" if "XGBoost" in self.hpo_config.model_type: hpo_log.info("> fit xgboost model") dtrain = xgboost.DMatrix(data=X_train, label=y_train) @@ -127,7 +124,7 @@ def fit(self, X_train, y_train): @timer_decorator def predict(self, trained_model, X_test, threshold=0.5): - """ Inference with the trained model on the unseen test data """ + """Inference with the trained model on the unseen test data""" hpo_log.info("predict with trained model ") if "XGBoost" in self.hpo_config.model_type: @@ -141,7 +138,7 @@ def predict(self, trained_model, X_test, threshold=0.5): @timer_decorator def score(self, y_test, predictions): - """ Score predictions vs ground truth labels on test data """ + """Score predictions vs ground truth labels on test data""" dataset_dtype = self.hpo_config.dataset_dtype score = accuracy_score(y_test.astype(dataset_dtype), predictions.astype(dataset_dtype)) @@ -150,7 +147,7 @@ def score(self, y_test, predictions): return score def save_best_model(self, score, trained_model, filename="saved_model"): - """ Persist/save model that sets a new high score """ + """Persist/save model that sets a new high score""" if score > self.best_score: self.best_score = score @@ -165,7 +162,7 @@ def cleanup(self, i_fold): hpo_log.info("end of cv-fold \n") def emit_final_score(self): - """ Emit score for parsing by the cloud HPO orchestrator """ + """Emit score for parsing by the cloud HPO orchestrator""" exec_time = time.perf_counter() - self.start_time hpo_log.info(f"total_time = {exec_time:.5f} s ") diff --git a/hyperparameter_tuning/rapids_bring_your_own/helper_functions.py b/hyperparameter_tuning/rapids_bring_your_own/helper_functions.py index 0f967b4f80..c818c6b531 100644 --- a/hyperparameter_tuning/rapids_bring_your_own/helper_functions.py +++ b/hyperparameter_tuning/rapids_bring_your_own/helper_functions.py @@ -14,11 +14,12 @@ # limitations under the License. # +import os import random +import traceback import uuid + import boto3 -import os -import traceback def recommend_instance_type(code_choice, dataset_directory): @@ -55,7 +56,7 @@ def recommend_instance_type(code_choice, dataset_directory): def validate_dockerfile(rapids_base_container, dockerfile_name="Dockerfile"): - """ Validate that our desired rapids base image matches the Dockerfile """ + """Validate that our desired rapids base image matches the Dockerfile""" with open(dockerfile_name, "r") as dockerfile_handle: if rapids_base_container not in dockerfile_handle.read(): raise Exception( @@ -116,7 +117,7 @@ def summarize_hpo_results(tuning_job_name): def download_best_model(bucket, s3_model_output, hpo_results, local_directory): - """ Download best model from S3""" + """Download best model from S3""" try: target_bucket = boto3.resource("s3").Bucket(bucket) path_prefix = os.path.join( diff --git a/hyperparameter_tuning/tensorflow2_mnist/code/test_train.py b/hyperparameter_tuning/tensorflow2_mnist/code/test_train.py index 998545ac3e..83ca5300a8 100644 --- a/hyperparameter_tuning/tensorflow2_mnist/code/test_train.py +++ b/hyperparameter_tuning/tensorflow2_mnist/code/test_train.py @@ -1,10 +1,9 @@ -from train import train, parse_args - -import sys -import os -import boto3 import json +import os +import sys +import boto3 +from train import parse_args, train DIRNAME = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.join(DIRNAME, "../../..") diff --git a/hyperparameter_tuning/tensorflow2_mnist/code/train.py b/hyperparameter_tuning/tensorflow2_mnist/code/train.py index ca5b0bf93a..7211e59b40 100644 --- a/hyperparameter_tuning/tensorflow2_mnist/code/train.py +++ b/hyperparameter_tuning/tensorflow2_mnist/code/train.py @@ -1,18 +1,17 @@ from __future__ import print_function import argparse +import gzip +import json import logging import os -import json -import gzip -import numpy as np -import traceback import sys +import traceback +import numpy as np import tensorflow as tf -from tensorflow.keras.layers import Dense, Flatten, Conv2D from tensorflow.keras import Model - +from tensorflow.keras.layers import Conv2D, Dense, Flatten logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/hyperparameter_tuning/tensorflow_mnist/mnist.py b/hyperparameter_tuning/tensorflow_mnist/mnist.py index d43754952d..756721a49c 100644 --- a/hyperparameter_tuning/tensorflow_mnist/mnist.py +++ b/hyperparameter_tuning/tensorflow_mnist/mnist.py @@ -1,4 +1,5 @@ import os + import tensorflow as tf from tensorflow.python.estimator.model_fn import ModeKeys as Modes diff --git a/hyperparameter_tuning/tensorflow_mnist/utils.py b/hyperparameter_tuning/tensorflow_mnist/utils.py index cf8af1b2b1..c883d56685 100644 --- a/hyperparameter_tuning/tensorflow_mnist/utils.py +++ b/hyperparameter_tuning/tensorflow_mnist/utils.py @@ -1,5 +1,6 @@ """Converts MNIST data to TFRecords file format with Example protos.""" import os + import tensorflow as tf diff --git a/introduction_to_amazon_algorithms/ipinsights_login/generate_data.py b/introduction_to_amazon_algorithms/ipinsights_login/generate_data.py index ec10515bf7..010c76afbb 100644 --- a/introduction_to_amazon_algorithms/ipinsights_login/generate_data.py +++ b/introduction_to_amazon_algorithms/ipinsights_login/generate_data.py @@ -10,14 +10,15 @@ # See the License for the specific language governing permissions and limitations under the License. from __future__ import print_function -import multiprocessing as mp -import numpy as np + +import datetime import gzip as gz +import multiprocessing as mp import socket import struct from functools import partial -import datetime +import numpy as np # Script Parameters NUM_PROCESSES = 4 diff --git a/introduction_to_amazon_algorithms/lda_topic_modeling/generate_example_data.py b/introduction_to_amazon_algorithms/lda_topic_modeling/generate_example_data.py index a257924483..d8b03dfc40 100644 --- a/introduction_to_amazon_algorithms/lda_topic_modeling/generate_example_data.py +++ b/introduction_to_amazon_algorithms/lda_topic_modeling/generate_example_data.py @@ -7,12 +7,11 @@ # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. import matplotlib -import matplotlib.pyplot as plt import matplotlib.cm as cm +import matplotlib.pyplot as plt import numpy as np import scipy as sp import scipy.stats - from matplotlib.gridspec import GridSpec, GridSpecFromSubplotSpec diff --git a/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/concat_db.py b/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/concat_db.py index 9b11085b6a..51ff2adcd3 100644 --- a/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/concat_db.py +++ b/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/concat_db.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -from imdb import Imdb import random +from imdb import Imdb + class ConcatDB(Imdb): """ diff --git a/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/im2rec.py b/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/im2rec.py index d6ad10aceb..ecb8657259 100644 --- a/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/im2rec.py +++ b/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/im2rec.py @@ -18,18 +18,20 @@ # under the License. from __future__ import print_function + import os import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "../python")) -import mxnet as mx -import random import argparse -import cv2 +import random import time import traceback +import cv2 +import mxnet as mx + try: import multiprocessing except ImportError: diff --git a/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/imdb.py b/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/imdb.py index b10d32aec3..c9414326a4 100644 --- a/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/imdb.py +++ b/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/imdb.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -import numpy as np import os.path as osp +import numpy as np + class Imdb(object): """ diff --git a/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/pascal_voc.py b/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/pascal_voc.py index ae3d2a054a..d2b1826fef 100644 --- a/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/pascal_voc.py +++ b/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/pascal_voc.py @@ -16,11 +16,13 @@ # under the License. from __future__ import print_function + import os -import numpy as np -from imdb import Imdb import xml.etree.ElementTree as ET + import cv2 +import numpy as np +from imdb import Imdb class PascalVoc(Imdb): diff --git a/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/prepare_dataset.py b/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/prepare_dataset.py index 2ccdefee80..6acff43479 100644 --- a/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/prepare_dataset.py +++ b/introduction_to_amazon_algorithms/managed_spot_training_object_detection/tools/prepare_dataset.py @@ -19,14 +19,16 @@ # under the License. from __future__ import print_function -import sys, os + import argparse +import os import subprocess +import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "..")) -from pascal_voc import PascalVoc from concat_db import ConcatDB +from pascal_voc import PascalVoc def load_pascal(image_set, year, devkit_path, shuffle=False): diff --git a/introduction_to_amazon_algorithms/ntm_synthetic/generate_example_data.py b/introduction_to_amazon_algorithms/ntm_synthetic/generate_example_data.py index b2d2d3b489..78a725e8b8 100644 --- a/introduction_to_amazon_algorithms/ntm_synthetic/generate_example_data.py +++ b/introduction_to_amazon_algorithms/ntm_synthetic/generate_example_data.py @@ -7,8 +7,8 @@ # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. import matplotlib -import matplotlib.pyplot as plt import matplotlib.cm as cm +import matplotlib.pyplot as plt import numpy as np import scipy as sp import scipy.stats diff --git a/introduction_to_amazon_algorithms/object_detection_birds/tools/birdsOnEdge.py b/introduction_to_amazon_algorithms/object_detection_birds/tools/birdsOnEdge.py index fda8b4e957..41d15f6780 100644 --- a/introduction_to_amazon_algorithms/object_detection_birds/tools/birdsOnEdge.py +++ b/introduction_to_amazon_algorithms/object_detection_birds/tools/birdsOnEdge.py @@ -4,19 +4,20 @@ # All Rights Reserved. * # * # ***************************************************** -from threading import Thread, Event, Timer -import os +import datetime import json -import numpy as np -import greengrasssdk +import os import sys -import datetime import time -import awscam -import cv2 import urllib import zipfile +from threading import Event, Thread, Timer + +import awscam +import cv2 +import greengrasssdk import mo +import numpy as np # Create a greengrass core sdk client client = greengrasssdk.client("iot-data") @@ -55,7 +56,7 @@ class LocalDisplay(Thread): """ def __init__(self, resolution): - """ resolution - Desired resolution of the project stream """ + """resolution - Desired resolution of the project stream""" # Initialize the base class, so that the object can run on its own # thread. super(LocalDisplay, self).__init__() @@ -107,7 +108,7 @@ def join(self): def greengrass_infinite_infer_run(): - """ Entry point of the lambda function""" + """Entry point of the lambda function""" client.publish(topic=iot_topic, payload="Start of run loop...") diff --git a/introduction_to_amazon_algorithms/object_detection_birds/tools/im2rec.py b/introduction_to_amazon_algorithms/object_detection_birds/tools/im2rec.py index d6ad10aceb..ecb8657259 100644 --- a/introduction_to_amazon_algorithms/object_detection_birds/tools/im2rec.py +++ b/introduction_to_amazon_algorithms/object_detection_birds/tools/im2rec.py @@ -18,18 +18,20 @@ # under the License. from __future__ import print_function + import os import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "../python")) -import mxnet as mx -import random import argparse -import cv2 +import random import time import traceback +import cv2 +import mxnet as mx + try: import multiprocessing except ImportError: diff --git a/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/concat_db.py b/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/concat_db.py index 9b11085b6a..51ff2adcd3 100644 --- a/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/concat_db.py +++ b/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/concat_db.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -from imdb import Imdb import random +from imdb import Imdb + class ConcatDB(Imdb): """ diff --git a/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/im2rec.py b/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/im2rec.py index d6ad10aceb..ecb8657259 100644 --- a/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/im2rec.py +++ b/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/im2rec.py @@ -18,18 +18,20 @@ # under the License. from __future__ import print_function + import os import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "../python")) -import mxnet as mx -import random import argparse -import cv2 +import random import time import traceback +import cv2 +import mxnet as mx + try: import multiprocessing except ImportError: diff --git a/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/imdb.py b/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/imdb.py index b10d32aec3..c9414326a4 100644 --- a/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/imdb.py +++ b/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/imdb.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -import numpy as np import os.path as osp +import numpy as np + class Imdb(object): """ diff --git a/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/pascal_voc.py b/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/pascal_voc.py index ae3d2a054a..d2b1826fef 100644 --- a/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/pascal_voc.py +++ b/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/pascal_voc.py @@ -16,11 +16,13 @@ # under the License. from __future__ import print_function + import os -import numpy as np -from imdb import Imdb import xml.etree.ElementTree as ET + import cv2 +import numpy as np +from imdb import Imdb class PascalVoc(Imdb): diff --git a/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/prepare_dataset.py b/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/prepare_dataset.py index 2ccdefee80..6acff43479 100644 --- a/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/prepare_dataset.py +++ b/introduction_to_amazon_algorithms/object_detection_pascalvoc_coco/tools/prepare_dataset.py @@ -19,14 +19,16 @@ # under the License. from __future__ import print_function -import sys, os + import argparse +import os import subprocess +import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "..")) -from pascal_voc import PascalVoc from concat_db import ConcatDB +from pascal_voc import PascalVoc def load_pascal(image_set, year, devkit_path, shuffle=False): diff --git a/introduction_to_amazon_algorithms/seq2seq_translation_en-de/create_vocab_proto.py b/introduction_to_amazon_algorithms/seq2seq_translation_en-de/create_vocab_proto.py index bc596d37b6..8467ad0c31 100644 --- a/introduction_to_amazon_algorithms/seq2seq_translation_en-de/create_vocab_proto.py +++ b/introduction_to_amazon_algorithms/seq2seq_translation_en-de/create_vocab_proto.py @@ -7,22 +7,21 @@ # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. import argparse -from collections import Counter -from contextlib import ExitStack import gzip import io -from itertools import chain, islice import json import logging +import multiprocessing import os import pickle import struct -import multiprocessing +from collections import Counter +from contextlib import ExitStack +from itertools import chain, islice +from typing import Dict, Generator, Iterable, List, Mapping -from record_pb2 import Record import boto3 -from typing import Dict, Iterable, Mapping, Generator -from typing import List +from record_pb2 import Record logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) diff --git a/introduction_to_amazon_algorithms/seq2seq_translation_en-de/record_pb2.py b/introduction_to_amazon_algorithms/seq2seq_translation_en-de/record_pb2.py index b968490d80..86b1bddb33 100644 --- a/introduction_to_amazon_algorithms/seq2seq_translation_en-de/record_pb2.py +++ b/introduction_to_amazon_algorithms/seq2seq_translation_en-de/record_pb2.py @@ -5,10 +5,10 @@ _b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode("latin1")) from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pb2 from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) diff --git a/introduction_to_amazon_algorithms/xgboost_abalone/abalone.py b/introduction_to_amazon_algorithms/xgboost_abalone/abalone.py index e4a08c0436..5b0599d71f 100644 --- a/introduction_to_amazon_algorithms/xgboost_abalone/abalone.py +++ b/introduction_to_amazon_algorithms/xgboost_abalone/abalone.py @@ -16,14 +16,13 @@ import json import logging import os -import pandas as pd import pickle as pkl +import pandas as pd +import xgboost as xgb from sagemaker_containers import entry_point -from sagemaker_xgboost_container.data_utils import get_dmatrix from sagemaker_xgboost_container import distributed - -import xgboost as xgb +from sagemaker_xgboost_container.data_utils import get_dmatrix def _xgb_train(params, dtrain, evals, num_boost_round, model_dir, is_master): diff --git a/introduction_to_amazon_algorithms/xgboost_abalone/inference.py b/introduction_to_amazon_algorithms/xgboost_abalone/inference.py index acde502f06..bb00fb6955 100644 --- a/introduction_to_amazon_algorithms/xgboost_abalone/inference.py +++ b/introduction_to_amazon_algorithms/xgboost_abalone/inference.py @@ -15,7 +15,6 @@ import pickle as pkl import numpy as np - import sagemaker_xgboost_container.encoder as xgb_encoders diff --git a/introduction_to_applying_machine_learning/gluon_recommender_system/recommender.py b/introduction_to_applying_machine_learning/gluon_recommender_system/recommender.py index 7880e6a7d7..c9bf58bf9e 100644 --- a/introduction_to_applying_machine_learning/gluon_recommender_system/recommender.py +++ b/introduction_to_applying_machine_learning/gluon_recommender_system/recommender.py @@ -1,11 +1,12 @@ -import logging import json -import time +import logging import os +import time + import mxnet as mx +import numpy as np from mxnet import gluon, nd, ndarray from mxnet.metric import MSE -import numpy as np os.system("pip install pandas") import pandas as pd diff --git a/prep_data/image_data_guide/training_pytorch/pytorch_train.py b/prep_data/image_data_guide/training_pytorch/pytorch_train.py index 1798bc6e25..9ed5f8b253 100644 --- a/prep_data/image_data_guide/training_pytorch/pytorch_train.py +++ b/prep_data/image_data_guide/training_pytorch/pytorch_train.py @@ -1,14 +1,13 @@ -import os -import time +import argparse import copy -import shutil +import os import pathlib -import argparse +import shutil +import time import torch import torchvision as tv - # the training fuction is based off the sample training fuction provided # by Pytorch in their transfer learning tutorial: # https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html diff --git a/prep_data/image_data_guide/training_tensorflow/tensorflow_train.py b/prep_data/image_data_guide/training_tensorflow/tensorflow_train.py index 5dd230b792..8a589d606b 100644 --- a/prep_data/image_data_guide/training_tensorflow/tensorflow_train.py +++ b/prep_data/image_data_guide/training_tensorflow/tensorflow_train.py @@ -1,6 +1,7 @@ +import argparse import os import pathlib -import argparse + import tensorflow as tf import tensorflow_datasets as tfds diff --git a/reinforcement_learning/bandits_recsys_movielens_testbed/misc.py b/reinforcement_learning/bandits_recsys_movielens_testbed/misc.py index 8adda51671..5cec938528 100644 --- a/reinforcement_learning/bandits_recsys_movielens_testbed/misc.py +++ b/reinforcement_learning/bandits_recsys_movielens_testbed/misc.py @@ -15,16 +15,16 @@ import base64 import contextlib +import json import os -import time import shlex import shutil import subprocess import sys import tempfile +import time import boto3 -import json def wait_for_s3_object( diff --git a/reinforcement_learning/bandits_recsys_movielens_testbed/src/env.py b/reinforcement_learning/bandits_recsys_movielens_testbed/src/env.py index 65c96f2c62..c42ac6c245 100644 --- a/reinforcement_learning/bandits_recsys_movielens_testbed/src/env.py +++ b/reinforcement_learning/bandits_recsys_movielens_testbed/src/env.py @@ -1,4 +1,5 @@ import os + import numpy as np diff --git a/reinforcement_learning/bandits_recsys_movielens_testbed/src/io_utils.py b/reinforcement_learning/bandits_recsys_movielens_testbed/src/io_utils.py index 9a80fe1830..bbbe8f4cdc 100644 --- a/reinforcement_learning/bandits_recsys_movielens_testbed/src/io_utils.py +++ b/reinforcement_learning/bandits_recsys_movielens_testbed/src/io_utils.py @@ -1,9 +1,10 @@ import json import logging -import boto3 -from pathlib import Path -import shutil import os +import shutil +from pathlib import Path + +import boto3 import pandas as pd logger = logging.getLogger(__name__) diff --git a/reinforcement_learning/bandits_recsys_movielens_testbed/src/train.py b/reinforcement_learning/bandits_recsys_movielens_testbed/src/train.py index 59abacfad2..4b9f6a2006 100644 --- a/reinforcement_learning/bandits_recsys_movielens_testbed/src/train.py +++ b/reinforcement_learning/bandits_recsys_movielens_testbed/src/train.py @@ -5,14 +5,14 @@ from env import MovieLens100KEnv from io_utils import extract_model from vw_agent import VWAgent -from vw_utils import MODEL_CHANNEL, MODEL_OUTPUT_DIR, DATA_OUTPUT_DIR +from vw_utils import DATA_OUTPUT_DIR, MODEL_CHANNEL, MODEL_OUTPUT_DIR logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) def main(): - """ Train a Vowpal Wabbit (VW) model through C++ process. """ + """Train a Vowpal Wabbit (VW) model through C++ process.""" channel_names = json.loads(os.environ["SM_CHANNELS"]) hyperparameters = json.loads(os.environ["SM_HPS"]) diff --git a/reinforcement_learning/bandits_recsys_movielens_testbed/src/vw_agent.py b/reinforcement_learning/bandits_recsys_movielens_testbed/src/vw_agent.py index 6703659fce..fe6fab9d21 100644 --- a/reinforcement_learning/bandits_recsys_movielens_testbed/src/vw_agent.py +++ b/reinforcement_learning/bandits_recsys_movielens_testbed/src/vw_agent.py @@ -1,12 +1,13 @@ -import subprocess -import os +import json import logging +import os +import subprocess + import numpy as np -import json class VWError(Exception): - """ Class for errors """ + """Class for errors""" def __init__(self, message): super(VWError, self).__init__() @@ -14,7 +15,7 @@ def __init__(self, message): class VWModelDown(Exception): - """ When the model is down """ + """When the model is down""" def __init__(self): super(VWModelDown, self).__init__("The model is down") @@ -309,7 +310,7 @@ def load_model(metadata_loc, weights_loc, test_only=True, quiet_mode=True, outpu return VWAgent(**metadata) def save_model(self, close=False): - """Call to save metadata. close=True closes the VW process """ + """Call to save metadata. close=True closes the VW process""" metadata_file = os.path.join(self.output_dir, "vw.metadata") with open(metadata_file, "w") as f: f.write(json.dumps(self.args)) diff --git a/reinforcement_learning/bandits_recsys_movielens_testbed/src/vw_utils.py b/reinforcement_learning/bandits_recsys_movielens_testbed/src/vw_utils.py index 7b0ea0d065..232ea6cf2d 100644 --- a/reinforcement_learning/bandits_recsys_movielens_testbed/src/vw_utils.py +++ b/reinforcement_learning/bandits_recsys_movielens_testbed/src/vw_utils.py @@ -1,5 +1,5 @@ -import os import json +import os TRAIN_CHANNEL = "training" EVAL_CHANNEL = "evaluation" diff --git a/reinforcement_learning/bandits_statlog_vw_customEnv/sim_app/sim_app_utils.py b/reinforcement_learning/bandits_statlog_vw_customEnv/sim_app/sim_app_utils.py index f10e445950..88d5f43c20 100644 --- a/reinforcement_learning/bandits_statlog_vw_customEnv/sim_app/sim_app_utils.py +++ b/reinforcement_learning/bandits_statlog_vw_customEnv/sim_app/sim_app_utils.py @@ -1,8 +1,8 @@ -from statlog_sim_app import remove_underrepresented_classes, classification_to_bandit_problem +import boto3 import numpy as np import pandas as pd -import boto3 from src.io_utils import parse_s3_uri +from statlog_sim_app import classification_to_bandit_problem, remove_underrepresented_classes def prepare_statlog_warm_start_data(data_file, batch_size): diff --git a/reinforcement_learning/bandits_statlog_vw_customEnv/sim_app/statlog_sim_app.py b/reinforcement_learning/bandits_statlog_vw_customEnv/sim_app/statlog_sim_app.py index c1f0fa25f0..17d427d61c 100644 --- a/reinforcement_learning/bandits_statlog_vw_customEnv/sim_app/statlog_sim_app.py +++ b/reinforcement_learning/bandits_statlog_vw_customEnv/sim_app/statlog_sim_app.py @@ -1,12 +1,13 @@ -import pandas as pd +import datetime +import io +import json import time import uuid -import boto3 from urllib.parse import urlparse -import datetime -import json -import io + +import boto3 import numpy as np +import pandas as pd def remove_underrepresented_classes(features, labels, thresh=0.0005): diff --git a/reinforcement_learning/bandits_statlog_vw_customEnv/src/eval-cfa-vw.py b/reinforcement_learning/bandits_statlog_vw_customEnv/src/eval-cfa-vw.py index ccdc63c0fc..494bdcbb32 100644 --- a/reinforcement_learning/bandits_statlog_vw_customEnv/src/eval-cfa-vw.py +++ b/reinforcement_learning/bandits_statlog_vw_customEnv/src/eval-cfa-vw.py @@ -1,13 +1,12 @@ import argparse import json +import logging import os from pathlib import Path -import logging -import numpy as np +import numpy as np +from io_utils import CSVReader, download_manifest_data, extract_model, validate_experience from vw_model import VWModel - -from io_utils import extract_model, CSVReader, validate_experience, download_manifest_data from vw_utils import EVAL_CHANNEL, MODEL_CHANNEL logging.basicConfig(level=logging.INFO) diff --git a/reinforcement_learning/bandits_statlog_vw_customEnv/src/io_utils.py b/reinforcement_learning/bandits_statlog_vw_customEnv/src/io_utils.py index e605f33ab2..6073e0412f 100644 --- a/reinforcement_learning/bandits_statlog_vw_customEnv/src/io_utils.py +++ b/reinforcement_learning/bandits_statlog_vw_customEnv/src/io_utils.py @@ -1,9 +1,10 @@ import json import logging -import boto3 -from pathlib import Path -import shutil import os +import shutil +from pathlib import Path + +import boto3 import pandas as pd logger = logging.getLogger(__name__) diff --git a/reinforcement_learning/bandits_statlog_vw_customEnv/src/train-vw.py b/reinforcement_learning/bandits_statlog_vw_customEnv/src/train-vw.py index f6e0b5469e..ab22be8950 100644 --- a/reinforcement_learning/bandits_statlog_vw_customEnv/src/train-vw.py +++ b/reinforcement_learning/bandits_statlog_vw_customEnv/src/train-vw.py @@ -1,18 +1,17 @@ import argparse import json +import logging import os from pathlib import Path -import logging +from io_utils import CSVReader, extract_model, validate_experience from vw_model import VWModel - -from io_utils import extract_model, CSVReader, validate_experience from vw_utils import ( - TRAIN_CHANNEL, MODEL_CHANNEL, MODEL_OUTPUT_PATH, - save_vw_model, + TRAIN_CHANNEL, save_vw_metadata, + save_vw_model, ) logging.basicConfig(level=logging.DEBUG) @@ -20,7 +19,7 @@ def main(): - """ Train a Vowpal Wabbit (VW) model through C++ process. """ + """Train a Vowpal Wabbit (VW) model through C++ process.""" channel_names = json.loads(os.environ["SM_CHANNELS"]) hyperparameters = json.loads(os.environ["SM_HPS"]) diff --git a/reinforcement_learning/bandits_statlog_vw_customEnv/src/vw_model.py b/reinforcement_learning/bandits_statlog_vw_customEnv/src/vw_model.py index ee755d846e..046f02a1ff 100644 --- a/reinforcement_learning/bandits_statlog_vw_customEnv/src/vw_model.py +++ b/reinforcement_learning/bandits_statlog_vw_customEnv/src/vw_model.py @@ -1,6 +1,7 @@ -import subprocess -import os import logging +import os +import subprocess + import numpy as np logger = logging.getLogger("vw_model.VWModel") @@ -8,7 +9,7 @@ class VWError(Exception): - """ Class for errors """ + """Class for errors""" def __init__(self, message): super(VWError, self).__init__() @@ -16,7 +17,7 @@ def __init__(self, message): class VWModelDown(Exception): - """ When the model is down """ + """When the model is down""" def __init__(self): super(VWModelDown, self).__init__("The model is down") diff --git a/reinforcement_learning/bandits_statlog_vw_customEnv/src/vw_utils.py b/reinforcement_learning/bandits_statlog_vw_customEnv/src/vw_utils.py index ffe1a2d04b..3e42cb4714 100644 --- a/reinforcement_learning/bandits_statlog_vw_customEnv/src/vw_utils.py +++ b/reinforcement_learning/bandits_statlog_vw_customEnv/src/vw_utils.py @@ -1,5 +1,5 @@ -import os import json +import os TRAIN_CHANNEL = "training" EVAL_CHANNEL = "evaluation" diff --git a/reinforcement_learning/common/docker_utils.py b/reinforcement_learning/common/docker_utils.py index 5487032802..f5c3486dfa 100644 --- a/reinforcement_learning/common/docker_utils.py +++ b/reinforcement_learning/common/docker_utils.py @@ -15,16 +15,16 @@ import base64 import contextlib +import json import os -import time import shlex import shutil import subprocess import sys import tempfile +import time import boto3 -import json IMAGE_TEMPLATE = "{account}.dkr.ecr.{region}.amazonaws.com/{image_name}:{version}" diff --git a/reinforcement_learning/common/env_utils.py b/reinforcement_learning/common/env_utils.py index 8b812445bd..1dd37f0956 100644 --- a/reinforcement_learning/common/env_utils.py +++ b/reinforcement_learning/common/env_utils.py @@ -1,8 +1,9 @@ +import json +from pathlib import Path + import gym import numpy as np import pandas as pd -import json -from pathlib import Path gym.logger.set_level(40) diff --git a/reinforcement_learning/common/misc.py b/reinforcement_learning/common/misc.py index 1ec1d21a01..6a48b3a284 100644 --- a/reinforcement_learning/common/misc.py +++ b/reinforcement_learning/common/misc.py @@ -15,16 +15,16 @@ import base64 import contextlib +import json import os -import time import shlex import shutil import subprocess import sys import tempfile +import time import boto3 -import json def wait_for_s3_object( diff --git a/reinforcement_learning/common/sagemaker_rl/coach_launcher.py b/reinforcement_learning/common/sagemaker_rl/coach_launcher.py index 4e822e44bc..f1644a183d 100644 --- a/reinforcement_learning/common/sagemaker_rl/coach_launcher.py +++ b/reinforcement_learning/common/sagemaker_rl/coach_launcher.py @@ -1,24 +1,25 @@ -from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters -from rl_coach.agents.policy_gradients_agent import PolicyGradientsAgentParameters -from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager -from rl_coach.graph_managers.graph_manager import ScheduleParameters -from rl_coach.base_parameters import VisualizationParameters, TaskParameters, Frameworks -from rl_coach.utils import short_dynamic_import -from rl_coach.core_types import SelectedPhaseOnlyDumpFilter, MaxDumpFilter, RunPhase -import rl_coach.core_types -from rl_coach import logger -from rl_coach.logger import screen import argparse import copy +import glob import logging import os -import sys -import shutil -import glob import re +import shutil +import sys -from .configuration_list import ConfigurationList +import rl_coach.core_types +from rl_coach import logger +from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters +from rl_coach.agents.policy_gradients_agent import PolicyGradientsAgentParameters +from rl_coach.base_parameters import Frameworks, TaskParameters, VisualizationParameters from rl_coach.coach import CoachLauncher +from rl_coach.core_types import MaxDumpFilter, RunPhase, SelectedPhaseOnlyDumpFilter +from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager +from rl_coach.graph_managers.graph_manager import ScheduleParameters +from rl_coach.logger import screen +from rl_coach.utils import short_dynamic_import + +from .configuration_list import ConfigurationList screen.set_use_colors(False) # Simple text logging so it looks good in CloudWatch diff --git a/reinforcement_learning/common/sagemaker_rl/mpi_launcher.py b/reinforcement_learning/common/sagemaker_rl/mpi_launcher.py index e42ca5ad30..94ef785fa5 100644 --- a/reinforcement_learning/common/sagemaker_rl/mpi_launcher.py +++ b/reinforcement_learning/common/sagemaker_rl/mpi_launcher.py @@ -1,19 +1,15 @@ -import stat - +import os import shlex +import signal import socket +import stat +import subprocess import sys - import textwrap -from contextlib import contextmanager -import signal - -import os -import subprocess import time +from contextlib import contextmanager import sagemaker_containers - from retrying import retry from sagemaker_containers import _logging from sagemaker_containers.beta import framework @@ -204,7 +200,7 @@ def is_master(self, hosts, current_host): class MPIWorker(object): - """ MPI Worker""" + """MPI Worker""" @retry( stop_max_delay=30000 * 1000, wait_fixed=1000, retry_on_result=lambda result: result is False diff --git a/reinforcement_learning/common/sagemaker_rl/onnx_utils.py b/reinforcement_learning/common/sagemaker_rl/onnx_utils.py index 120eecb51c..1ade41a494 100644 --- a/reinforcement_learning/common/sagemaker_rl/onnx_utils.py +++ b/reinforcement_learning/common/sagemaker_rl/onnx_utils.py @@ -2,7 +2,7 @@ ONNX Utils to support multiple output heads in agent networks, until future releases of MXNet support this. """ import onnx -from onnx import helper, checker, TensorProto +from onnx import TensorProto, checker, helper def get_correct_outputs(model): diff --git a/reinforcement_learning/common/sagemaker_rl/orchestrator/clients/ddb/experiment_db_client.py b/reinforcement_learning/common/sagemaker_rl/orchestrator/clients/ddb/experiment_db_client.py index 75f7435dc7..29dc477945 100644 --- a/reinforcement_learning/common/sagemaker_rl/orchestrator/clients/ddb/experiment_db_client.py +++ b/reinforcement_learning/common/sagemaker_rl/orchestrator/clients/ddb/experiment_db_client.py @@ -1,4 +1,5 @@ import logging + from boto3.dynamodb.conditions import Key from orchestrator.exceptions.ddb_client_exceptions import RecordAlreadyExistsException diff --git a/reinforcement_learning/common/sagemaker_rl/orchestrator/clients/ddb/join_db_client.py b/reinforcement_learning/common/sagemaker_rl/orchestrator/clients/ddb/join_db_client.py index 8e3ebb8111..a8de8756e1 100644 --- a/reinforcement_learning/common/sagemaker_rl/orchestrator/clients/ddb/join_db_client.py +++ b/reinforcement_learning/common/sagemaker_rl/orchestrator/clients/ddb/join_db_client.py @@ -1,4 +1,5 @@ import logging + from boto3.dynamodb.conditions import Key from orchestrator.exceptions.ddb_client_exceptions import RecordAlreadyExistsException diff --git a/reinforcement_learning/common/sagemaker_rl/orchestrator/resource_manager.py b/reinforcement_learning/common/sagemaker_rl/orchestrator/resource_manager.py index e6d090c69b..153ff60d2f 100644 --- a/reinforcement_learning/common/sagemaker_rl/orchestrator/resource_manager.py +++ b/reinforcement_learning/common/sagemaker_rl/orchestrator/resource_manager.py @@ -4,16 +4,13 @@ import boto3 import sagemaker -from botocore.exceptions import ClientError -from sagemaker.local.local_session import LocalSession - -from orchestrator.exceptions.ddb_client_exceptions import RecordAlreadyExistsException - from boto3.dynamodb.conditions import Key - +from botocore.exceptions import ClientError from orchestrator.clients.ddb.experiment_db_client import ExperimentDbClient from orchestrator.clients.ddb.join_db_client import JoinDbClient from orchestrator.clients.ddb.model_db_client import ModelDbClient +from orchestrator.exceptions.ddb_client_exceptions import RecordAlreadyExistsException +from sagemaker.local.local_session import LocalSession logger = logging.getLogger(__name__) diff --git a/reinforcement_learning/common/sagemaker_rl/orchestrator/utils/cloudwatch_logger.py b/reinforcement_learning/common/sagemaker_rl/orchestrator/utils/cloudwatch_logger.py index 0428d26617..afd8b6212d 100644 --- a/reinforcement_learning/common/sagemaker_rl/orchestrator/utils/cloudwatch_logger.py +++ b/reinforcement_learning/common/sagemaker_rl/orchestrator/utils/cloudwatch_logger.py @@ -1,5 +1,5 @@ -import time import json +import time class CloudWatchLogger: diff --git a/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/experiment_manager.py b/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/experiment_manager.py index c0854a8a48..7a618b1916 100644 --- a/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/experiment_manager.py +++ b/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/experiment_manager.py @@ -3,13 +3,12 @@ import time from datetime import datetime, timedelta from enum import Enum -from threading import Thread, Event -from packaging import version +from threading import Event, Thread import boto3 import docker - import sagemaker +from packaging import version logging.basicConfig() logger = logging.getLogger("orchestrator") @@ -30,26 +29,24 @@ raise e from botocore.exceptions import ClientError -from sagemaker.local.local_session import LocalSession - +from orchestrator.clients.ddb.experiment_db_client import ExperimentDbClient from orchestrator.clients.ddb.join_db_client import JoinDbClient from orchestrator.clients.ddb.model_db_client import ModelDbClient -from orchestrator.clients.ddb.experiment_db_client import ExperimentDbClient -from orchestrator.workflow.manager.join_manager import JoinManager -from orchestrator.workflow.manager.model_manager import ModelManager -from orchestrator.workflow.datatypes.experiment_record import ExperimentRecord -from orchestrator.resource_manager import Predictor -from orchestrator.resource_manager import ResourceManager -from orchestrator.utils.cloudwatch_logger import CloudWatchLogger from orchestrator.exceptions.ddb_client_exceptions import RecordAlreadyExistsException from orchestrator.exceptions.workflow_exceptions import ( - UnhandledWorkflowException, + EvalScoreNotAvailableException, + InvalidUsageException, SageMakerHostingException, SageMakerTrainingJobException, + UnhandledWorkflowException, WorkflowJoiningJobException, - EvalScoreNotAvailableException, - InvalidUsageException, ) +from orchestrator.resource_manager import Predictor, ResourceManager +from orchestrator.utils.cloudwatch_logger import CloudWatchLogger +from orchestrator.workflow.datatypes.experiment_record import ExperimentRecord +from orchestrator.workflow.manager.join_manager import JoinManager +from orchestrator.workflow.manager.model_manager import ModelManager +from sagemaker.local.local_session import LocalSession class HostingState(str, Enum): diff --git a/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/join_manager.py b/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/join_manager.py index 15acdb6ced..176bc87f9b 100644 --- a/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/join_manager.py +++ b/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/join_manager.py @@ -1,20 +1,21 @@ -import boto3 +import io +import json import logging import os -import io -import time import re -import json +import time from datetime import datetime, timedelta from threading import Thread + +import boto3 from botocore.exceptions import ClientError from orchestrator.clients.ddb.join_db_client import JoinDbClient -from orchestrator.workflow.datatypes.join_job_record import JoinJobRecord from orchestrator.exceptions.ddb_client_exceptions import RecordAlreadyExistsException from orchestrator.exceptions.workflow_exceptions import ( - UnhandledWorkflowException, JoinQueryIdsNotAvailableException, + UnhandledWorkflowException, ) +from orchestrator.workflow.datatypes.join_job_record import JoinJobRecord logger = logging.getLogger("orchestrator") diff --git a/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/model_manager.py b/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/model_manager.py index af9fe4f884..7cce7d6838 100644 --- a/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/model_manager.py +++ b/reinforcement_learning/common/sagemaker_rl/orchestrator/workflow/manager/model_manager.py @@ -1,24 +1,22 @@ +import json +import logging +import re +import sys import time from enum import Enum from io import StringIO -import re -import sys +from threading import Thread -import sagemaker import boto3 -import json -import logging -from threading import Thread -from sagemaker.rl.estimator import RLEstimator -from sagemaker.local.local_session import LocalSession -from sagemaker.analytics import TrainingJobAnalytics +import sagemaker from botocore.exceptions import ClientError - from orchestrator.clients.ddb.model_db_client import ModelDbClient -from orchestrator.workflow.datatypes.model_record import ModelRecord from orchestrator.exceptions.ddb_client_exceptions import RecordAlreadyExistsException from orchestrator.exceptions.workflow_exceptions import UnhandledWorkflowException - +from orchestrator.workflow.datatypes.model_record import ModelRecord +from sagemaker.analytics import TrainingJobAnalytics +from sagemaker.local.local_session import LocalSession +from sagemaker.rl.estimator import RLEstimator from src.vw_utils import EVAL_CHANNEL logger = logging.getLogger("orchestrator") diff --git a/reinforcement_learning/common/sagemaker_rl/ray_launcher.py b/reinforcement_learning/common/sagemaker_rl/ray_launcher.py index 07f85c9886..11400e4d9b 100644 --- a/reinforcement_learning/common/sagemaker_rl/ray_launcher.py +++ b/reinforcement_learning/common/sagemaker_rl/ray_launcher.py @@ -3,17 +3,17 @@ import subprocess import sys import time -from shutil import copyfile from enum import Enum +from shutil import copyfile import boto3 - import ray from ray.tune import run_experiments -from .tf_serving_utils import export_tf_serving, natural_keys, change_permissions_recursive + from .configuration_list import ConfigurationList -from .sage_cluster_communicator import SageClusterCommunicator from .docker_utils import get_ip_from_host +from .sage_cluster_communicator import SageClusterCommunicator +from .tf_serving_utils import change_permissions_recursive, export_tf_serving, natural_keys TERMINATION_SIGNAL = "JOB_TERMINATED" INTERMEDIATE_DIR = "/opt/ml/output/intermediate" diff --git a/reinforcement_learning/common/sagemaker_rl/sage_cluster_communicator.py b/reinforcement_learning/common/sagemaker_rl/sage_cluster_communicator.py index 7ff115e616..558b38bec2 100644 --- a/reinforcement_learning/common/sagemaker_rl/sage_cluster_communicator.py +++ b/reinforcement_learning/common/sagemaker_rl/sage_cluster_communicator.py @@ -1,9 +1,10 @@ -import boto3 -import os import io import json +import os import time +import boto3 + class SageClusterCommunicator: def __init__(self): diff --git a/reinforcement_learning/common/sagemaker_rl/stable_baselines_launcher.py b/reinforcement_learning/common/sagemaker_rl/stable_baselines_launcher.py index ef9bf72f13..76d6eb9564 100644 --- a/reinforcement_learning/common/sagemaker_rl/stable_baselines_launcher.py +++ b/reinforcement_learning/common/sagemaker_rl/stable_baselines_launcher.py @@ -1,14 +1,13 @@ -import gym -import roboschool import os +import gym +import roboschool from gym.wrappers.monitoring.video_recorder import VideoRecorder -from stable_baselines.ppo1 import PPO1 -from stable_baselines.common import set_global_seeds +from mpi4py import MPI from stable_baselines.bench import Monitor -from stable_baselines.common import tf_util +from stable_baselines.common import set_global_seeds, tf_util from stable_baselines.common.policies import MlpPolicy -from mpi4py import MPI +from stable_baselines.ppo1 import PPO1 class RewScale(gym.RewardWrapper): diff --git a/reinforcement_learning/common/sagemaker_rl/tf_serving_utils.py b/reinforcement_learning/common/sagemaker_rl/tf_serving_utils.py index 545c671c05..4bb9448fd0 100644 --- a/reinforcement_learning/common/sagemaker_rl/tf_serving_utils.py +++ b/reinforcement_learning/common/sagemaker_rl/tf_serving_utils.py @@ -1,7 +1,8 @@ -import ray import os import re +import ray + try: from ray.rllib.utils.framework import try_import_tf diff --git a/reinforcement_learning/common/tests/unit/test_ray_launcher.py b/reinforcement_learning/common/tests/unit/test_ray_launcher.py index 3b4cba25c5..4529c49130 100644 --- a/reinforcement_learning/common/tests/unit/test_ray_launcher.py +++ b/reinforcement_learning/common/tests/unit/test_ray_launcher.py @@ -11,9 +11,9 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import absolute_import -import pytest -from mock import Mock, MagicMock, patch +import pytest +from mock import MagicMock, Mock, patch from sagemaker_rl.ray_launcher import SageMakerRayLauncher diff --git a/reinforcement_learning/rl_cartpole_batch_coach/src/evaluate-coach.py b/reinforcement_learning/rl_cartpole_batch_coach/src/evaluate-coach.py index 3bf2b48615..0995f9dd59 100644 --- a/reinforcement_learning/rl_cartpole_batch_coach/src/evaluate-coach.py +++ b/reinforcement_learning/rl_cartpole_batch_coach/src/evaluate-coach.py @@ -1,9 +1,10 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher, CoachConfigurationList import argparse import os + import rl_coach from rl_coach.base_parameters import Frameworks, TaskParameters from rl_coach.core_types import EnvironmentSteps +from sagemaker_rl.coach_launcher import CoachConfigurationList, SageMakerCoachPresetLauncher def inplace_replace_in_file(filepath, old, new): diff --git a/reinforcement_learning/rl_cartpole_batch_coach/src/preset-cartpole-ddqnbcq-env.py b/reinforcement_learning/rl_cartpole_batch_coach/src/preset-cartpole-ddqnbcq-env.py index 62b3332f31..65a0c4970f 100644 --- a/reinforcement_learning/rl_cartpole_batch_coach/src/preset-cartpole-ddqnbcq-env.py +++ b/reinforcement_learning/rl_cartpole_batch_coach/src/preset-cartpole-ddqnbcq-env.py @@ -1,34 +1,33 @@ from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters +from rl_coach.agents.ddqn_bcq_agent import DDQNBCQAgentParameters, KNNParameters from rl_coach.agents.dqn_agent import DQNAgentParameters -from rl_coach.agents.ddqn_bcq_agent import DDQNBCQAgentParameters from rl_coach.architectures.layers import Dense from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, DistributedCoachSynchronizationType, + PresetValidationParameters, + VisualizationParameters, ) from rl_coach.core_types import ( - TrainingSteps, + CsvDataset, EnvironmentEpisodes, EnvironmentSteps, RunPhase, - CsvDataset, + TrainingSteps, ) from rl_coach.environments.gym_environment import GymVectorEnvironment, mujoco_v2 from rl_coach.exploration_policies.e_greedy import EGreedyParameters from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager from rl_coach.graph_managers.batch_rl_graph_manager import BatchRLGraphManager from rl_coach.graph_managers.graph_manager import ScheduleParameters +from rl_coach.memories.episodic import EpisodicExperienceReplayParameters from rl_coach.memories.memory import MemoryGranularity from rl_coach.schedules import LinearSchedule -from rl_coach.memories.episodic import EpisodicExperienceReplayParameters -from rl_coach.agents.ddqn_bcq_agent import KNNParameters from rl_coach.spaces import ( - SpacesDefinition, DiscreteActionSpace, - VectorObservationSpace, - StateSpace, RewardSpace, + SpacesDefinition, + StateSpace, + VectorObservationSpace, ) # #################### diff --git a/reinforcement_learning/rl_cartpole_batch_coach/src/preset-cartpole-ddqnbcq.py b/reinforcement_learning/rl_cartpole_batch_coach/src/preset-cartpole-ddqnbcq.py index 49744504af..b6cabf5de7 100644 --- a/reinforcement_learning/rl_cartpole_batch_coach/src/preset-cartpole-ddqnbcq.py +++ b/reinforcement_learning/rl_cartpole_batch_coach/src/preset-cartpole-ddqnbcq.py @@ -1,23 +1,22 @@ -from rl_coach.agents.ddqn_bcq_agent import DDQNBCQAgentParameters -from rl_coach.base_parameters import VisualizationParameters, PresetValidationParameters +from rl_coach.agents.ddqn_bcq_agent import DDQNBCQAgentParameters, KNNParameters +from rl_coach.base_parameters import PresetValidationParameters, VisualizationParameters from rl_coach.core_types import ( - TrainingSteps, + CsvDataset, EnvironmentEpisodes, EnvironmentSteps, RunPhase, - CsvDataset, + TrainingSteps, ) from rl_coach.graph_managers.batch_rl_graph_manager import BatchRLGraphManager from rl_coach.graph_managers.graph_manager import ScheduleParameters -from rl_coach.schedules import LinearSchedule from rl_coach.memories.episodic import EpisodicExperienceReplayParameters -from rl_coach.agents.ddqn_bcq_agent import KNNParameters +from rl_coach.schedules import LinearSchedule from rl_coach.spaces import ( - SpacesDefinition, DiscreteActionSpace, - VectorObservationSpace, - StateSpace, RewardSpace, + SpacesDefinition, + StateSpace, + VectorObservationSpace, ) # #################### diff --git a/reinforcement_learning/rl_cartpole_batch_coach/src/train-coach.py b/reinforcement_learning/rl_cartpole_batch_coach/src/train-coach.py index 73cd345771..9d31fc61c4 100644 --- a/reinforcement_learning/rl_cartpole_batch_coach/src/train-coach.py +++ b/reinforcement_learning/rl_cartpole_batch_coach/src/train-coach.py @@ -1,6 +1,7 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher import shutil +from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher + class MyLauncher(SageMakerCoachPresetLauncher): def default_preset_name(self): diff --git a/reinforcement_learning/rl_cartpole_coach/src/deploy-mxnet-coach.py b/reinforcement_learning/rl_cartpole_coach/src/deploy-mxnet-coach.py index 7f0439d92b..5f93328ca8 100644 --- a/reinforcement_learning/rl_cartpole_coach/src/deploy-mxnet-coach.py +++ b/reinforcement_learning/rl_cartpole_coach/src/deploy-mxnet-coach.py @@ -1,9 +1,10 @@ -import os import json +import os + import mxnet as mx -from mxnet.contrib import onnx as onnx_mxnet -from mxnet import gluon, nd import numpy as np +from mxnet import gluon, nd +from mxnet.contrib import onnx as onnx_mxnet def model_fn(model_dir): diff --git a/reinforcement_learning/rl_cartpole_coach/src/evaluate-coach.py b/reinforcement_learning/rl_cartpole_coach/src/evaluate-coach.py index 3bf2b48615..0995f9dd59 100644 --- a/reinforcement_learning/rl_cartpole_coach/src/evaluate-coach.py +++ b/reinforcement_learning/rl_cartpole_coach/src/evaluate-coach.py @@ -1,9 +1,10 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher, CoachConfigurationList import argparse import os + import rl_coach from rl_coach.base_parameters import Frameworks, TaskParameters from rl_coach.core_types import EnvironmentSteps +from sagemaker_rl.coach_launcher import CoachConfigurationList, SageMakerCoachPresetLauncher def inplace_replace_in_file(filepath, old, new): diff --git a/reinforcement_learning/rl_cartpole_coach/src/preset-cartpole-clippedppo.py b/reinforcement_learning/rl_cartpole_coach/src/preset-cartpole-clippedppo.py index b80e7fbdb2..bfa308765c 100644 --- a/reinforcement_learning/rl_cartpole_coach/src/preset-cartpole-clippedppo.py +++ b/reinforcement_learning/rl_cartpole_coach/src/preset-cartpole-clippedppo.py @@ -1,11 +1,11 @@ from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters from rl_coach.architectures.layers import Dense from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, DistributedCoachSynchronizationType, + PresetValidationParameters, + VisualizationParameters, ) -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps, RunPhase +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, RunPhase, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment, mujoco_v2 from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters from rl_coach.exploration_policies.e_greedy import EGreedyParameters diff --git a/reinforcement_learning/rl_cartpole_coach/src/preset-cartpole-dqn.py b/reinforcement_learning/rl_cartpole_coach/src/preset-cartpole-dqn.py index 1f0dcc92fe..4a92448e38 100644 --- a/reinforcement_learning/rl_cartpole_coach/src/preset-cartpole-dqn.py +++ b/reinforcement_learning/rl_cartpole_coach/src/preset-cartpole-dqn.py @@ -1,10 +1,10 @@ from rl_coach.agents.dqn_agent import DQNAgentParameters from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, DistributedCoachSynchronizationType, + PresetValidationParameters, + VisualizationParameters, ) -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager from rl_coach.graph_managers.graph_manager import ScheduleParameters diff --git a/reinforcement_learning/rl_cartpole_ray/src/evaluate-ray.py b/reinforcement_learning/rl_cartpole_ray/src/evaluate-ray.py index e141a09b29..6b8b011589 100644 --- a/reinforcement_learning/rl_cartpole_ray/src/evaluate-ray.py +++ b/reinforcement_learning/rl_cartpole_ray/src/evaluate-ray.py @@ -1,16 +1,13 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import argparse import json import os -import numpy as np - import gym -from gym import wrappers +import numpy as np import ray +from gym import wrappers from ray.rllib.models import ModelCatalog from ray.tune.registry import register_env diff --git a/reinforcement_learning/rl_cartpole_ray/src/train-rl-cartpole-ray.py b/reinforcement_learning/rl_cartpole_ray/src/train-rl-cartpole-ray.py index a9ca1dd0ae..e6045f62d6 100644 --- a/reinforcement_learning/rl_cartpole_ray/src/train-rl-cartpole-ray.py +++ b/reinforcement_learning/rl_cartpole_ray/src/train-rl-cartpole-ray.py @@ -5,7 +5,6 @@ import ray from ray.tune import run_experiments from ray.tune.registry import register_env - from sagemaker_rl.ray_launcher import SageMakerRayLauncher diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/copy_to_sagemaker_container.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/copy_to_sagemaker_container.py index 04fb1341b3..6e7dc6abe4 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/copy_to_sagemaker_container.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/copy_to_sagemaker_container.py @@ -1,4 +1,5 @@ import subprocess + import boto3 SAGEMAKER_DOCKER_MARKOV_PATH = "/opt/amazon/markov" diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/sim_app_bundler.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/sim_app_bundler.py index 8a09d7b8e0..852cba15a3 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/sim_app_bundler.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/sim_app_bundler.py @@ -15,11 +15,11 @@ # Clean the build python3 sim_app_bundler.py --clean """ +import argparse import os +import shutil import subprocess import tarfile -import argparse -import shutil UNTARRED_SIM_APP_OUTPUT_PATH = "build/simapp/" BUILD_PATH = "build" diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/model_validator/response_handlers.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/model_validator/response_handlers.py index 2fe53fe609..249e424220 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/model_validator/response_handlers.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/model_validator/response_handlers.py @@ -1,5 +1,6 @@ -import logging import json +import logging + import flask logger = logging.getLogger(__name__) diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/model_validator/validator.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/model_validator/validator.py index 422172537d..952834c070 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/model_validator/validator.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/model_validator/validator.py @@ -2,14 +2,15 @@ from gevent import monkey monkey.patch_all() +import logging import os -import flask import shutil -import logging -from flask import request, json -from .utils import extract_custom_attributes, run_cmd +import flask +from flask import json, request + from .response_handlers import ResponseHandler +from .utils import extract_custom_attributes, run_cmd response_handler = ResponseHandler diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/serve b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/serve index ffe1563dcd..6cbf4e2823 100755 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/serve +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/lib/serve @@ -15,16 +15,16 @@ # timeout MODEL_SERVER_TIMEOUT workers_count * 10 seconds from __future__ import print_function + from gevent import monkey, wait + monkey.patch_all() +import math import multiprocessing import os import signal import subprocess import sys -import math - - cpu_count = multiprocessing.cpu_count() workers_count = math.ceil(cpu_count * 0.75) # https://docs.gunicorn.org/en/stable/design.html diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/bot_cars_agent_ctrl.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/bot_cars_agent_ctrl.py index 297a3a44d4..542cb69439 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/bot_cars_agent_ctrl.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/bot_cars_agent_ctrl.py @@ -1,35 +1,33 @@ """This module implements concrete agent controllers for the rollout worker""" import math -import numpy as np import os import random -import rospkg -import rospy import threading +import numpy as np +import rospkg +import rospy from gazebo_msgs.msg import ModelState from gazebo_msgs.srv import SpawnModel from geometry_msgs.msg import Pose - +from markov import utils +from markov.agent_ctrl.agent_ctrl_interface import AgentCtrlInterface from markov.agent_ctrl.constants import BOT_CAR_Z from markov.agent_ctrl.utils import get_normalized_progress -from markov.track_geom.constants import SPAWN_SDF_MODEL, ObstacleDimensions, TrackLane -from markov.track_geom.track_data import TrackData -from markov.track_geom.spline.track_spline import TrackSpline -from markov.track_geom.spline.lane_change_spline import LaneChangeSpline -from markov.track_geom.utils import euler_to_quaternion -from markov.agent_ctrl.agent_ctrl_interface import AgentCtrlInterface -from markov.rospy_wrappers import ServiceProxyWrapper -from markov import utils -from markov.reset.constants import AgentPhase, AgentInfo -from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.domain_randomizations.constants import ModelRandomizerType from markov.domain_randomizations.randomizer_manager import RandomizerManager from markov.domain_randomizations.visual.model_visual_randomizer import ModelVisualRandomizer -from markov.domain_randomizations.constants import ModelRandomizerType -from markov.gazebo_tracker.trackers.set_model_state_tracker import SetModelStateTracker from markov.gazebo_tracker.abs_tracker import AbstractTracker from markov.gazebo_tracker.constants import TrackerPriority - +from markov.gazebo_tracker.trackers.set_model_state_tracker import SetModelStateTracker +from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.reset.constants import AgentInfo, AgentPhase +from markov.rospy_wrappers import ServiceProxyWrapper +from markov.track_geom.constants import SPAWN_SDF_MODEL, ObstacleDimensions, TrackLane +from markov.track_geom.spline.lane_change_spline import LaneChangeSpline +from markov.track_geom.spline.track_spline import TrackSpline +from markov.track_geom.track_data import TrackData +from markov.track_geom.utils import euler_to_quaternion from shapely.geometry import Point diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/obstacles_agent_ctrl.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/obstacles_agent_ctrl.py index 80a858471c..5bfb93542c 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/obstacles_agent_ctrl.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/obstacles_agent_ctrl.py @@ -1,23 +1,23 @@ """This module implements concrete agent controllers for the rollout worker""" -import numpy as np import os import random + +import numpy as np import rospkg import rospy - from gazebo_msgs.msg import ModelState from gazebo_msgs.srv import SpawnModel -from markov.agent_ctrl.constants import BOT_CAR_Z, OBSTACLE_Z, OBSTACLE_NAME_PREFIX -from markov.track_geom.constants import SPAWN_SDF_MODEL, SPAWN_URDF_MODEL, ObstacleDimensions -from markov.track_geom.track_data import TrackData -from markov.agent_ctrl.agent_ctrl_interface import AgentCtrlInterface -from markov.rospy_wrappers import ServiceProxyWrapper from markov import utils -from markov.reset.constants import AgentInfo +from markov.agent_ctrl.agent_ctrl_interface import AgentCtrlInterface +from markov.agent_ctrl.constants import BOT_CAR_Z, OBSTACLE_NAME_PREFIX, OBSTACLE_Z +from markov.domain_randomizations.constants import ModelRandomizerType from markov.domain_randomizations.randomizer_manager import RandomizerManager from markov.domain_randomizations.visual.model_visual_randomizer import ModelVisualRandomizer -from markov.domain_randomizations.constants import ModelRandomizerType from markov.gazebo_tracker.trackers.set_model_state_tracker import SetModelStateTracker +from markov.reset.constants import AgentInfo +from markov.rospy_wrappers import ServiceProxyWrapper +from markov.track_geom.constants import SPAWN_SDF_MODEL, SPAWN_URDF_MODEL, ObstacleDimensions +from markov.track_geom.track_data import TrackData class ObstaclesCtrl(AgentCtrlInterface): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/rollout_agent_ctrl.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/rollout_agent_ctrl.py index b32ae60bde..24abbd6822 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/rollout_agent_ctrl.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/rollout_agent_ctrl.py @@ -1,64 +1,63 @@ """This module implements concrete agent controllers for the rollout worker""" import copy +import json +import logging +import math import time from collections import OrderedDict -import math -import numpy as np -import rospy -import logging -import json from threading import RLock -from gazebo_msgs.msg import ModelState -from std_msgs.msg import Float64, String -from shapely.geometry import Point import markov.agent_ctrl.constants as const +import numpy as np +import rospy +from gazebo_msgs.msg import ModelState from markov.agent_ctrl.agent_ctrl_interface import AgentCtrlInterface from markov.agent_ctrl.utils import ( - set_reward_and_metrics, - send_action, - load_action_space, - get_speed_factor, - get_normalized_progress, Logger, + get_normalized_progress, + get_speed_factor, + load_action_space, + send_action, + set_reward_and_metrics, ) -from markov.track_geom.constants import AgentPos, TrackNearDist, ObstacleDimensions, ParkLocation -from markov.track_geom.track_data import FiniteDifference, TrackData -from markov.track_geom.utils import euler_to_quaternion, pose_distance, apply_orientation -from markov.metrics.constants import StepMetrics, EpisodeStatus +from markov.boto.s3.constants import ModelMetadataKeys from markov.cameras.camera_manager import CameraManager from markov.common import ObserverInterface -from markov.log_handler.deepracer_exceptions import RewardFunctionError, GenericRolloutException +from markov.constants import DEFAULT_PARK_POSITION +from markov.gazebo_tracker.abs_tracker import AbstractTracker +from markov.gazebo_tracker.constants import TrackerPriority +from markov.gazebo_tracker.trackers.get_link_state_tracker import GetLinkStateTracker +from markov.gazebo_tracker.trackers.get_model_state_tracker import GetModelStateTracker +from markov.gazebo_tracker.trackers.set_model_state_tracker import SetModelStateTracker +from markov.log_handler.deepracer_exceptions import GenericRolloutException, RewardFunctionError +from markov.metrics.constants import EpisodeStatus, StepMetrics from markov.reset.constants import ( - AgentPhase, + ZERO_SPEED_AGENT_PHASES, AgentCtrlStatus, AgentInfo, + AgentPhase, RaceCtrlStatus, - ZERO_SPEED_AGENT_PHASES, ) from markov.reset.utils import construct_reset_rules_manager +from markov.track_geom.constants import AgentPos, ObstacleDimensions, ParkLocation, TrackNearDist +from markov.track_geom.track_data import FiniteDifference, TrackData +from markov.track_geom.utils import apply_orientation, euler_to_quaternion, pose_distance from markov.utils import get_racecar_idx from markov.virtual_event.constants import ( - WebRTCCarControl, - CarControlMode, - CarControlStatus, MAX_SPEED, MIN_SPEED, - CarControlTopic, + PAUSE_TIME_BEFORE_START, WEBRTC_CAR_CTRL_FORMAT, + CarControlMode, + CarControlStatus, + CarControlTopic, + WebRTCCarControl, ) from markov.visual_effects.effects.blink_effect import BlinkEffect from markov.visualizations.reward_distributions import RewardDataPublisher -from markov.constants import DEFAULT_PARK_POSITION -from markov.gazebo_tracker.trackers.get_link_state_tracker import GetLinkStateTracker -from markov.gazebo_tracker.trackers.get_model_state_tracker import GetModelStateTracker -from markov.gazebo_tracker.trackers.set_model_state_tracker import SetModelStateTracker -from markov.gazebo_tracker.abs_tracker import AbstractTracker -from markov.gazebo_tracker.constants import TrackerPriority -from markov.boto.s3.constants import ModelMetadataKeys -from markov.virtual_event.constants import PAUSE_TIME_BEFORE_START - from rl_coach.core_types import RunPhase +from shapely.geometry import Point +from std_msgs.msg import Float64, String LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/training_agent_ctrl.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/training_agent_ctrl.py index 330a4bbbdc..5050a2d9bc 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/training_agent_ctrl.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/training_agent_ctrl.py @@ -1,6 +1,6 @@ """This module defines the concrete classes for training""" -from markov.agent_ctrl.utils import load_action_space from markov.agent_ctrl.agent_ctrl_interface import AgentCtrlInterface +from markov.agent_ctrl.utils import load_action_space class TrainingCtrl(AgentCtrlInterface): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/utils.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/utils.py index c892c0d21a..477fcc8f1c 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/utils.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agent_ctrl/utils.py @@ -1,27 +1,27 @@ """This module should house utility methods for the agent control classes""" import json -import math import logging -import numpy as np -import markov.agent_ctrl.constants as const +import math -from markov.metrics.constants import StepMetrics +import markov.agent_ctrl.constants as const +import numpy as np from markov.agent_ctrl.constants import RewardParam -from markov.track_geom.constants import AgentPos, TrackNearDist, TrackNearPnts -from markov.log_handler.logger import Logger +from markov.boto.s3.constants import ActionSpaceTypes, ModelMetadataKeys, TrainingAlgorithm +from markov.constants import SIMAPP_VERSION_1, SIMAPP_VERSION_2, SIMAPP_VERSION_3, SIMAPP_VERSION_4 from markov.log_handler.constants import ( SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_SIMULATION_WORKER_EXCEPTION, ) -from markov.log_handler.exception_handler import log_and_exit from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger +from markov.metrics.constants import StepMetrics from markov.multi_agent_coach.action_space_configs import ( ClippedPPOActionSpaceConfig, SACActionSpaceConfig, ) +from markov.track_geom.constants import AgentPos, TrackNearDist, TrackNearPnts from scipy.spatial.transform import Rotation -from markov.constants import SIMAPP_VERSION_1, SIMAPP_VERSION_2, SIMAPP_VERSION_3, SIMAPP_VERSION_4 -from markov.boto.s3.constants import ModelMetadataKeys, ActionSpaceTypes, TrainingAlgorithm LOGGER = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/rollout_agent_factory.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/rollout_agent_factory.py index ab1a22357c..cf445ade0e 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/rollout_agent_factory.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/rollout_agent_factory.py @@ -1,13 +1,13 @@ """This module is used to create agents for the rollout worker""" from markov.agent_ctrl.bot_cars_agent_ctrl import BotCarsCtrl from markov.agent_ctrl.constants import ConfigParams -from markov.agent_ctrl.rollout_agent_ctrl import RolloutCtrl from markov.agent_ctrl.obstacles_agent_ctrl import ObstaclesCtrl +from markov.agent_ctrl.rollout_agent_ctrl import RolloutCtrl from markov.agents.agent import Agent from markov.agents.utils import construct_sensor, get_network_settings -from markov.sensors.sensors_rollout import SensorFactory -from markov.cameras.frustum_manager import FrustumManager from markov.boto.s3.constants import ModelMetadataKeys +from markov.cameras.frustum_manager import FrustumManager +from markov.sensors.sensors_rollout import SensorFactory def create_rollout_agent(agent_config, metrics, run_phase_subject): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/training_agent_factory.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/training_agent_factory.py index 40741b2aa0..641e3ca844 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/training_agent_factory.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/training_agent_factory.py @@ -3,8 +3,8 @@ from markov.agent_ctrl.training_agent_ctrl import TrainingCtrl from markov.agents.agent import Agent from markov.agents.utils import construct_sensor, get_network_settings -from markov.sensors.sensors_training import SensorFactory from markov.boto.s3.constants import ModelMetadataKeys +from markov.sensors.sensors_training import SensorFactory def create_training_agent(agent_config): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/utils.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/utils.py index 2e12da5aad..7052cfa88c 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/utils.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/agents/utils.py @@ -1,8 +1,9 @@ """Utility methods for the agent’s module""" -from typing import List, Any -from markov.architecture.constants import EmbedderType, ActivationFunctions, NeuralNetwork, Input -from markov.common import ObserverInterface +from typing import Any, List + +from markov.architecture.constants import ActivationFunctions, EmbedderType, Input, NeuralNetwork from markov.architecture.custom_architectures import DEFAULT_MIDDLEWARE, VGG_MIDDLEWARE +from markov.common import ObserverInterface from markov.sensors.composite_sensor import CompositeSensor diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/contrib.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/contrib.py index a3a91c9a1e..3274b1377e 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/contrib.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/contrib.py @@ -9,7 +9,7 @@ class Attention(object): """ def __init__(self, units: int): - """units - number of hidden units to use in the in the score wights """ + """units - number of hidden units to use in the in the score wights""" self.units = units def __call__( diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/custom_architectures.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/custom_architectures.py index c4fbb93b4a..081af3d5e5 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/custom_architectures.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/custom_architectures.py @@ -1,6 +1,6 @@ """This module contains common schemes""" -from markov.architecture.constants import SchemeInfo, Input, ActivationFunctions +from markov.architecture.constants import ActivationFunctions, Input, SchemeInfo # Default network DEFAULT_INPUT_EMBEDDER = { diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/embedder_factory.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/embedder_factory.py index baca64c93d..cb6d3c1b39 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/embedder_factory.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/architecture/embedder_factory.py @@ -1,10 +1,10 @@ """This module is used to programtically create custom archetectures, it creates input embedders and middle ware. """ -from markov.architecture.constants import SchemeInfo, ActivationFunctions, EmbedderType -from rl_coach.architectures.middleware_parameters import FCMiddlewareParameters +from markov.architecture.constants import ActivationFunctions, EmbedderType, SchemeInfo from rl_coach.architectures.embedder_parameters import InputEmbedderParameters -from rl_coach.architectures.layers import Conv2d, Dense, BatchnormActivationDropout +from rl_coach.architectures.layers import BatchnormActivationDropout, Conv2d, Dense +from rl_coach.architectures.middleware_parameters import FCMiddlewareParameters def create_scheme(info_dict): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/auth/refreshed_session.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/auth/refreshed_session.py index 52de77109d..4ce39223ae 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/auth/refreshed_session.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/auth/refreshed_session.py @@ -6,11 +6,12 @@ https://dev.to/li_chastina/auto-refresh-aws-tokens-using-iam-role-and-boto3-2cjf https://github.com/cloud-custodian/cloud-custodian/blob/master/c7n/credentials.py """ -import pytz from datetime import datetime, timedelta + +import pytz +from boto3 import Session from botocore.credentials import RefreshableCredentials from botocore.session import get_session -from boto3 import Session def refreshed_session(region="us-east-1"): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/deepracer_boto_client.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/deepracer_boto_client.py index 52c067a585..3ee5192740 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/deepracer_boto_client.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/deepracer_boto_client.py @@ -1,15 +1,15 @@ """This module implement deepracer boto client""" import abc -import time -import random import logging -import botocore -import boto3 +import random +import time -from markov.log_handler.logger import Logger -from markov.constants import NUM_RETRIES, CONNECT_TIMEOUT +import boto3 +import botocore from markov.boto.constants import BOTO_ERROR_MSG_FORMAT +from markov.constants import CONNECT_TIMEOUT, NUM_RETRIES +from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/constants.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/constants.py index 676b43a6d9..a5dadf33a0 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/constants.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/constants.py @@ -1,7 +1,6 @@ """This module contains all s3 related prefix, key, and local path constants""" import os - from enum import Enum # python version diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint.py index 6b2041268f..cdad02aa40 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint.py @@ -1,36 +1,36 @@ """This module implements checkpoint file""" -import os import io -import logging import json +import logging +import os import time + import boto3 import botocore - -from rl_coach.checkpoint import CheckpointStateFile -from rl_coach.data_stores.data_store import SyncFiles -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.constants import ( - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_S3_DATA_STORE_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_400, -) from markov.boto.s3.constants import ( + BEST_CHECKPOINT, CHECKPOINT_POSTFIX_DIR, COACH_CHECKPOINT_POSTFIX, DEEPRACER_CHECKPOINT_KEY_POSTFIX, FINISHED_FILE_KEY_POSTFIX, - LOCKFILE_KEY_POSTFIX, - BEST_CHECKPOINT, LAST_CHECKPOINT, + LOCKFILE_KEY_POSTFIX, ) from markov.boto.s3.files.checkpoint_files.deepracer_checkpoint_json import DeepracerCheckpointJson from markov.boto.s3.files.checkpoint_files.rl_coach_checkpoint import RLCoachCheckpoint from markov.boto.s3.files.checkpoint_files.rl_coach_sync_file import RlCoachSyncFile from markov.boto.s3.files.checkpoint_files.tensorflow_model import TensorflowModel +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_400, + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_S3_DATA_STORE_EXCEPTION, + SIMAPP_SIMULATION_WORKER_EXCEPTION, +) +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger +from rl_coach.checkpoint import CheckpointStateFile +from rl_coach.data_stores.data_store import SyncFiles LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/deepracer_checkpoint_json.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/deepracer_checkpoint_json.py index 9467730ffc..b50c76935d 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/deepracer_checkpoint_json.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/deepracer_checkpoint_json.py @@ -1,25 +1,25 @@ """This module implements deepracer checkpoint json file specifically""" -import os -import logging import json -import botocore +import logging +import os -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.deepracer_exceptions import GenericNonFatalException -from markov.log_handler.constants import ( - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_EVENT_ERROR_CODE_400, - SIMAPP_SIMULATION_WORKER_EXCEPTION, -) -from markov.boto.s3.s3_client import S3Client +import botocore from markov.boto.s3.constants import ( + BEST_CHECKPOINT, DEEPRACER_CHECKPOINT_KEY_POSTFIX, DEEPRACER_CHECKPOINT_LOCAL_PATH_FORMAT, - BEST_CHECKPOINT, LAST_CHECKPOINT, ) +from markov.boto.s3.s3_client import S3Client +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_400, + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_SIMULATION_WORKER_EXCEPTION, +) +from markov.log_handler.deepracer_exceptions import GenericNonFatalException +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/rl_coach_checkpoint.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/rl_coach_checkpoint.py index 66e7ac1f1e..32d3f60da2 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/rl_coach_checkpoint.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/rl_coach_checkpoint.py @@ -1,29 +1,29 @@ """This module implements rl coach coach_checkpoint file specifically""" +import logging import os import re -import logging -import botocore -from rl_coach.checkpoint import CheckpointStateFile -from markov.utils import get_s3_kms_extra_args -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.constants import ( - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_EVENT_ERROR_CODE_400, - SIMAPP_SIMULATION_WORKER_EXCEPTION, -) -from markov.boto.s3.s3_client import S3Client +import botocore from markov.boto.s3.constants import ( - COACH_CHECKPOINT_POSTFIX, - COACH_CHECKPOINT_LOCAL_PATH_FORMAT, - TEMP_COACH_CHECKPOINT_LOCAL_PATH_FORMAT, - OLD_COACH_CHECKPOINT_POSTFIX, - OLD_COACH_CHECKPOINT_LOCAL_PATH_FORMAT, BEST_CHECKPOINT, + COACH_CHECKPOINT_LOCAL_PATH_FORMAT, + COACH_CHECKPOINT_POSTFIX, LAST_CHECKPOINT, + OLD_COACH_CHECKPOINT_LOCAL_PATH_FORMAT, + OLD_COACH_CHECKPOINT_POSTFIX, + TEMP_COACH_CHECKPOINT_LOCAL_PATH_FORMAT, ) +from markov.boto.s3.s3_client import S3Client +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_400, + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_SIMULATION_WORKER_EXCEPTION, +) +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger +from markov.utils import get_s3_kms_extra_args +from rl_coach.checkpoint import CheckpointStateFile LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/rl_coach_sync_file.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/rl_coach_sync_file.py index 148140e15f..d28e41328f 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/rl_coach_sync_file.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/rl_coach_sync_file.py @@ -1,20 +1,20 @@ """This module implements rl coach sync file specifically""" -import os import io -import botocore import logging +import os -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit +import botocore +from markov.boto.s3.constants import SYNC_FILES_LOCAL_PATH_FORMAT_DICT, SYNC_FILES_POSTFIX_DICT +from markov.boto.s3.s3_client import S3Client from markov.log_handler.constants import ( - SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_EVENT_ERROR_CODE_400, - SIMAPP_SIMULATION_WORKER_EXCEPTION, + SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_S3_DATA_STORE_EXCEPTION, + SIMAPP_SIMULATION_WORKER_EXCEPTION, ) -from markov.boto.s3.s3_client import S3Client -from markov.boto.s3.constants import SYNC_FILES_POSTFIX_DICT, SYNC_FILES_LOCAL_PATH_FORMAT_DICT +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/tensorflow_model.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/tensorflow_model.py index a48a11c123..7797c91a12 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/tensorflow_model.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/checkpoint_files/tensorflow_model.py @@ -1,33 +1,33 @@ """This module implements tf model ckpt and pb file specifically""" +import logging import os +import queue import re import shutil -import queue -import logging import tensorflow as tf -from rl_coach.checkpoint import CheckpointStateFile, _filter_checkpoint_files, SingleCheckpoint from markov import utils -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.constants import ( - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_EVENT_ERROR_CODE_400, - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_S3_DATA_STORE_EXCEPTION, -) from markov.boto.s3.constants import ( CHECKPOINT_LOCAL_DIR_FORMAT, CHECKPOINT_POSTFIX_DIR, + FROZEN_HEAD_OUTPUT_GRAPH_FORMAT_MAPPING, NUM_MODELS_TO_KEEP, - TEMP_RENAME_FOLDER, SM_MODEL_PB_TEMP_FOLDER, - FROZEN_HEAD_OUTPUT_GRAPH_FORMAT_MAPPING, + TEMP_RENAME_FOLDER, ActionSpaceTypes, TrainingAlgorithm, ) from markov.boto.s3.s3_client import S3Client +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_400, + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_S3_DATA_STORE_EXCEPTION, + SIMAPP_SIMULATION_WORKER_EXCEPTION, +) +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger +from rl_coach.checkpoint import CheckpointStateFile, SingleCheckpoint, _filter_checkpoint_files LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/hyperparameters.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/hyperparameters.py index 0aae18897f..f22f4334c5 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/hyperparameters.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/hyperparameters.py @@ -1,18 +1,18 @@ """This module implements hyperparameters file""" -import os import io import json import logging -import botocore +import os +import botocore from markov.boto.s3.s3_client import S3Client -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit from markov.log_handler.constants import ( SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_SIMULATION_WORKER_EXCEPTION, ) +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/ip_config.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/ip_config.py index 0be865efaa..e0296abd01 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/ip_config.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/ip_config.py @@ -1,24 +1,23 @@ """This module implements s3 client for ip config""" -import os import io -import time import json -import socket import logging -import botocore +import os +import socket +import time -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.logger import Logger +import botocore +from markov.boto.s3.constants import IP_ADDRESS_POSTFIX, IP_DONE_POSTFIX, SAGEMAKER_WAIT_TIME +from markov.boto.s3.s3_client import S3Client from markov.log_handler.constants import ( - SIMAPP_SIMULATION_WORKER_EXCEPTION, + SIMAPP_ENVIRONMENT_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_S3_DATA_STORE_EXCEPTION, - SIMAPP_ENVIRONMENT_EXCEPTION, + SIMAPP_SIMULATION_WORKER_EXCEPTION, ) - -from markov.boto.s3.constants import SAGEMAKER_WAIT_TIME, IP_ADDRESS_POSTFIX, IP_DONE_POSTFIX -from markov.boto.s3.s3_client import S3Client +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/metrics.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/metrics.py index dbacc20f65..a52e2bacf4 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/metrics.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/metrics.py @@ -2,13 +2,13 @@ import logging -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit +from markov.boto.s3.s3_client import S3Client from markov.log_handler.constants import ( - SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_SIMULATION_WORKER_EXCEPTION, ) -from markov.boto.s3.s3_client import S3Client +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/model_metadata.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/model_metadata.py index cb5879634a..eaecce2dce 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/model_metadata.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/model_metadata.py @@ -1,19 +1,19 @@ """This module implements s3 client for model metadata""" -import os -import logging import json +import logging +import os -from markov.log_handler.logger import Logger +from markov.architecture.constants import Input, NeuralNetwork +from markov.boto.s3.constants import ActionSpaceTypes, ModelMetadataKeys, TrainingAlgorithm +from markov.boto.s3.s3_client import S3Client +from markov.constants import SIMAPP_VERSION_1, SIMAPP_VERSION_2 from markov.log_handler.constants import ( SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_SIMULATION_WORKER_EXCEPTION, ) from markov.log_handler.exception_handler import log_and_exit -from markov.architecture.constants import Input, NeuralNetwork -from markov.constants import SIMAPP_VERSION_2, SIMAPP_VERSION_1 -from markov.boto.s3.constants import ModelMetadataKeys, TrainingAlgorithm, ActionSpaceTypes -from markov.boto.s3.s3_client import S3Client +from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/reward_function.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/reward_function.py index 3d424ca56c..2b53711f9e 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/reward_function.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/reward_function.py @@ -1,18 +1,18 @@ """This module implements s3 client for reward function""" -import os import logging -import botocore +import os +import botocore from markov.boto.s3.s3_client import S3Client -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit from markov.log_handler.constants import ( - SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_EVENT_ERROR_CODE_400, + SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_S3_DATA_STORE_EXCEPTION, SIMAPP_SIMULATION_WORKER_EXCEPTION, ) +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/simtrace_video.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/simtrace_video.py index 65dc1d12f0..aaf22d2af8 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/simtrace_video.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/simtrace_video.py @@ -1,11 +1,11 @@ """This module implements s3 client for simtrace, pip, 45degrees, and topview videos""" -import os import logging +import os -from markov.log_handler.logger import Logger -from markov.boto.s3.s3_client import S3Client from markov.boto.s3.constants import SIMTRACE_VIDEO_POSTFIX_DICT +from markov.boto.s3.s3_client import S3Client +from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/virtual_event_best_sector_time.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/virtual_event_best_sector_time.py index 61047bbbb0..e665aa88b5 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/virtual_event_best_sector_time.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/virtual_event_best_sector_time.py @@ -1,28 +1,28 @@ """This module implements s3 client for virtual event best sector time""" -import os -import sys import json import logging -import botocore +import os +import sys -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.constants import ( - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_EVENT_SYSTEM_ERROR, -) -from markov.log_handler.deepracer_exceptions import GenericNonFatalException +import botocore from markov.boto.s3.constants import ( - SECTOR_TIME_S3_POSTFIX, PYTHON_2, PYTHON_3, + SECTOR_TIME_FORMAT_DICT, + SECTOR_TIME_S3_POSTFIX, SECTOR_X_FORMAT, TrackSectorTime, - SECTOR_TIME_FORMAT_DICT, ) from markov.boto.s3.s3_client import S3Client +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_EVENT_SYSTEM_ERROR, + SIMAPP_SIMULATION_WORKER_EXCEPTION, +) +from markov.log_handler.deepracer_exceptions import GenericNonFatalException +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/yaml_file.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/yaml_file.py index 6368cf96b2..092c0f2d7d 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/yaml_file.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/files/yaml_file.py @@ -1,33 +1,32 @@ """This module implements s3 client for yaml file""" -import os import logging -import yaml -import botocore +import os -from markov.utils import force_list -from markov.rollout_constants import BodyShellType -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.constants import ( - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_SIMULATION_WORKER_EXCEPTION, -) -from markov.reset.constants import RaceType -from markov.boto.s3.s3_client import S3Client -from markov.boto.s3.utils import is_power_of_two +import botocore +import yaml from markov.boto.s3.constants import ( - MODEL_METADATA_S3_POSTFIX, - AgentType, - YamlKey, - TRAINING_MANDATORY_YAML_KEY, EVAL_MANDATORY_YAML_KEY, + F1_RACE_TYPE, + F1_SHELL_USERS_LIST, + MODEL_METADATA_S3_POSTFIX, TOUR_MANDATORY_YAML_KEY, + TRAINING_MANDATORY_YAML_KEY, VIRUTAL_EVENT_MANDATORY_YAML_KEY, - F1_SHELL_USERS_LIST, - F1_RACE_TYPE, + AgentType, + YamlKey, ) -from markov.rollout_constants import CarColorType +from markov.boto.s3.s3_client import S3Client +from markov.boto.s3.utils import is_power_of_two +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_SIMULATION_WORKER_EXCEPTION, +) +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger +from markov.reset.constants import RaceType +from markov.rollout_constants import BodyShellType, CarColorType +from markov.utils import force_list LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/s3_client.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/s3_client.py index 6b152894a4..8bf47d0c78 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/s3_client.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/s3/s3_client.py @@ -1,18 +1,17 @@ """This module implement s3 client""" import botocore - -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.deepracer_exceptions import GenericNonFatalException +from markov.boto.constants import BotoClientNames +from markov.boto.deepracer_boto_client import DeepRacerBotoClient from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_400, + SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_EVENT_SYSTEM_ERROR, SIMAPP_EVENT_USER_ERROR, - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_EVENT_ERROR_CODE_400, SIMAPP_S3_DATA_STORE_EXCEPTION, ) -from markov.boto.constants import BotoClientNames -from markov.boto.deepracer_boto_client import DeepRacerBotoClient +from markov.log_handler.deepracer_exceptions import GenericNonFatalException +from markov.log_handler.exception_handler import log_and_exit class S3Client(DeepRacerBotoClient): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/sqs/sqs_client.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/sqs/sqs_client.py index c570316550..43cf389e5e 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/sqs/sqs_client.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/boto/sqs/sqs_client.py @@ -1,16 +1,17 @@ """This module implement sqs client""" import logging + import botocore -from markov.log_handler.exception_handler import log_and_exit +from markov.boto.constants import BotoClientNames +from markov.boto.deepracer_boto_client import DeepRacerBotoClient +from markov.boto.sqs.constants import StatusIndicator from markov.log_handler.constants import ( SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_SQS_DELETE_MESSAGE_EXCEPTION, ) +from markov.log_handler.exception_handler import log_and_exit from markov.log_handler.logger import Logger -from markov.boto.sqs.constants import StatusIndicator -from markov.boto.constants import BotoClientNames -from markov.boto.deepracer_boto_client import DeepRacerBotoClient LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/camera_utils.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/camera_utils.py index 96873e1d9d..4dabc14cad 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/camera_utils.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/camera_utils.py @@ -1,10 +1,11 @@ -import rospy -import time import threading +import time + +import rospy from markov.cameras.camera_factory import CameraFactory -from markov.log_handler.deepracer_exceptions import GenericRolloutException from markov.defaults import DEFAULT_MAIN_CAMERA, DEFAULT_SUB_CAMERA from markov.gazebo_tracker.trackers.get_model_state_tracker import GetModelStateTracker +from markov.log_handler.deepracer_exceptions import GenericRolloutException is_configure_camera_called = False configure_camera_function_lock = threading.Lock() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/abs_camera.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/abs_camera.py index 3a3628edf0..26b3c094ad 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/abs_camera.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/abs_camera.py @@ -1,11 +1,11 @@ import abc import threading +from gazebo_msgs.srv import SpawnModel +from markov.cameras.camera_manager import CameraManager from markov.log_handler.deepracer_exceptions import GenericRolloutException from markov.rospy_wrappers import ServiceProxyWrapper from markov.track_geom.constants import SPAWN_SDF_MODEL -from markov.cameras.camera_manager import CameraManager -from gazebo_msgs.srv import SpawnModel # Python 2 and 3 compatible Abstract class ABC = abc.ABCMeta("ABC", (object,), {}) diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/camera_factory.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/camera_factory.py index acdef5777b..af6df99d0c 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/camera_factory.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/camera_factory.py @@ -1,6 +1,7 @@ +from enum import Enum + from markov.cameras.handlers import FollowCarCamera, TopCamera from markov.log_handler.deepracer_exceptions import GenericRolloutException -from enum import Enum class CameraType(Enum): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/camera_manager.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/camera_manager.py index 0d8db228bd..b2b1c0a7a2 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/camera_manager.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/camera_manager.py @@ -1,10 +1,10 @@ import copy -import threading import logging +import threading -from markov.log_handler.deepracer_exceptions import GenericRolloutException from markov.gazebo_tracker.abs_tracker import AbstractTracker from markov.gazebo_tracker.trackers.get_model_state_tracker import GetModelStateTracker +from markov.log_handler.deepracer_exceptions import GenericRolloutException from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/constants.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/constants.py index e865d5fdd2..d8aaa1defa 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/constants.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/constants.py @@ -1,7 +1,9 @@ """This module houses the constants for the camera package""" from enum import Enum, unique + import numpy as np + # Define Gazebo World default direction unit vectors class GazeboWorld(object): forward = np.array((1.0, 0, 0)) diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/frustum.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/frustum.py index 3398592dfe..7429d7b1e9 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/frustum.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/frustum.py @@ -1,12 +1,13 @@ -import numpy as np import math import threading -from markov.log_handler.deepracer_exceptions import GenericRolloutException + +import numpy as np from markov.architecture.constants import Input -from markov.track_geom.utils import euler_to_quaternion, quaternion_to_euler, apply_orientation from markov.cameras.constants import GazeboWorld from markov.cameras.utils import normalize, project_to_2d, ray_plane_intersect from markov.constants import SIMAPP_VERSION_3 +from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.track_geom.utils import apply_orientation, euler_to_quaternion, quaternion_to_euler class Frustum(object): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/frustum_manager.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/frustum_manager.py index 2849983c5f..358978af2c 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/frustum_manager.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/frustum_manager.py @@ -1,6 +1,7 @@ import threading -from markov.log_handler.deepracer_exceptions import GenericRolloutException + from markov.cameras.frustum import Frustum +from markov.log_handler.deepracer_exceptions import GenericRolloutException class FrustumManager(object): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/handlers/follow_car_camera.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/handlers/follow_car_camera.py index 55ae5d7355..18d7aaf068 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/handlers/follow_car_camera.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/handlers/follow_car_camera.py @@ -1,12 +1,12 @@ import math -import numpy as np +import numpy as np from gazebo_msgs.msg import ModelState, ModelStates from geometry_msgs.msg import Pose from markov.cameras import utils -from markov.track_geom.utils import euler_to_quaternion, quaternion_to_euler, apply_orientation from markov.cameras.abs_camera import AbstractCamera from markov.gazebo_tracker.trackers.set_model_state_tracker import SetModelStateTracker +from markov.track_geom.utils import apply_orientation, euler_to_quaternion, quaternion_to_euler class FollowCarCamera(AbstractCamera): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/handlers/top_camera.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/handlers/top_camera.py index 7d69a2f4d7..3819e11fcb 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/handlers/top_camera.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/handlers/top_camera.py @@ -1,17 +1,17 @@ -import math import logging +import math import xml.etree.ElementTree as ET -import rospy -from deepracer_simulation_environment.srv import TopCamDataSrvResponse, TopCamDataSrv +import rospy +from deepracer_simulation_environment.srv import TopCamDataSrv, TopCamDataSrvResponse from gazebo_msgs.msg import ModelState from geometry_msgs.msg import Pose -from markov.track_geom.track_data import TrackData -from markov.track_geom.utils import euler_to_quaternion from markov.cameras.abs_camera import AbstractCamera from markov.cameras.constants import CameraSettings -from markov.log_handler.logger import Logger from markov.gazebo_tracker.trackers.set_model_state_tracker import SetModelStateTracker +from markov.log_handler.logger import Logger +from markov.track_geom.track_data import TrackData +from markov.track_geom.utils import euler_to_quaternion # Height value is determined from AWS track and is maintained to prevent z fighting in top down # view diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/utils.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/utils.py index ef267d1992..1d281f516c 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/utils.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/cameras/utils.py @@ -1,7 +1,8 @@ import math + import numpy as np from markov.log_handler.deepracer_exceptions import GenericRolloutException -from markov.track_geom.utils import euler_to_quaternion, apply_orientation, inverse_quaternion +from markov.track_geom.utils import apply_orientation, euler_to_quaternion, inverse_quaternion def get_angle_between_two_points_2d_rad(pt1, pt2): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/deepracer_memory.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/deepracer_memory.py index eceb3e4e16..a20ce9fc47 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/deepracer_memory.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/deepracer_memory.py @@ -1,16 +1,15 @@ -from threading import Thread, Event, Lock +import logging import pickle import queue -import logging -import redis import uuid +from threading import Event, Lock, Thread +import redis +from markov.architecture.constants import NeuralNetwork +from markov.log_handler.logger import Logger +from rl_coach.core_types import Episode from rl_coach.memories.backend.memory import MemoryBackend from rl_coach.memories.backend.redis import RedisPubSubMemoryBackendParameters -from rl_coach.core_types import Episode - -from markov.log_handler.logger import Logger -from markov.architecture.constants import NeuralNetwork LOG = Logger(__name__, logging.INFO).get_logger() @@ -72,7 +71,7 @@ def get_endpoint_helper(redis_address, redis_port): class DeepRacerRolloutBackEnd(MemoryBackend): - """ Class used by the rollout worker to publish data to the training worker""" + """Class used by the rollout worker to publish data to the training worker""" def __init__(self, params, num_consecutive_playing_steps, agent_name): """params - Struct containing all the necessary redis parammeters, @@ -183,12 +182,12 @@ def store(self, obj): self.last_episode_num += self.num_workers def get_endpoint(self): - """Returns a dict with the redis address and port """ + """Returns a dict with the redis address and port""" return get_endpoint_helper(self.params.redis_address, self.params.redis_port) class DeepRacerTrainerBackEnd(MemoryBackend): - """Class used by the training worker to retrieve the data from the rollout worker """ + """Class used by the training worker to retrieve the data from the rollout worker""" def __init__(self, params, agents_params): """params - Struct containing all the necessary redis parammeters, @@ -248,15 +247,15 @@ def data_handler(self, message, agent_name): LOG.info("Trainer data handler error: %s", ex) def get_rollout_steps(self): - """Returns the total number of steps in a rollout """ + """Returns the total number of steps in a rollout""" return self.rollout_steps def get_total_episodes_in_rollout(self): - """Return the total number of episodes collected in the rollout """ + """Return the total number of episodes collected in the rollout""" return self.total_episodes_in_rollout def publish_worker(self, agent_name): - """ Worker responsible for requesting data from the rollout worker""" + """Worker responsible for requesting data from the rollout worker""" while True: try: if self.request_data: @@ -343,5 +342,5 @@ def fetch(self, num_consecutive_playing_steps=None): continue def get_endpoint(self): - """Returns a dict with the redis address and port """ + """Returns a dict with the redis address and port""" return get_endpoint_helper(self.params.redis_address, self.params.redis_port) diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/domain_randomizations/visual/light_randomizer.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/domain_randomizations/visual/light_randomizer.py index 14c40ffd8d..d2dbe050c6 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/domain_randomizations/visual/light_randomizer.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/domain_randomizations/visual/light_randomizer.py @@ -1,23 +1,21 @@ import logging -import numpy as np from enum import Enum -from markov.rospy_wrappers import ServiceProxyWrapper +import numpy as np +import rospy +from gazebo_msgs.srv import SetLightProperties, SetLightPropertiesRequest from markov.domain_randomizations.abs_randomizer import AbstractRandomizer -from markov.log_handler.logger import Logger from markov.domain_randomizations.constants import ( - GazeboServiceName, - RangeType, - RANGE_MIN, RANGE_MAX, - Color, + RANGE_MIN, Attenuation, + Color, + GazeboServiceName, + RangeType, ) - -import rospy +from markov.log_handler.logger import Logger +from markov.rospy_wrappers import ServiceProxyWrapper from std_msgs.msg import ColorRGBA -from gazebo_msgs.srv import SetLightProperties, SetLightPropertiesRequest - logger = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/domain_randomizations/visual/model_visual_randomizer.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/domain_randomizations/visual/model_visual_randomizer.py index 0f634b30b5..0586a20637 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/domain_randomizations/visual/model_visual_randomizer.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/domain_randomizations/visual/model_visual_randomizer.py @@ -1,25 +1,22 @@ import logging -import numpy as np -from markov.rospy_wrappers import ServiceProxyWrapper +import numpy as np +import rospy +from deepracer_msgs.srv import GetVisualNames, GetVisualNamesRequest +from gazebo_msgs.srv import GetModelProperties, GetModelPropertiesRequest from markov.domain_randomizations.abs_randomizer import AbstractRandomizer -from markov.log_handler.logger import Logger from markov.domain_randomizations.constants import ( - ModelRandomizerType, - GazeboServiceName, - Color, - RANGE_MIN, RANGE_MAX, + RANGE_MIN, + Color, + GazeboServiceName, + ModelRandomizerType, ) from markov.gazebo_tracker.trackers.set_visual_color_tracker import SetVisualColorTracker - -import rospy -from gazebo_msgs.srv import GetModelProperties, GetModelPropertiesRequest +from markov.log_handler.logger import Logger +from markov.rospy_wrappers import ServiceProxyWrapper from std_msgs.msg import ColorRGBA -from deepracer_msgs.srv import GetVisualNames, GetVisualNamesRequest - - logger = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/environments/deepracer_racetrack_env.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/environments/deepracer_racetrack_env.py index 576d2f76b4..ebb691baa7 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/environments/deepracer_racetrack_env.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/environments/deepracer_racetrack_env.py @@ -1,17 +1,15 @@ """This module defines the interface between rl coach and the agents enviroment""" from __future__ import print_function -from typing import List, Union, Dict -from rl_coach.base_parameters import AgentParameters, VisualizationParameters -from rl_coach.environments.environment import LevelSelection -from rl_coach.filters.filter import NoInputFilter, NoOutputFilter +from typing import Dict, List, Union from markov.agents.agent import Agent from markov.agents.utils import RunPhaseSubject from markov.constants import SIMAPP_VERSION_4 -from markov.multi_agent_coach.multi_agent_environment import ( - MultiAgentEnvironment, - MultiAgentEnvironmentParameters, +from markov.domain_randomizations.randomizer_manager import RandomizerManager +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_SIMULATION_WORKER_EXCEPTION, ) from markov.log_handler.deepracer_exceptions import ( GenericRolloutException, @@ -19,11 +17,13 @@ RewardFunctionError, ) from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.constants import ( - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_500, +from markov.multi_agent_coach.multi_agent_environment import ( + MultiAgentEnvironment, + MultiAgentEnvironmentParameters, ) -from markov.domain_randomizations.randomizer_manager import RandomizerManager +from rl_coach.base_parameters import AgentParameters, VisualizationParameters +from rl_coach.environments.environment import LevelSelection +from rl_coach.filters.filter import NoInputFilter, NoOutputFilter # Max number of steps to allow per episode MAX_STEPS = 100000 diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/evaluation_worker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/evaluation_worker.py index 43ff6cc576..b76945a03f 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/evaluation_worker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/evaluation_worker.py @@ -5,60 +5,59 @@ import os import time from threading import Thread -import rospy -from rl_coach.base_parameters import TaskParameters -from rl_coach.core_types import EnvironmentSteps -from rl_coach.data_stores.data_store import SyncFiles +import rospy from markov import utils -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.constants import ( - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_500, -) -from markov.constants import SIMAPP_VERSION_2, DEFAULT_PARK_POSITION, ROLLOUT_WORKER_PROFILER_PATH from markov.agent_ctrl.constants import ConfigParams from markov.agents.rollout_agent_factory import ( - create_rollout_agent, - create_obstacles_agent, create_bot_cars_agent, + create_obstacles_agent, + create_rollout_agent, ) from markov.agents.utils import RunPhaseSubject +from markov.boto.s3.constants import ( + CAMERA_45DEGREE_LOCAL_PATH_FORMAT, + CAMERA_PIP_MP4_LOCAL_PATH_FORMAT, + CAMERA_TOPVIEW_LOCAL_PATH_FORMAT, + MODEL_METADATA_LOCAL_PATH_FORMAT, + MODEL_METADATA_S3_POSTFIX, + SIMTRACE_EVAL_LOCAL_PATH_FORMAT, + ModelMetadataKeys, + SimtraceVideoNames, +) +from markov.boto.s3.files.checkpoint import Checkpoint +from markov.boto.s3.files.model_metadata import ModelMetadata +from markov.boto.s3.files.simtrace_video import SimtraceVideo +from markov.boto.s3.utils import get_s3_key +from markov.camera_utils import configure_camera +from markov.constants import DEFAULT_PARK_POSITION, ROLLOUT_WORKER_PROFILER_PATH, SIMAPP_VERSION_2 from markov.defaults import reward_function +from markov.environments.constants import LINK_NAMES, STEERING_TOPICS, VELOCITY_TOPICS +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_SIMULATION_WORKER_EXCEPTION, +) from markov.log_handler.deepracer_exceptions import GenericRolloutError, GenericRolloutException -from markov.environments.constants import VELOCITY_TOPICS, STEERING_TOPICS, LINK_NAMES -from markov.metrics.s3_metrics import EvalMetrics -from markov.metrics.iteration_data import IterationData +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger from markov.metrics.constants import MetricsS3Keys -from markov.s3_boto_data_store import S3BotoDataStore, S3BotoDataStoreParameters -from markov.sagemaker_graph_manager import get_graph_manager +from markov.metrics.iteration_data import IterationData +from markov.metrics.s3_metrics import EvalMetrics +from markov.reset.constants import RaceType from markov.rollout_utils import ( PhaseObserver, - signal_robomaker_markov_package_ready, configure_environment_randomizer, get_robomaker_profiler_env, + signal_robomaker_markov_package_ready, ) from markov.rospy_wrappers import ServiceProxyWrapper -from markov.camera_utils import configure_camera +from markov.s3_boto_data_store import S3BotoDataStore, S3BotoDataStoreParameters +from markov.sagemaker_graph_manager import get_graph_manager from markov.track_geom.track_data import TrackData from markov.track_geom.utils import get_start_positions -from markov.boto.s3.constants import ( - MODEL_METADATA_LOCAL_PATH_FORMAT, - MODEL_METADATA_S3_POSTFIX, - SIMTRACE_EVAL_LOCAL_PATH_FORMAT, - CAMERA_PIP_MP4_LOCAL_PATH_FORMAT, - CAMERA_45DEGREE_LOCAL_PATH_FORMAT, - CAMERA_TOPVIEW_LOCAL_PATH_FORMAT, - SimtraceVideoNames, - ModelMetadataKeys, -) -from markov.boto.s3.files.model_metadata import ModelMetadata -from markov.boto.s3.files.simtrace_video import SimtraceVideo -from markov.boto.s3.files.checkpoint import Checkpoint -from markov.boto.s3.utils import get_s3_key -from markov.reset.constants import RaceType - +from rl_coach.base_parameters import TaskParameters +from rl_coach.core_types import EnvironmentSteps +from rl_coach.data_stores.data_store import SyncFiles from std_srvs.srv import Empty, EmptyRequest logger = Logger(__name__, logging.INFO).get_logger() @@ -187,7 +186,7 @@ def evaluation_worker( def main(): - """ Main function for evaluation worker """ + """Main function for evaluation worker""" parser = argparse.ArgumentParser() parser.add_argument( "-p", diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/exploration_policies/deepracer_categorical.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/exploration_policies/deepracer_categorical.py index c3fcdec643..dbf9f2c0c4 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/exploration_policies/deepracer_categorical.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/exploration_policies/deepracer_categorical.py @@ -1,8 +1,7 @@ from typing import List import numpy as np - -from rl_coach.core_types import RunPhase, ActionType +from rl_coach.core_types import ActionType, RunPhase from rl_coach.exploration_policies.exploration_policy import ( DiscreteActionExplorationPolicy, ExplorationParameters, diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/filters/observation/observation_binary_filter.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/filters/observation/observation_binary_filter.py index 7a500fb64e..af7c3fdc14 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/filters/observation/observation_binary_filter.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/filters/observation/observation_binary_filter.py @@ -1,13 +1,13 @@ -from rl_coach.core_types import ObservationType -from rl_coach.filters.observation.observation_filter import ObservationFilter -from rl_coach.spaces import ObservationSpace +import numpy as np from markov.environments.constants import ( NUMBER_OF_LIDAR_SECTORS, NUMBER_OF_LIDAR_VALUES_IN_EACH_SECTOR, SECTOR_LIDAR_CLIPPING_DIST, ) -import numpy as np from markov.log_handler.deepracer_exceptions import GenericRolloutException +from rl_coach.core_types import ObservationType +from rl_coach.filters.observation.observation_filter import ObservationFilter +from rl_coach.spaces import ObservationSpace class ObservationBinarySectorFilter(ObservationFilter): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/abs_tracker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/abs_tracker.py index 430d25c05f..8890b807ff 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/abs_tracker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/abs_tracker.py @@ -1,6 +1,7 @@ import abc -from markov.gazebo_tracker.tracker_manager import TrackerManager + import markov.gazebo_tracker.constants as consts +from markov.gazebo_tracker.tracker_manager import TrackerManager # Python 2 and 3 compatible Abstract class ABC = abc.ABCMeta("ABC", (object,), {}) diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/tracker_manager.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/tracker_manager.py index b8426200ba..39da032560 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/tracker_manager.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/tracker_manager.py @@ -1,16 +1,16 @@ -import threading import logging -from rosgraph_msgs.msg import Clock +import threading + +import markov.gazebo_tracker.constants as consts import rospy -from markov.log_handler.deepracer_exceptions import GenericRolloutException -from markov.log_handler.exception_handler import log_and_exit from markov.log_handler.constants import ( - SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_SIMULATION_WORKER_EXCEPTION, ) -import markov.gazebo_tracker.constants as consts +from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.log_handler.exception_handler import log_and_exit from markov.log_handler.logger import Logger - +from rosgraph_msgs.msg import Clock logger = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/get_link_state_tracker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/get_link_state_tracker.py index 67dc6fe9cf..ffc67fa12d 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/get_link_state_tracker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/get_link_state_tracker.py @@ -1,14 +1,14 @@ -import threading -from markov.log_handler.deepracer_exceptions import GenericRolloutException import copy -import rospy +import threading +import markov.gazebo_tracker.constants as consts +import rospy from deepracer_msgs.srv import GetLinkStates from gazebo_msgs.srv import GetLinkStateResponse -from markov.track_geom.constants import GET_LINK_STATES -from markov.rospy_wrappers import ServiceProxyWrapper from markov.gazebo_tracker.abs_tracker import AbstractTracker -import markov.gazebo_tracker.constants as consts +from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.rospy_wrappers import ServiceProxyWrapper +from markov.track_geom.constants import GET_LINK_STATES class GetLinkStateTracker(AbstractTracker): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/get_model_state_tracker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/get_model_state_tracker.py index c158fadb85..d3711d12fc 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/get_model_state_tracker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/get_model_state_tracker.py @@ -1,14 +1,14 @@ -import threading -from markov.log_handler.deepracer_exceptions import GenericRolloutException import copy -import rospy +import threading +import markov.gazebo_tracker.constants as consts +import rospy from deepracer_msgs.srv import GetModelStates from gazebo_msgs.srv import GetModelStateResponse -from markov.track_geom.constants import GET_MODEL_STATES -from markov.rospy_wrappers import ServiceProxyWrapper from markov.gazebo_tracker.abs_tracker import AbstractTracker -import markov.gazebo_tracker.constants as consts +from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.rospy_wrappers import ServiceProxyWrapper +from markov.track_geom.constants import GET_MODEL_STATES class GetModelStateTracker(AbstractTracker): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_model_state_tracker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_model_state_tracker.py index 2b8f82a7ba..a17b390f08 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_model_state_tracker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_model_state_tracker.py @@ -1,13 +1,13 @@ import threading -from markov.log_handler.deepracer_exceptions import GenericRolloutException -import rospy +import markov.gazebo_tracker.constants as consts +import rospy from deepracer_msgs.srv import SetModelStates from gazebo_msgs.srv import SetModelStateResponse -from markov.track_geom.constants import SET_MODEL_STATES -from markov.rospy_wrappers import ServiceProxyWrapper from markov.gazebo_tracker.abs_tracker import AbstractTracker -import markov.gazebo_tracker.constants as consts +from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.rospy_wrappers import ServiceProxyWrapper +from markov.track_geom.constants import SET_MODEL_STATES class SetModelStateTracker(AbstractTracker): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_visual_color_tracker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_visual_color_tracker.py index f7f44d9f07..92c5ce0e35 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_visual_color_tracker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_visual_color_tracker.py @@ -1,13 +1,13 @@ import threading -from markov.log_handler.deepracer_exceptions import GenericRolloutException -import rospy from collections import OrderedDict +import markov.gazebo_tracker.constants as consts +import rospy +from deepracer_msgs.srv import SetVisualColorResponse, SetVisualColors, SetVisualColorsRequest from markov.domain_randomizations.constants import GazeboServiceName -from markov.rospy_wrappers import ServiceProxyWrapper from markov.gazebo_tracker.abs_tracker import AbstractTracker -import markov.gazebo_tracker.constants as consts -from deepracer_msgs.srv import SetVisualColors, SetVisualColorsRequest, SetVisualColorResponse +from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.rospy_wrappers import ServiceProxyWrapper class SetVisualColorTracker(AbstractTracker): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_visual_transparency_tracker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_visual_transparency_tracker.py index a595403765..a896058065 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_visual_transparency_tracker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_tracker/trackers/set_visual_transparency_tracker.py @@ -1,18 +1,17 @@ import threading from collections import OrderedDict -from markov.log_handler.deepracer_exceptions import GenericRolloutException -import rospy - -from markov.domain_randomizations.constants import GazeboServiceName -from markov.rospy_wrappers import ServiceProxyWrapper -from markov.gazebo_tracker.abs_tracker import AbstractTracker import markov.gazebo_tracker.constants as consts +import rospy from deepracer_msgs.srv import ( SetVisualTransparencies, SetVisualTransparenciesRequest, SetVisualTransparencyResponse, ) +from markov.domain_randomizations.constants import GazeboServiceName +from markov.gazebo_tracker.abs_tracker import AbstractTracker +from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.rospy_wrappers import ServiceProxyWrapper class SetVisualTransparencyTracker(AbstractTracker): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_utils/model_updater.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_utils/model_updater.py index fa76f35813..64acd1564f 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_utils/model_updater.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/gazebo_utils/model_updater.py @@ -1,17 +1,5 @@ -import rospy import markov.rollout_constants as const -from markov.log_handler.deepracer_exceptions import GenericRolloutException -from markov.domain_randomizations.constants import GazeboServiceName -from markov.rospy_wrappers import ServiceProxyWrapper -from markov.track_geom.constants import SET_MODEL_STATE -from markov.track_geom.utils import euler_to_quaternion -from markov.gazebo_tracker.trackers.get_model_state_tracker import GetModelStateTracker -from markov.gazebo_tracker.trackers.set_model_state_tracker import SetModelStateTracker -from std_msgs.msg import ColorRGBA -from std_srvs.srv import Empty, EmptyRequest -from geometry_msgs.msg import Pose -from gazebo_msgs.msg import ModelState -from gazebo_msgs.srv import SetModelState, GetModelProperties, GetModelPropertiesRequest +import rospy from deepracer_msgs.srv import ( GetVisualNames, GetVisualNamesRequest, @@ -24,6 +12,18 @@ SetVisualVisibles, SetVisualVisiblesRequest, ) +from gazebo_msgs.msg import ModelState +from gazebo_msgs.srv import GetModelProperties, GetModelPropertiesRequest, SetModelState +from geometry_msgs.msg import Pose +from markov.domain_randomizations.constants import GazeboServiceName +from markov.gazebo_tracker.trackers.get_model_state_tracker import GetModelStateTracker +from markov.gazebo_tracker.trackers.set_model_state_tracker import SetModelStateTracker +from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.rospy_wrappers import ServiceProxyWrapper +from markov.track_geom.constants import SET_MODEL_STATE +from markov.track_geom.utils import euler_to_quaternion +from std_msgs.msg import ColorRGBA +from std_srvs.srv import Empty, EmptyRequest GAZEBO_SERVICES = [ "PAUSE_PHYSICS", diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/log_handler/deepracer_exceptions.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/log_handler/deepracer_exceptions.py index 4414ac7747..57a14cf1aa 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/log_handler/deepracer_exceptions.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/log_handler/deepracer_exceptions.py @@ -1,179 +1,208 @@ -'''This module should house all custom exceptions''' +"""This module should house all custom exceptions""" import logging + +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_400, + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_EVENT_SYSTEM_ERROR, + SIMAPP_EVENT_USER_ERROR, + SIMAPP_SIMULATION_WORKER_EXCEPTION, + SIMAPP_TRAINING_WORKER_EXCEPTION, + SIMAPP_VALIDATION_WORKER_EXCEPTION, +) from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.constants import (SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_TRAINING_WORKER_EXCEPTION, - SIMAPP_VALIDATION_WORKER_EXCEPTION, - SIMAPP_EVENT_SYSTEM_ERROR, - SIMAPP_EVENT_USER_ERROR, - SIMAPP_EVENT_ERROR_CODE_400, - SIMAPP_EVENT_ERROR_CODE_500) from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() class RewardFunctionError(Exception): - '''This exception is for user errors associated with the reward function''' + """This exception is for user errors associated with the reward function""" + def __init__(self, msg): - '''msg - This should be text containing information about what caused - the exception, for example "cannot divide by zero - ''' + """msg - This should be text containing information about what caused + the exception, for example "cannot divide by zero + """ super(RewardFunctionError, self).__init__(msg) self.msg = msg def log_except_and_exit(self): - '''Logs the exception to cloud watch and exits the sim app''' - log_and_exit("Reward function error: {}" - .format(self.msg), - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_400) + """Logs the exception to cloud watch and exits the sim app""" + log_and_exit( + "Reward function error: {}".format(self.msg), + SIMAPP_SIMULATION_WORKER_EXCEPTION, + SIMAPP_EVENT_ERROR_CODE_400, + ) + class GenericTrainerException(Exception): - '''This exception is a generic exception for the training worker''' + """This exception is a generic exception for the training worker""" + def __init__(self, msg): - '''msg - This should be text containing information about what caused - the exception, for example "cannot divide by zero - ''' + """msg - This should be text containing information about what caused + the exception, for example "cannot divide by zero + """ super(GenericTrainerException, self).__init__(msg) self.msg = msg def log_except_and_exit(self): - '''Logs the exception to cloud watch and exits the app''' - log_and_exit("Training worker failed: {}" - .format(self.msg), - SIMAPP_TRAINING_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_500) + """Logs the exception to cloud watch and exits the app""" + log_and_exit( + "Training worker failed: {}".format(self.msg), + SIMAPP_TRAINING_WORKER_EXCEPTION, + SIMAPP_EVENT_ERROR_CODE_500, + ) + class GenericTrainerError(Exception): - '''This exception is a generic error for the training worker''' + """This exception is a generic error for the training worker""" + def __init__(self, msg): - '''msg - This should be text containing information about what caused - the exception, for example "cannot divide by zero - ''' + """msg - This should be text containing information about what caused + the exception, for example "cannot divide by zero + """ super(GenericTrainerError, self).__init__(msg) self.msg = msg def log_except_and_exit(self): - '''Logs the exception to cloud watch and exits the app''' - log_and_exit("Training worker failed: {}" - .format(self.msg), - SIMAPP_TRAINING_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_400) + """Logs the exception to cloud watch and exits the app""" + log_and_exit( + "Training worker failed: {}".format(self.msg), + SIMAPP_TRAINING_WORKER_EXCEPTION, + SIMAPP_EVENT_ERROR_CODE_400, + ) + class GenericRolloutException(Exception): - '''This exception is a generic exception for the rollout worker''' + """This exception is a generic exception for the rollout worker""" + def __init__(self, msg): - '''msg - This should be text containing information about what caused - the exception, for example "cannot divide by zero - ''' + """msg - This should be text containing information about what caused + the exception, for example "cannot divide by zero + """ super(GenericRolloutException, self).__init__(msg) self.msg = msg def log_except_and_exit(self): - '''Logs the exception to cloud watch and exits the sim app''' - log_and_exit("Rollout worker failed: {}" - .format(self.msg), - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_500) + """Logs the exception to cloud watch and exits the sim app""" + log_and_exit( + "Rollout worker failed: {}".format(self.msg), + SIMAPP_SIMULATION_WORKER_EXCEPTION, + SIMAPP_EVENT_ERROR_CODE_500, + ) + class GenericRolloutError(Exception): - ''''This exception is a generic error for the rollout worker''' + """'This exception is a generic error for the rollout worker""" + def __init__(self, msg): - '''msg - This should be text containing information about what caused - the exception, for example "cannot divide by zero - ''' + """msg - This should be text containing information about what caused + the exception, for example "cannot divide by zero + """ super(GenericRolloutError, self).__init__(msg) self.msg = msg def log_except_and_exit(self): - '''Logs the exception to cloud watch and exits the sim app''' - log_and_exit("Rollout worker failed: {}" - .format(self.msg), - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_400) + """Logs the exception to cloud watch and exits the sim app""" + log_and_exit( + "Rollout worker failed: {}".format(self.msg), + SIMAPP_SIMULATION_WORKER_EXCEPTION, + SIMAPP_EVENT_ERROR_CODE_400, + ) + class GenericValidatorException(Exception): - '''This exception is a generic exception for the validation worker''' + """This exception is a generic exception for the validation worker""" + def __init__(self, msg): - '''msg - This should be text containing information about what caused - the exception, for example "cannot divide by zero - ''' + """msg - This should be text containing information about what caused + the exception, for example "cannot divide by zero + """ super(GenericValidatorException, self).__init__(msg) self.msg = msg def log_except_and_exit(self): - '''Logs the exception to cloud watch and exits the sim app''' - log_and_exit("Validation worker failed: {}" - .format(self.msg), - SIMAPP_VALIDATION_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_500) + """Logs the exception to cloud watch and exits the sim app""" + log_and_exit( + "Validation worker failed: {}".format(self.msg), + SIMAPP_VALIDATION_WORKER_EXCEPTION, + SIMAPP_EVENT_ERROR_CODE_500, + ) + class GenericValidatorError(Exception): - ''''This exception is a generic error for the validation worker''' + """'This exception is a generic error for the validation worker""" + def __init__(self, msg): - '''msg - This should be text containing information about what caused - the exception, for example "cannot divide by zero - ''' + """msg - This should be text containing information about what caused + the exception, for example "cannot divide by zero + """ super(GenericValidatorError, self).__init__(msg) self.msg = msg def log_except_and_exit(self): - '''Logs the exception to cloud watch and exits the sim app''' - log_and_exit("Validation worker failed: {}" - .format(self.msg), - SIMAPP_VALIDATION_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_400) + """Logs the exception to cloud watch and exits the sim app""" + log_and_exit( + "Validation worker failed: {}".format(self.msg), + SIMAPP_VALIDATION_WORKER_EXCEPTION, + SIMAPP_EVENT_ERROR_CODE_400, + ) + class GenericException(Exception): - '''This exception is a generic exception''' + """This exception is a generic exception""" + def __init__(self, msg): - '''msg - This should be text containing information about what caused - the exception, for example "cannot divide by zero - ''' + """msg - This should be text containing information about what caused + the exception, for example "cannot divide by zero + """ super(GenericException, self).__init__(msg) self.msg = msg def log_except_and_exit(self, worker): - '''Logs the exception to cloud watch and exits the worker - worker - String indicating which worker is throwing the exception - ''' - log_and_exit("Validation worker failed: {}" - .format(self.msg), - worker, - SIMAPP_EVENT_ERROR_CODE_500) + """Logs the exception to cloud watch and exits the worker + worker - String indicating which worker is throwing the exception + """ + log_and_exit( + "Validation worker failed: {}".format(self.msg), worker, SIMAPP_EVENT_ERROR_CODE_500 + ) + class GenericError(Exception): - ''''This exception is a generic error''' + """'This exception is a generic error""" + def __init__(self, msg): - '''msg - This should be text containing information about what caused - the exception, for example "cannot divide by zero - ''' + """msg - This should be text containing information about what caused + the exception, for example "cannot divide by zero + """ super(GenericError, self).__init__(msg) self.msg = msg def log_except_and_exit(self, worker): - '''Logs the exception to cloud watch and exits the worker - worker - String indicating which worker is throwing the exception - ''' - log_and_exit("Validation worker failed: {}" - .format(self.msg), - worker, - SIMAPP_EVENT_ERROR_CODE_400) + """Logs the exception to cloud watch and exits the worker + worker - String indicating which worker is throwing the exception + """ + log_and_exit( + "Validation worker failed: {}".format(self.msg), worker, SIMAPP_EVENT_ERROR_CODE_400 + ) class GenericNonFatalException(Exception): - ''''This exception is a generic non fatal exception that can be resolved by retry. - In this case, we should log the error and not exit the simapp - ''' - def __init__(self, error_msg, - error_code=SIMAPP_EVENT_ERROR_CODE_500, - error_name=SIMAPP_EVENT_SYSTEM_ERROR): + """'This exception is a generic non fatal exception that can be resolved by retry. + In this case, we should log the error and not exit the simapp + """ + + def __init__( + self, + error_msg, + error_code=SIMAPP_EVENT_ERROR_CODE_500, + error_name=SIMAPP_EVENT_SYSTEM_ERROR, + ): """Initialize the GenericNonFatalException error type. Args: error_msg (str): The message detailing why we are raising this exception. - error_code (str, optional): The error code for this exception. + error_code (str, optional): The error code for this exception. Defaults to SIMAPP_EVENT_ERROR_CODE_500. error_name (str, optional): Gives the exception a birth name. Defaults to SIMAPP_EVENT_SYSTEM_ERROR. """ @@ -190,7 +219,7 @@ def error_code(self): str: The error code for the current non fatal exception. """ return self._error_code - + @property def error_name(self): """Return a read only version of the error message. @@ -211,9 +240,10 @@ def error_msg(self): def log_except_and_continue(self): """Log the exception and continue. - TODO: implemement logic here if we decide to log non fatal error metrics - to cloudwatch. + TODO: implemement logic here if we decide to log non fatal error metrics + to cloudwatch. """ - error_msg = "DeepRacer non-fatal error - code: {} message: {}".format(self._error_code, - self._error_msg) + error_msg = "DeepRacer non-fatal error - code: {} message: {}".format( + self._error_code, self._error_msg + ) LOG.error(error_msg) diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/log_handler/exception_handler.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/log_handler/exception_handler.py index 6484e1a97c..1fdc9743d7 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/log_handler/exception_handler.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/log_handler/exception_handler.py @@ -1,24 +1,25 @@ # Handles logging and graceful exit +import datetime +import inspect +import io import json import logging import os -import io -import datetime -import inspect -from collections import OrderedDict -import traceback import re +import traceback +from collections import OrderedDict + from markov.log_handler.constants import ( + ERROR_HANDLER_EXCEPTION_FAULT_CODE, + EXCEPTION_HANDLER_SYNC_FILE, + FAULT_MAP, + SIMAPP_ERROR_EXIT, SIMAPP_ERROR_HANDLER_EXCEPTION, + SIMAPP_EVENT_ERROR_CODE_400, + SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_EVENT_SYSTEM_ERROR, SIMAPP_EVENT_USER_ERROR, - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_EVENT_ERROR_CODE_400, - SIMAPP_ERROR_EXIT, - FAULT_MAP, UNCLASSIFIED_FAULT_CODE, - EXCEPTION_HANDLER_SYNC_FILE, - ERROR_HANDLER_EXCEPTION_FAULT_CODE, ) from markov.log_handler.logger import Logger diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/memories/deepracer_memory.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/memories/deepracer_memory.py index ec52c8948c..72f17498cd 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/memories/deepracer_memory.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/memories/deepracer_memory.py @@ -1,21 +1,17 @@ import ast - +import math import pickle +import random from copy import deepcopy - -import math - -import pandas as pd from typing import List, Tuple, Union -import numpy as np -import random -from rl_coach.core_types import Transition, Episode +import numpy as np +import pandas as pd +from rl_coach.core_types import CsvDataset, Episode, Transition from rl_coach.filters.filter import InputFilter from rl_coach.logger import screen from rl_coach.memories.memory import Memory, MemoryGranularity, MemoryParameters -from rl_coach.utils import ReaderWriterLock, ProgressBar -from rl_coach.core_types import CsvDataset +from rl_coach.utils import ProgressBar, ReaderWriterLock class DeepRacerMemoryParameters(MemoryParameters): @@ -72,7 +68,7 @@ def length(self, lock: bool = False) -> int: return length def num_complete_episodes(self): - """ Get the number of complete episodes in ER """ + """Get the number of complete episodes in ER""" length = self._length - 1 return length diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/metrics/constants.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/metrics/constants.py index feb74d7341..d42f8e1306 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/metrics/constants.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/metrics/constants.py @@ -1,8 +1,9 @@ """This module houses all the constants for the metrics package""" -from enum import Enum, unique -from collections import OrderedDict -import os import logging +import os +from collections import OrderedDict +from enum import Enum, unique + from markov.log_handler.logger import Logger LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/metrics/s3_metrics.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/metrics/s3_metrics.py index 0f112fd046..03f438347e 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/metrics/s3_metrics.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/metrics/s3_metrics.py @@ -1,43 +1,44 @@ """This module houses the metric objects for the sim app""" -import math -import time import json import logging +import math import os -from collections import OrderedDict import statistics +import time +from collections import OrderedDict + import boto3 import botocore import rospy -from deepracer_simulation_environment.srv import VideoMetricsSrvResponse, VideoMetricsSrv +from deepracer_simulation_environment.srv import VideoMetricsSrv, VideoMetricsSrvResponse from geometry_msgs.msg import Point32 -from markov.constants import BEST_CHECKPOINT, LAST_CHECKPOINT, METRICS_VERSION +from markov.boto.s3.constants import ( + SIMTRACE_EVAL_LOCAL_PATH_FORMAT, + SIMTRACE_TRAINING_LOCAL_PATH_FORMAT, +) +from markov.boto.s3.files.metrics import Metrics from markov.common import ObserverInterface +from markov.constants import BEST_CHECKPOINT, LAST_CHECKPOINT, METRICS_VERSION +from markov.gazebo_tracker.abs_tracker import AbstractTracker +from markov.gazebo_tracker.constants import TrackerPriority +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_SIMULATION_WORKER_EXCEPTION, +) +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger from markov.metrics.constants import ( - MetricsS3Keys, - StepMetrics, + BestModelMetricType, EpisodeStatus, + MetricsS3Keys, Mp4VideoMetrics, - BestModelMetricType, + StepMetrics, ) from markov.metrics.metrics_interface import MetricsInterface +from markov.track_geom.track_data import TrackData from markov.utils import get_boto_config, get_s3_kms_extra_args -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.constants import ( - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_500, -) from rl_coach.checkpoint import CheckpointStateFile from rl_coach.core_types import RunPhase -from markov.gazebo_tracker.abs_tracker import AbstractTracker -from markov.gazebo_tracker.constants import TrackerPriority -from markov.track_geom.track_data import TrackData -from markov.boto.s3.constants import ( - SIMTRACE_EVAL_LOCAL_PATH_FORMAT, - SIMTRACE_TRAINING_LOCAL_PATH_FORMAT, -) -from markov.boto.s3.files.metrics import Metrics LOGGER = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/action_space_configs.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/action_space_configs.py index 577b50c439..72a002f980 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/action_space_configs.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/action_space_configs.py @@ -1,15 +1,16 @@ """This module contains the available action space configurations for the training algorithms""" import abc -import numpy as np import logging + +import numpy as np +from markov.boto.s3.constants import ActionSpaceTypes, ModelMetadataKeys, TrainingAlgorithm from markov.log_handler.constants import ( SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_SIMULATION_WORKER_EXCEPTION, ) from markov.log_handler.exception_handler import log_and_exit -from rl_coach.spaces import DiscreteActionSpace, BoxActionSpace -from markov.boto.s3.constants import ModelMetadataKeys, TrainingAlgorithm, ActionSpaceTypes from markov.multi_agent_coach.spaces import ScalableBoxActionSpace +from rl_coach.spaces import BoxActionSpace, DiscreteActionSpace class ActionSpaceConfigInterface(object, metaclass=abc.ABCMeta): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/agents/sac_agent.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/agents/sac_agent.py index 405ae9d986..310f83d1ed 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/agents/sac_agent.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/agents/sac_agent.py @@ -14,33 +14,33 @@ # limitations under the License. # -from typing import Union import copy -import numpy as np from collections import OrderedDict +from typing import Union -from rl_coach.agents.agent import Agent -from rl_coach.agents.policy_optimization_agent import PolicyOptimizationAgent +import numpy as np +import tensorflow as tf from markov.multi_agent_coach.architectures.head_parameters import ( - SACQHeadParameters, SACPolicyHeadParameters, + SACQHeadParameters, ) +from rl_coach.agents.agent import Agent +from rl_coach.agents.policy_optimization_agent import PolicyOptimizationAgent +from rl_coach.architectures.embedder_parameters import InputEmbedderParameters from rl_coach.architectures.head_parameters import VHeadParameters from rl_coach.architectures.middleware_parameters import FCMiddlewareParameters +from rl_coach.architectures.network_wrapper import NetworkWrapper from rl_coach.base_parameters import ( - AlgorithmParameters, - NetworkParameters, AgentParameters, + AlgorithmParameters, EmbedderScheme, MiddlewareScheme, + NetworkParameters, ) from rl_coach.core_types import ActionInfo, EnvironmentSteps, RunPhase from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters from rl_coach.memories.non_episodic.experience_replay import ExperienceReplayParameters -from rl_coach.architectures.embedder_parameters import InputEmbedderParameters from rl_coach.spaces import BoxActionSpace -import tensorflow as tf -from rl_coach.architectures.network_wrapper import NetworkWrapper # There are 3 networks in SAC implementation. All have the same topology but parameters are not shared. # The networks are: diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/architectures/tensorflow_components/heads/sac_head.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/architectures/tensorflow_components/heads/sac_head.py index ff7b6befcb..13e1f1714e 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/architectures/tensorflow_components/heads/sac_head.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/architectures/tensorflow_components/heads/sac_head.py @@ -14,15 +14,14 @@ # limitations under the License. # +import numpy as np import tensorflow as tf - -from rl_coach.architectures.tensorflow_components.layers import Dense from rl_coach.architectures.tensorflow_components.heads.head import Head +from rl_coach.architectures.tensorflow_components.layers import Dense from rl_coach.base_parameters import AgentParameters from rl_coach.core_types import ActionProbabilities from rl_coach.spaces import SpacesDefinition from rl_coach.utils import eps -import numpy as np class SACPolicyHead(Head): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/architectures/tensorflow_components/heads/sac_q_head.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/architectures/tensorflow_components/heads/sac_q_head.py index 2214f074d0..89927cad53 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/architectures/tensorflow_components/heads/sac_q_head.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/architectures/tensorflow_components/heads/sac_q_head.py @@ -15,12 +15,11 @@ # import tensorflow as tf - -from rl_coach.architectures.tensorflow_components.layers import Dense from rl_coach.architectures.tensorflow_components.heads.head import Head +from rl_coach.architectures.tensorflow_components.layers import Dense from rl_coach.base_parameters import AgentParameters from rl_coach.core_types import QActionStateValue -from rl_coach.spaces import SpacesDefinition, BoxActionSpace +from rl_coach.spaces import BoxActionSpace, SpacesDefinition class SACQHead(Head): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_environment.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_environment.py index 2f54a2a9bc..9c59fb6a51 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_environment.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_environment.py @@ -1,13 +1,11 @@ import time -from typing import Union, List +from typing import List, Union import numpy as np - -from rl_coach.base_parameters import Parameters -from rl_coach.base_parameters import VisualizationParameters -from rl_coach.core_types import GoalType, ActionType, EnvResponse, RunPhase -from rl_coach.environments.environment_interface import EnvironmentInterface +from rl_coach.base_parameters import Parameters, VisualizationParameters +from rl_coach.core_types import ActionType, EnvResponse, GoalType, RunPhase from rl_coach.environments.environment import LevelSelection +from rl_coach.environments.environment_interface import EnvironmentInterface from rl_coach.spaces import ActionSpace, ObservationSpace, RewardSpace, StateSpace from rl_coach.utils import force_list diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_graph_manager.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_graph_manager.py index cb84a8d06e..dcd9cf6753 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_graph_manager.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_graph_manager.py @@ -1,3 +1,4 @@ +import contextlib import copy import glob import os @@ -5,47 +6,45 @@ from collections import OrderedDict from distutils.dir_util import copy_tree, remove_tree from typing import Dict, List, Tuple -import contextlib +import markov.deepracer_memory as deepracer_memory +from markov.multi_agent_coach.multi_agent_level_manager import MultiAgentLevelManager from rl_coach.base_parameters import ( - iterable_to_items, - TaskParameters, + AgentParameters, DistributedTaskParameters, Frameworks, - VisualizationParameters, PresetValidationParameters, RunType, - AgentParameters, + TaskParameters, + VisualizationParameters, + iterable_to_items, ) from rl_coach.checkpoint import ( + CheckpointState, + CheckpointStateReader, CheckpointStateUpdater, - get_checkpoint_state, SingleCheckpoint, - CheckpointState, + get_checkpoint_state, ) from rl_coach.core_types import ( - TotalStepsCounter, - RunPhase, - PlayingStepsType, - TrainingSteps, EnvironmentEpisodes, EnvironmentSteps, + PlayingStepsType, + RunPhase, StepMethod, - Transition, TimeTypes, + TotalStepsCounter, + TrainingSteps, + Transition, ) +from rl_coach.data_stores.data_store import SyncFiles +from rl_coach.data_stores.data_store_impl import get_data_store as data_store_creator from rl_coach.environments.environment import Environment, EnvironmentParameters from rl_coach.graph_managers.graph_manager import ScheduleParameters -from rl_coach.logger import screen, Logger +from rl_coach.logger import Logger, screen +from rl_coach.memories.backend.memory_impl import get_memory_backend from rl_coach.saver import SaverCollection from rl_coach.utils import set_cpu, short_dynamic_import -from rl_coach.data_stores.data_store_impl import get_data_store as data_store_creator -from rl_coach.memories.backend.memory_impl import get_memory_backend -from rl_coach.data_stores.data_store import SyncFiles -from rl_coach.checkpoint import CheckpointStateReader - -import markov.deepracer_memory as deepracer_memory -from markov.multi_agent_coach.multi_agent_level_manager import MultiAgentLevelManager class MultiAgentGraphManager(object): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_level_manager.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_level_manager.py index 214f3d68e9..9b8477a86a 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_level_manager.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/multi_agent_level_manager.py @@ -1,14 +1,14 @@ import copy -from typing import Union, Dict +from typing import Dict, Union -from rl_coach.agents.composite_agent import CompositeAgent from rl_coach.agents.agent_interface import AgentInterface +from rl_coach.agents.composite_agent import CompositeAgent from rl_coach.core_types import ( - EnvResponse, ActionInfo, - RunPhase, ActionType, EnvironmentSteps, + EnvResponse, + RunPhase, Transition, ) from rl_coach.environments.environment import Environment diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/spaces.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/spaces.py index 1f66d27d5b..4fd99215a6 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/spaces.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/multi_agent_coach/spaces.py @@ -1,12 +1,13 @@ +from typing import Dict, List, Union + import numpy as np -from rl_coach.spaces import BoxActionSpace -from typing import Union, List, Dict +from markov.boto.s3.constants import ModelMetadataKeys from markov.log_handler.constants import ( SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_SIMULATION_WORKER_EXCEPTION, ) from markov.log_handler.exception_handler import log_and_exit -from markov.boto.s3.constants import ModelMetadataKeys +from rl_coach.spaces import BoxActionSpace class ScalableBoxActionSpace(BoxActionSpace): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/abstract_reset_rule.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/abstract_reset_rule.py index 88d473a032..308a3a43c0 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/abstract_reset_rule.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/abstract_reset_rule.py @@ -1,6 +1,7 @@ """This class defines an interface for how agents reset with the environment. """ import abc + from markov.log_handler.deepracer_exceptions import GenericRolloutException diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/reset_rules_manager.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/reset_rules_manager.py index 2796ffe702..0095096e7a 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/reset_rules_manager.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/reset_rules_manager.py @@ -1,8 +1,8 @@ """This module implement reset rule manager""" from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.metrics.constants import EpisodeStatus from markov.reset.abstract_reset_rule import AbstractResetRule from markov.reset.constants import AgentCtrlStatus, AgentInfo -from markov.metrics.constants import EpisodeStatus class ResetRulesManager: diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/crash_reset_rule.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/crash_reset_rule.py index 5b5341df09..166b8673a6 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/crash_reset_rule.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/crash_reset_rule.py @@ -1,10 +1,10 @@ """This module implements concrete reset rule for crash""" +from markov.metrics.constants import EpisodeStatus from markov.reset.abstract_reset_rule import AbstractResetRule from markov.reset.constants import AgentCtrlStatus, AgentInfo -from markov.track_geom.track_data import TrackData from markov.track_geom.constants import AgentPos -from markov.metrics.constants import EpisodeStatus +from markov.track_geom.track_data import TrackData class CrashResetRule(AbstractResetRule): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/episode_complete_reset_rule.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/episode_complete_reset_rule.py index 14c8237c50..80d23e0bfa 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/episode_complete_reset_rule.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/episode_complete_reset_rule.py @@ -1,9 +1,9 @@ """This module implements concrete reset rule for episode completion""" +from markov.metrics.constants import EpisodeStatus from markov.reset.abstract_reset_rule import AbstractResetRule from markov.reset.constants import AgentCtrlStatus, AgentInfo from markov.track_geom.track_data import TrackData -from markov.metrics.constants import EpisodeStatus class EpisodeCompleteResetRule(AbstractResetRule): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/immobilized_reset_rule.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/immobilized_reset_rule.py index 862e0c1bc2..8e9ed8802f 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/immobilized_reset_rule.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/immobilized_reset_rule.py @@ -1,10 +1,9 @@ """This module implements concrete reset rule for the immobilization""" import markov.agent_ctrl.constants as const - +from markov.metrics.constants import EpisodeStatus from markov.reset.abstract_reset_rule import AbstractResetRule from markov.reset.constants import AgentCtrlStatus, AgentPhase -from markov.metrics.constants import EpisodeStatus class ImmobilizedResetRule(AbstractResetRule): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/off_track_reset_rule.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/off_track_reset_rule.py index 0650dc2cdf..2e37e5c86d 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/off_track_reset_rule.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/off_track_reset_rule.py @@ -1,10 +1,10 @@ """This module implements concrete reset rule for off track""" +from markov.metrics.constants import EpisodeStatus from markov.reset.abstract_reset_rule import AbstractResetRule from markov.reset.constants import AgentCtrlStatus -from markov.track_geom.track_data import TrackData from markov.track_geom.constants import AgentPos -from markov.metrics.constants import EpisodeStatus +from markov.track_geom.track_data import TrackData class OffTrackResetRule(AbstractResetRule): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/race_time_rule.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/race_time_rule.py index 762adf631c..9b531bf0f9 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/race_time_rule.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/race_time_rule.py @@ -1,10 +1,11 @@ """This module implements concrete reset rule for race time up""" import logging + from markov.log_handler.logger import Logger +from markov.metrics.constants import EpisodeStatus from markov.reset.abstract_reset_rule import AbstractResetRule from markov.reset.constants import RaceCtrlStatus from markov.track_geom.track_data import TrackData -from markov.metrics.constants import EpisodeStatus LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/reverse_reset_rule.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/reverse_reset_rule.py index 7134b11f0c..bcb7fde8ee 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/reverse_reset_rule.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/rules/reverse_reset_rule.py @@ -1,9 +1,8 @@ """This module implements concrete reset rule for going reversed direction""" import rospy - +from markov.metrics.constants import EpisodeStatus from markov.reset.abstract_reset_rule import AbstractResetRule from markov.reset.constants import AgentCtrlStatus, AgentPhase -from markov.metrics.constants import EpisodeStatus class ReverseResetRule(AbstractResetRule): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/utils.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/utils.py index 25e41ada7f..efeebbcec4 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/utils.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/reset/utils.py @@ -3,11 +3,11 @@ from markov.agent_ctrl.constants import ConfigParams from markov.reset.reset_rules_manager import ResetRulesManager from markov.reset.rules.crash_reset_rule import CrashResetRule -from markov.reset.rules.immobilized_reset_rule import ImmobilizedResetRule from markov.reset.rules.episode_complete_reset_rule import EpisodeCompleteResetRule +from markov.reset.rules.immobilized_reset_rule import ImmobilizedResetRule from markov.reset.rules.off_track_reset_rule import OffTrackResetRule -from markov.reset.rules.reverse_reset_rule import ReverseResetRule from markov.reset.rules.race_time_rule import RaceTimeRule +from markov.reset.rules.reverse_reset_rule import ReverseResetRule from markov.virtual_event.constants import DEFAULT_RACE_DURATION diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_constants.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_constants.py index 5ac53daea8..972f8388f5 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_constants.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_constants.py @@ -1,4 +1,5 @@ from enum import Enum + from std_msgs.msg import ColorRGBA diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_utils.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_utils.py index c9c0a70a41..e0ad0cf595 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_utils.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_utils.py @@ -2,25 +2,25 @@ depend on ros, it should not be used in modules imported to the training worker """ import logging + import rospy -from std_msgs.msg import String -from std_srvs.srv import Empty, EmptyResponse +from deepracer_msgs.srv import GetLightNames, GetLightNamesRequest from markov.agents.utils import RunPhaseSubject from markov.common import ObserverInterface -from markov.log_handler.logger import Logger -from markov.rospy_wrappers import ServiceProxyWrapper -from markov.domain_randomizations.randomizer_manager import RandomizerManager -from markov.domain_randomizations.visual.light_randomizer import LightRandomizer -from markov.domain_randomizations.constants import GazeboServiceName from markov.constants import ( ROBOMAKER_IS_PROFILER_ON, ROBOMAKER_PROFILER_S3_BUCKET, ROBOMAKER_PROFILER_S3_PREFIX, ) +from markov.domain_randomizations.constants import GazeboServiceName +from markov.domain_randomizations.randomizer_manager import RandomizerManager +from markov.domain_randomizations.visual.light_randomizer import LightRandomizer +from markov.log_handler.logger import Logger +from markov.rospy_wrappers import ServiceProxyWrapper from markov.utils import str2bool -from deepracer_msgs.srv import GetLightNames, GetLightNamesRequest - from rl_coach.core_types import RunPhase +from std_msgs.msg import String +from std_srvs.srv import Empty, EmptyResponse LOG = Logger(__name__, logging.INFO).get_logger() # Mapping of the phase to the text to display @@ -72,7 +72,7 @@ class PhaseObserver(ObserverInterface): """ def __init__(self, topic: str, sink: RunPhaseSubject) -> None: - """topic - Topic for which to publish the phase """ + """topic - Topic for which to publish the phase""" self._phase_pub_ = rospy.Publisher(topic, String, queue_size=1) self._state_ = None sink.register(self) @@ -104,7 +104,7 @@ def configure_environment_randomizer(light_name_filter=None): def get_robomaker_profiler_env(): - """ Read robomaker profiler environment """ + """Read robomaker profiler environment""" is_profiler_on = str2bool(rospy.get_param(ROBOMAKER_IS_PROFILER_ON, False)) profiler_s3_bucker = rospy.get_param(ROBOMAKER_PROFILER_S3_BUCKET, None) profiler_s3_prefix = rospy.get_param(ROBOMAKER_PROFILER_S3_PREFIX, None) diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_worker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_worker.py index 90b35bfd9b..5ce8f6368b 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_worker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rollout_worker.py @@ -11,69 +11,67 @@ import logging import os import time -import rospy -import botocore - -from rl_coach.base_parameters import TaskParameters, DistributedCoachSynchronizationType, RunType -from rl_coach.checkpoint import CheckpointStateReader -from rl_coach.core_types import RunPhase, EnvironmentSteps -from rl_coach.logger import screen -from rl_coach.rollout_worker import should_stop -from rl_coach.utils import short_dynamic_import -from rl_coach.core_types import EnvironmentEpisodes +import botocore +import rospy from markov import utils -from markov.constants import ROLLOUT_WORKER_PROFILER_PATH -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit, simapp_exit_gracefully -from markov.log_handler.constants import ( - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_500, -) from markov.agent_ctrl.constants import ConfigParams from markov.agents.rollout_agent_factory import ( - create_rollout_agent, - create_obstacles_agent, create_bot_cars_agent, + create_obstacles_agent, + create_rollout_agent, ) from markov.agents.utils import RunPhaseSubject +from markov.boto.s3.constants import ( + CAMERA_45DEGREE_LOCAL_PATH_FORMAT, + CAMERA_PIP_MP4_LOCAL_PATH_FORMAT, + CAMERA_TOPVIEW_LOCAL_PATH_FORMAT, + HYPERPARAMETER_LOCAL_PATH_FORMAT, + HYPERPARAMETER_S3_POSTFIX, + IP_ADDRESS_LOCAL_PATH, + MODEL_METADATA_LOCAL_PATH_FORMAT, + REWARD_FUCTION_LOCAL_PATH_FORMAT, + SIMTRACE_TRAINING_LOCAL_PATH_FORMAT, + ModelMetadataKeys, + SimtraceVideoNames, +) +from markov.boto.s3.files.checkpoint import Checkpoint +from markov.boto.s3.files.hyperparameters import Hyperparameters +from markov.boto.s3.files.ip_config import IpConfig +from markov.boto.s3.files.model_metadata import ModelMetadata +from markov.boto.s3.files.reward_function import RewardFunction +from markov.boto.s3.files.simtrace_video import SimtraceVideo +from markov.boto.s3.s3_client import S3Client +from markov.boto.s3.utils import get_s3_key +from markov.camera_utils import configure_camera +from markov.constants import ROLLOUT_WORKER_PROFILER_PATH +from markov.deepracer_memory import DeepRacerRedisPubSubMemoryBackendParameters +from markov.environments.constants import LINK_NAMES, STEERING_TOPICS, VELOCITY_TOPICS +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_SIMULATION_WORKER_EXCEPTION, +) from markov.log_handler.deepracer_exceptions import GenericRolloutError, GenericRolloutException -from markov.environments.constants import VELOCITY_TOPICS, STEERING_TOPICS, LINK_NAMES +from markov.log_handler.exception_handler import log_and_exit, simapp_exit_gracefully +from markov.log_handler.logger import Logger +from markov.metrics.constants import MetricsS3Keys +from markov.metrics.iteration_data import IterationData from markov.metrics.s3_metrics import TrainingMetrics from markov.rollout_utils import ( PhaseObserver, - signal_robomaker_markov_package_ready, configure_environment_randomizer, get_robomaker_profiler_env, + signal_robomaker_markov_package_ready, ) -from markov.metrics.iteration_data import IterationData -from markov.metrics.constants import MetricsS3Keys +from markov.rospy_wrappers import ServiceProxyWrapper from markov.s3_boto_data_store import S3BotoDataStore, S3BotoDataStoreParameters from markov.sagemaker_graph_manager import get_graph_manager -from markov.rospy_wrappers import ServiceProxyWrapper -from markov.camera_utils import configure_camera -from markov.deepracer_memory import DeepRacerRedisPubSubMemoryBackendParameters -from markov.boto.s3.files.hyperparameters import Hyperparameters -from markov.boto.s3.files.model_metadata import ModelMetadata -from markov.boto.s3.files.reward_function import RewardFunction -from markov.boto.s3.files.simtrace_video import SimtraceVideo -from markov.boto.s3.files.ip_config import IpConfig -from markov.boto.s3.files.checkpoint import Checkpoint -from markov.boto.s3.utils import get_s3_key -from markov.boto.s3.constants import ( - HYPERPARAMETER_LOCAL_PATH_FORMAT, - MODEL_METADATA_LOCAL_PATH_FORMAT, - REWARD_FUCTION_LOCAL_PATH_FORMAT, - HYPERPARAMETER_S3_POSTFIX, - SIMTRACE_TRAINING_LOCAL_PATH_FORMAT, - CAMERA_PIP_MP4_LOCAL_PATH_FORMAT, - CAMERA_45DEGREE_LOCAL_PATH_FORMAT, - CAMERA_TOPVIEW_LOCAL_PATH_FORMAT, - SimtraceVideoNames, - IP_ADDRESS_LOCAL_PATH, - ModelMetadataKeys, -) -from markov.boto.s3.s3_client import S3Client +from rl_coach.base_parameters import DistributedCoachSynchronizationType, RunType, TaskParameters +from rl_coach.checkpoint import CheckpointStateReader +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, RunPhase +from rl_coach.logger import screen +from rl_coach.rollout_worker import should_stop +from rl_coach.utils import short_dynamic_import from std_srvs.srv import Empty, EmptyRequest logger = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rospy_wrappers.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rospy_wrappers.py index 03231e5469..01412a7b9e 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rospy_wrappers.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/rospy_wrappers.py @@ -1,15 +1,15 @@ """This module is intended for any wrappers that are needed for rospy""" +import logging import time + import rospy -import logging -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit +from markov.constants import ROBOMAKER_CANCEL_JOB_WAIT_TIME from markov.log_handler.constants import ( - SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_SIMULATION_WORKER_EXCEPTION, ) -from markov.constants import ROBOMAKER_CANCEL_JOB_WAIT_TIME - +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger ROS_SERVICE_ERROR_MSG_FORMAT = "ROS Service {0} call failed, Re-try count: {1}/{2}: {3}" diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/s3_boto_data_store.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/s3_boto_data_store.py index eae4db96e4..c841a8aec6 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/s3_boto_data_store.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/s3_boto_data_store.py @@ -1,29 +1,29 @@ import io import logging import os -import time import queue +import time from typing import Dict -import botocore -import boto3 -from rl_coach.checkpoint import CheckpointStateFile, _filter_checkpoint_files -from rl_coach.data_stores.data_store import DataStore, DataStoreParameters, SyncFiles -from markov.multi_agent_coach.multi_agent_graph_manager import MultiAgentGraphManager -from markov.utils import get_s3_kms_extra_args -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.deepracer_exceptions import GenericNonFatalException +import boto3 +import botocore +import tensorflow as tf +from markov.boto.s3.files.checkpoint import Checkpoint from markov.log_handler.constants import ( - SIMAPP_EVENT_SYSTEM_ERROR, - SIMAPP_EVENT_USER_ERROR, SIMAPP_EVENT_ERROR_CODE_400, SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_EVENT_SYSTEM_ERROR, + SIMAPP_EVENT_USER_ERROR, SIMAPP_S3_DATA_STORE_EXCEPTION, SIMAPP_SIMULATION_WORKER_EXCEPTION, ) -from markov.boto.s3.files.checkpoint import Checkpoint -import tensorflow as tf +from markov.log_handler.deepracer_exceptions import GenericNonFatalException +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger +from markov.multi_agent_coach.multi_agent_graph_manager import MultiAgentGraphManager +from markov.utils import get_s3_kms_extra_args +from rl_coach.checkpoint import CheckpointStateFile, _filter_checkpoint_files +from rl_coach.data_stores.data_store import DataStore, DataStoreParameters, SyncFiles LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sagemaker_graph_manager.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sagemaker_graph_manager.py index b733da979f..ebcb179a81 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sagemaker_graph_manager.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sagemaker_graph_manager.py @@ -1,39 +1,40 @@ import json + from markov.architecture.constants import Input from markov.architecture.embedder_factory import create_input_embedder, create_middle_embedder +from markov.boto.s3.constants import TrainingAlgorithm from markov.constants import ExplorationTypes, HyperParameterKeys, LossTypes from markov.environments.deepracer_racetrack_env import DeepRacerRacetrackEnvParameters -from markov.log_handler.exception_handler import log_and_exit +from markov.exploration_policies.deepracer_categorical import DeepRacerCategoricalParameters +from markov.filters.observation.observation_binary_filter import ObservationBinarySectorFilter from markov.log_handler.constants import ( SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_SIMULATION_WORKER_EXCEPTION, ) -from markov.multi_agent_coach.multi_agent_graph_manager import MultiAgentGraphManager +from markov.log_handler.exception_handler import log_and_exit +from markov.memories.deepracer_memory import DeepRacerMemoryParameters from markov.multi_agent_coach.agents.sac_agent import SoftActorCriticAgentParameters +from markov.multi_agent_coach.multi_agent_graph_manager import MultiAgentGraphManager from markov.multi_agent_coach.spaces import ScalableBoxActionSpace -from markov.memories.deepracer_memory import DeepRacerMemoryParameters -from markov.boto.s3.constants import TrainingAlgorithm -from rl_coach.spaces import DiscreteActionSpace -from rl_coach.memories.non_episodic.experience_replay import ExperienceReplayParameters -from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, DistributedCoachSynchronizationType, + PresetValidationParameters, RunType, + VisualizationParameters, ) -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps -from markov.exploration_policies.deepracer_categorical import DeepRacerCategoricalParameters +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, TrainingSteps +from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters from rl_coach.exploration_policies.e_greedy import EGreedyParameters from rl_coach.filters.filter import InputFilter +from rl_coach.filters.observation.observation_clipping_filter import ObservationClippingFilter from rl_coach.filters.observation.observation_rgb_to_y_filter import ObservationRGBToYFilter from rl_coach.filters.observation.observation_stacking_filter import ObservationStackingFilter from rl_coach.filters.observation.observation_to_uint8_filter import ObservationToUInt8Filter -from rl_coach.filters.observation.observation_clipping_filter import ObservationClippingFilter -from markov.filters.observation.observation_binary_filter import ObservationBinarySectorFilter from rl_coach.graph_managers.graph_manager import ScheduleParameters +from rl_coach.memories.non_episodic.experience_replay import ExperienceReplayParameters from rl_coach.schedules import LinearSchedule +from rl_coach.spaces import DiscreteActionSpace class DeepRacerClippedPPOAgentParams(ClippedPPOAgentParameters): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/samples/sample_collector.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/samples/sample_collector.py index c4c153a8fc..a2276955b7 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/samples/sample_collector.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/samples/sample_collector.py @@ -1,9 +1,9 @@ -import pickle import os +import pickle -from markov.utils import get_s3_kms_extra_args from markov.boto.s3.s3_client import S3Client from markov.log_handler.deepracer_exceptions import GenericTrainerException +from markov.utils import get_s3_kms_extra_args class SampleCollector: diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/composite_sensor.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/composite_sensor.py index 954e170e64..38b057ec8c 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/composite_sensor.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/composite_sensor.py @@ -1,8 +1,7 @@ """This module contains a composite sensor class for supporting agents with multiple sensors""" +from markov.sensors.sensor_interface import LidarInterface, SensorInterface from rl_coach.spaces import StateSpace -from markov.sensors.sensor_interface import SensorInterface, LidarInterface - class CompositeSensor(SensorInterface): """This class represents a composite sensor so that from the point of view of each agent there diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensor_interface.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensor_interface.py index 829c041cb4..69edbab855 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensor_interface.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensor_interface.py @@ -11,7 +11,7 @@ class SensorInterface(object, metaclass=abc.ABCMeta): @abc.abstractmethod def get_observation_space(self): - """ Return the observation space for this sensor """ + """Return the observation space for this sensor""" raise NotImplementedError("Sensor object must configure observation space") @abc.abstractmethod @@ -23,7 +23,7 @@ def get_state(self, block=True): @abc.abstractmethod def reset(self): - """ Reset the sensor data """ + """Reset the sensor data""" raise NotImplementedError("Sensor object must reset") @abc.abstractmethod diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensors_rollout.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensors_rollout.py index 6c508ba974..20aab8daac 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensors_rollout.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensors_rollout.py @@ -1,26 +1,26 @@ """This module contains the available sensors for the sim app""" import logging -import rospy + import numpy as np -from sensor_msgs.msg import Image as sensor_image -from sensor_msgs.msg import LaserScan -from PIL import Image +import rospy +from markov import utils +from markov.architecture.constants import Input +from markov.environments.constants import TRAINING_IMAGE_SIZE +from markov.log_handler.constants import SIMAPP_SIMULATION_WORKER_EXCEPTION +from markov.log_handler.deepracer_exceptions import GenericError, GenericRolloutException +from markov.log_handler.logger import Logger +from markov.sensors.sensor_interface import LidarInterface, SensorInterface from markov.sensors.utils import ( - get_observation_space, get_front_camera_embedders, get_left_camera_embedders, - get_stereo_camera_embedders, get_lidar_embedders, get_observation_embedder, + get_observation_space, + get_stereo_camera_embedders, ) -from markov.sensors.sensor_interface import SensorInterface, LidarInterface -from markov.environments.constants import TRAINING_IMAGE_SIZE -from markov.architecture.constants import Input -from markov.log_handler.deepracer_exceptions import GenericRolloutException, GenericError -from markov import utils -from markov.log_handler.logger import Logger -from markov.log_handler.constants import SIMAPP_SIMULATION_WORKER_EXCEPTION - +from PIL import Image +from sensor_msgs.msg import Image as sensor_image +from sensor_msgs.msg import LaserScan LOGGER = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensors_training.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensors_training.py index 966ade8f70..864d4bfc85 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensors_training.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/sensors_training.py @@ -1,16 +1,16 @@ """This module contains the available sensors for the sim app""" from markov.architecture.constants import Input -from markov.sensors.sensor_interface import SensorInterface, LidarInterface +from markov.log_handler.constants import SIMAPP_TRAINING_WORKER_EXCEPTION +from markov.log_handler.deepracer_exceptions import GenericError, GenericTrainerException +from markov.sensors.sensor_interface import LidarInterface, SensorInterface from markov.sensors.utils import ( - get_observation_space, get_front_camera_embedders, get_left_camera_embedders, - get_stereo_camera_embedders, get_lidar_embedders, get_observation_embedder, + get_observation_space, + get_stereo_camera_embedders, ) -from markov.log_handler.deepracer_exceptions import GenericTrainerException, GenericError -from markov.log_handler.constants import SIMAPP_TRAINING_WORKER_EXCEPTION class SensorFactory(object): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/utils.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/utils.py index eb234db44a..2dea118183 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/utils.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/sensors/utils.py @@ -1,18 +1,17 @@ """This module houses all utility methods for the sensor module""" import numpy as np - +from markov.architecture.constants import ActivationFunctions, Input, NeuralNetwork, SchemeInfo from markov.environments.constants import ( + SECTOR_LIDAR_CLIPPING_DIST, TRAINING_IMAGE_SIZE, TRAINING_LIDAR_SIZE, - SECTOR_LIDAR_CLIPPING_DIST, ) -from markov.architecture.constants import SchemeInfo, Input, ActivationFunctions, NeuralNetwork from markov.log_handler.deepracer_exceptions import GenericError from rl_coach.spaces import ( - StateSpace, ImageObservationSpace, - VectorObservationSpace, PlanarMapsObservationSpace, + StateSpace, + VectorObservationSpace, ) diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/state_machine/fsm.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/state_machine/fsm.py index 5ce470d258..6489cc05e5 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/state_machine/fsm.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/state_machine/fsm.py @@ -9,7 +9,7 @@ class FSM(object): """ def __init__(self, initial_state): - """ Initialize the components. """ + """Initialize the components.""" # Start state machine with an initial state self._current_state = initial_state diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/constants.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/constants.py index 85ab492e72..ab5fcf1176 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/constants.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/constants.py @@ -57,7 +57,7 @@ class AgentPos(Enum): class ObstacleDimensions(Enum): - """ The dimensions of different obstacle """ + """The dimensions of different obstacle""" BOX_OBSTACLE_DIMENSION = (0.4, 0.5) # Length(x), width(y) + 0.1 buffer for each # TODO: Fix the bot car dimension so that all collisions are detected diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/abstract_spline.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/abstract_spline.py index d7d5e64724..b8665972dc 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/abstract_spline.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/abstract_spline.py @@ -1,10 +1,9 @@ """This class defines an abstract spline""" import abc -from scipy.interpolate import spalde - from markov.track_geom.constants import SPLINE_DEGREE from markov.track_geom.track_data import TrackData +from scipy.interpolate import spalde # Python 2 and 3 compatible Abstract class ABC = abc.ABCMeta("ABC", (object,), {}) diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/lane_change_spline.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/lane_change_spline.py index f4fdfa0c39..b455bf36ca 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/lane_change_spline.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/lane_change_spline.py @@ -1,17 +1,16 @@ """This module implements concrete lane change spline""" import bisect -import numpy as np import random +import numpy as np +from markov.track_geom.constants import DIST_THRESHOLD, SPLINE_DEGREE +from markov.track_geom.spline.abstract_spline import AbstractSpline +from markov.track_geom.track_data import TrackLine from scipy.interpolate import splprep from shapely.geometry import Point from shapely.geometry.polygon import LineString -from markov.track_geom.constants import SPLINE_DEGREE, DIST_THRESHOLD -from markov.track_geom.track_data import TrackLine -from markov.track_geom.spline.abstract_spline import AbstractSpline - class LaneChangeSpline(AbstractSpline): def __init__( diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/track_spline.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/track_spline.py index bc5e5aa1b9..c1cc39d0d2 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/track_spline.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/spline/track_spline.py @@ -1,12 +1,10 @@ """This module implements concrete track spline""" import numpy as np - -from scipy.interpolate import splprep -from shapely.geometry import Point - from markov.track_geom.constants import SPLINE_DEGREE, TrackLane from markov.track_geom.spline.abstract_spline import AbstractSpline +from scipy.interpolate import splprep +from shapely.geometry import Point class TrackSpline(AbstractSpline): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/track_data.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/track_data.py index 655e381d85..6e707941aa 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/track_data.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/track_data.py @@ -1,28 +1,27 @@ """This module is used to manage the track related data""" -from collections import OrderedDict, deque -from enum import Enum, unique import math import os import threading +from collections import OrderedDict, deque +from enum import Enum, unique + import numpy as np import rospkg import rospy - from geometry_msgs.msg import Pose -from shapely.geometry import Point, Polygon -from shapely.geometry.polygon import LinearRing, LineString - +from markov import utils from markov.agent_ctrl.constants import RewardParam from markov.cameras.frustum_manager import FrustumManager -from markov.track_geom.constants import TrackNearPnts, TrackNearDist, ParkLocation +from markov.log_handler.deepracer_exceptions import GenericRolloutException +from markov.track_geom.constants import ParkLocation, TrackNearDist, TrackNearPnts from markov.track_geom.utils import ( - euler_to_quaternion, apply_orientation, + euler_to_quaternion, find_prev_next, quaternion_to_euler, ) -from markov.log_handler.deepracer_exceptions import GenericRolloutException -from markov import utils +from shapely.geometry import Point, Polygon +from shapely.geometry.polygon import LinearRing, LineString @unique diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/utils.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/utils.py index 7563983e0f..6f7dec902a 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/utils.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/track_geom/utils.py @@ -1,8 +1,9 @@ """This module should house utility methods for the track data classes""" import bisect -import numpy as np import math -from markov.track_geom.constants import START_POS_OFFSET, HIDE_POS_OFFSET, HIDE_POS_DELTA + +import numpy as np +from markov.track_geom.constants import HIDE_POS_DELTA, HIDE_POS_OFFSET, START_POS_OFFSET # The order of rotation applied: roll -> pitch -> yaw diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/training_worker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/training_worker.py index 0a48d1a84f..2a55be8e70 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/training_worker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/training_worker.py @@ -1,48 +1,46 @@ -import os -import shutil import argparse import json import logging import math +import os +import shutil + import botocore import numpy as np - -from rl_coach.base_parameters import TaskParameters, DistributedCoachSynchronizationType, RunType -from rl_coach import core_types -from rl_coach.data_stores.data_store import SyncFiles -from rl_coach.logger import screen -from rl_coach.utils import short_dynamic_import - +import tensorflow as tf from markov import utils -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.constants import ( - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_TRAINING_WORKER_EXCEPTION, -) -from markov.constants import SIMAPP_VERSION_2, TRAINING_WORKER_PROFILER_PATH from markov.agent_ctrl.constants import ConfigParams from markov.agents.training_agent_factory import create_training_agent -from markov.s3_boto_data_store import S3BotoDataStore, S3BotoDataStoreParameters -from markov.boto.s3.constants import TrainingAlgorithm -from markov.sagemaker_graph_manager import get_graph_manager -from markov.samples.sample_collector import SampleCollector -from markov.deepracer_memory import DeepRacerRedisPubSubMemoryBackendParameters -from markov.boto.s3.files.hyperparameters import Hyperparameters -from markov.boto.s3.files.model_metadata import ModelMetadata -from markov.boto.s3.files.ip_config import IpConfig -from markov.boto.s3.files.checkpoint import Checkpoint -from markov.boto.s3.utils import get_s3_key from markov.boto.s3.constants import ( + FROZEN_HEAD_OUTPUT_GRAPH_FORMAT_MAPPING, + HYPERPARAMETER_S3_POSTFIX, MODEL_METADATA_LOCAL_PATH_FORMAT, MODEL_METADATA_S3_POSTFIX, - HYPERPARAMETER_S3_POSTFIX, - FROZEN_HEAD_OUTPUT_GRAPH_FORMAT_MAPPING, ModelMetadataKeys, + TrainingAlgorithm, ) +from markov.boto.s3.files.checkpoint import Checkpoint +from markov.boto.s3.files.hyperparameters import Hyperparameters +from markov.boto.s3.files.ip_config import IpConfig +from markov.boto.s3.files.model_metadata import ModelMetadata from markov.boto.s3.s3_client import S3Client - -import tensorflow as tf +from markov.boto.s3.utils import get_s3_key +from markov.constants import SIMAPP_VERSION_2, TRAINING_WORKER_PROFILER_PATH +from markov.deepracer_memory import DeepRacerRedisPubSubMemoryBackendParameters +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_TRAINING_WORKER_EXCEPTION, +) +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger +from markov.s3_boto_data_store import S3BotoDataStore, S3BotoDataStoreParameters +from markov.sagemaker_graph_manager import get_graph_manager +from markov.samples.sample_collector import SampleCollector +from rl_coach import core_types +from rl_coach.base_parameters import DistributedCoachSynchronizationType, RunType, TaskParameters +from rl_coach.data_stores.data_store import SyncFiles +from rl_coach.logger import screen +from rl_coach.utils import short_dynamic_import tf.logging.set_verbosity(tf.logging.DEBUG) diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/utils.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/utils.py index 8ccd1af3c1..dd9508630f 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/utils.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/utils.py @@ -1,46 +1,46 @@ -import threading +import cProfile +import io import json import logging import os -import io +import pstats +import random import re +import shutil import signal import socket +import threading import time -import cProfile -import pstats -import random - from itertools import count -from markov.log_handler.constants import ( - SIMAPP_ENVIRONMENT_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_S3_DATA_STORE_EXCEPTION, - SIMAPP_EVENT_ERROR_CODE_400, - SIMAPP_SIMULATION_WORKER_EXCEPTION, - SIMAPP_DONE_EXIT, -) -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit, simapp_exit_gracefully -from markov.log_handler.deepracer_exceptions import GenericException + +import boto3 +import botocore from markov.constants import ( - ROBOMAKER_CANCEL_JOB_WAIT_TIME, - NUM_RETRIES, - CONNECT_TIMEOUT, BEST_CHECKPOINT, + CONNECT_TIMEOUT, + HYPERPARAMETERS, LAST_CHECKPOINT, - SAGEMAKER_S3_KMS_CMK_ARN, + NUM_RETRIES, + ROBOMAKER_CANCEL_JOB_WAIT_TIME, ROBOMAKER_S3_KMS_CMK_ARN, S3_KMS_CMK_ARN_ENV, - HYPERPARAMETERS, SAGEMAKER_IS_PROFILER_ON, SAGEMAKER_PROFILER_S3_BUCKET, SAGEMAKER_PROFILER_S3_PREFIX, + SAGEMAKER_S3_KMS_CMK_ARN, S3KmsEncryption, ) -import boto3 -import botocore -import shutil +from markov.log_handler.constants import ( + SIMAPP_DONE_EXIT, + SIMAPP_ENVIRONMENT_EXCEPTION, + SIMAPP_EVENT_ERROR_CODE_400, + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_S3_DATA_STORE_EXCEPTION, + SIMAPP_SIMULATION_WORKER_EXCEPTION, +) +from markov.log_handler.deepracer_exceptions import GenericException +from markov.log_handler.exception_handler import log_and_exit, simapp_exit_gracefully +from markov.log_handler.logger import Logger logger = Logger(__name__, logging.INFO).get_logger() @@ -82,8 +82,8 @@ def cancel_simulation_job(backoff_time_sec=1.0, max_retry_attempts=5): max_retry_attempts (int): maximum number of retry """ import rospy - from robomaker_simulation_msgs.srv import Cancel from markov.rospy_wrappers import ServiceProxyWrapper + from robomaker_simulation_msgs.srv import Cancel requestCancel = ServiceProxyWrapper("/robomaker/job/cancel", Cancel) @@ -104,7 +104,7 @@ def cancel_simulation_job(backoff_time_sec=1.0, max_retry_attempts=5): def str2bool(flag): - """ bool: convert flag to boolean if it is string and return it else return its initial bool value """ + """bool: convert flag to boolean if it is string and return it else return its initial bool value""" if not isinstance(flag, bool): if flag.lower() == "false": flag = False @@ -332,7 +332,7 @@ def test_internet_connection(aws_region): def get_sagemaker_profiler_env(): - """ Read the sagemaker profiler environment variables """ + """Read the sagemaker profiler environment variables""" is_profiler_on, profiler_s3_bucker, profiler_s3_prefix = False, None, None hyperparams = os.environ.get("SM_TRAINING_ENV", "") hyperparams_dict = json.loads(hyperparams) @@ -431,7 +431,7 @@ def enable_profiling(self, val): self._enable_profiling = val def start(self): - """ Start the profiler """ + """Start the profiler""" if self.enable_profiling: if not self._profiler: self._profiler = cProfile.Profile() @@ -441,7 +441,7 @@ def start(self): raise GenericException("Profiler is in use!") def stop(self): - """ Stop the profiler and upload the data to S3 """ + """Stop the profiler and upload the data to S3""" if self._profiler_owner == self: if self._profiler: self._profiler.disable() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/validation_worker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/validation_worker.py index 74a1788b5a..32d8196d5b 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/validation_worker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/validation_worker.py @@ -15,29 +15,28 @@ import logging import pickle -from rl_coach.base_parameters import TaskParameters -from rl_coach.logger import screen -from rl_coach.data_stores.data_store import SyncFiles -from rl_coach.core_types import EnvironmentSteps, RunPhase - from markov import utils +from markov.agent_ctrl.constants import ConfigParams +from markov.agents.training_agent_factory import create_training_agent +from markov.architecture.constants import Input +from markov.boto.s3.constants import MODEL_METADATA_S3_POSTFIX, ModelMetadataKeys +from markov.boto.s3.files.checkpoint import Checkpoint +from markov.boto.s3.files.model_metadata import ModelMetadata +from markov.boto.s3.utils import get_s3_key from markov.constants import BEST_CHECKPOINT, LAST_CHECKPOINT, SIMAPP_VERSION_2 -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit from markov.log_handler.constants import ( - SIMAPP_VALIDATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_400, SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_VALIDATION_WORKER_EXCEPTION, ) -from markov.agent_ctrl.constants import ConfigParams -from markov.agents.training_agent_factory import create_training_agent +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger from markov.s3_boto_data_store import S3BotoDataStore, S3BotoDataStoreParameters from markov.sagemaker_graph_manager import get_graph_manager -from markov.architecture.constants import Input -from markov.boto.s3.files.model_metadata import ModelMetadata -from markov.boto.s3.files.checkpoint import Checkpoint -from markov.boto.s3.utils import get_s3_key -from markov.boto.s3.constants import MODEL_METADATA_S3_POSTFIX, ModelMetadataKeys +from rl_coach.base_parameters import TaskParameters +from rl_coach.core_types import EnvironmentSteps, RunPhase +from rl_coach.data_stores.data_store import SyncFiles +from rl_coach.logger import screen logger = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event/utils.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event/utils.py index a1be758a5e..c4ca3e4c9d 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event/utils.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event/utils.py @@ -1,20 +1,21 @@ """This module prodives utils for virtual event.""" -import logging import json +import logging from datetime import datetime + from jsonschema import validate from jsonschema.exceptions import ValidationError -from markov.log_handler.logger import Logger +from markov.log_handler.constants import SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_EVENT_SYSTEM_ERROR from markov.log_handler.deepracer_exceptions import GenericNonFatalException -from markov.log_handler.constants import SIMAPP_EVENT_SYSTEM_ERROR, SIMAPP_EVENT_ERROR_CODE_500 +from markov.log_handler.logger import Logger from markov.virtual_event.constants import ( - WebRTCCarControl, + CAR_CONTROL_INPUT_SCHEMA, + RACER_INFO_JSON_SCHEMA, + TIME_FORMAT, CarControlMode, CarControlStatus, CarControlTopic, - RACER_INFO_JSON_SCHEMA, - CAR_CONTROL_INPUT_SCHEMA, - TIME_FORMAT, + WebRTCCarControl, ) LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event/virtual_event_manager.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event/virtual_event_manager.py index 5eabf81999..e4f4d2fdb3 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event/virtual_event_manager.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event/virtual_event_manager.py @@ -1,69 +1,48 @@ """This module is responsible for launching virtual event jobs""" +import io import json import logging +import math import os -import io +import shutil import time from collections import namedtuple -import math -import shutil + import boto3 import botocore +import markov.rollout_constants as const +import rospkg import rospy import tensorflow as tf -import rospkg -from rl_coach.base_parameters import TaskParameters -from rl_coach.core_types import EnvironmentSteps +from deepracer_simulation_environment.srv import ( + VirtualEventVideoEditSrv, + VirtualEventVideoEditSrvRequest, +) +from gazebo_msgs.msg import ModelState from markov import utils from markov.agent_ctrl.constants import ConfigParams -from markov.agents.utils import RunPhaseSubject from markov.agents.rollout_agent_factory import ( - create_rollout_agent, - create_obstacles_agent, create_bot_cars_agent, + create_obstacles_agent, + create_rollout_agent, ) +from markov.agents.utils import RunPhaseSubject from markov.architecture.constants import Input from markov.auth.refreshed_session import refreshed_session -from markov.camera_utils import configure_camera -from markov.constants import SIMAPP_VERSION_2 -from markov.defaults import reward_function -from markov.environments.constants import LINK_NAMES, STEERING_TOPICS, VELOCITY_TOPICS -from markov.utils import get_racecar_idx -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit -from markov.log_handler.deepracer_exceptions import GenericNonFatalException -from markov.log_handler.constants import ( - SIMAPP_EVENT_SYSTEM_ERROR, - SIMAPP_EVENT_USER_ERROR, - SIMAPP_EVENT_ERROR_CODE_400, - SIMAPP_EVENT_ERROR_CODE_500, - SIMAPP_SQS_RECEIVE_MESSAGE_EXCEPTION, - SIMAPP_VIRTUAL_EVENT_RACE_EXCEPTION, -) -from markov.metrics.s3_metrics import EvalMetrics -from markov.metrics.constants import MetricsS3Keys -from markov.rospy_wrappers import ServiceProxyWrapper -from markov.rollout_utils import ( - PhaseObserver, - configure_environment_randomizer, - get_robomaker_profiler_env, - signal_robomaker_markov_package_ready, -) -from markov.sagemaker_graph_manager import get_graph_manager from markov.boto.s3.constants import ( CAMERA_45DEGREE_LOCAL_PATH_FORMAT, CAMERA_PIP_MP4_LOCAL_PATH_FORMAT, CAMERA_TOPVIEW_LOCAL_PATH_FORMAT, MODEL_METADATA_LOCAL_PATH_FORMAT, MODEL_METADATA_S3_POSTFIX, - SIMTRACE_EVAL_LOCAL_PATH_FORMAT, S3_RACE_STATUS_FILE_NAME, - RaceStatusKeys, - SimtraceVideoNames, SECTOR_TIME_LOCAL_PATH, SECTOR_TIME_S3_POSTFIX, SECTOR_X_FORMAT, + SIMTRACE_EVAL_LOCAL_PATH_FORMAT, ModelMetadataKeys, + RaceStatusKeys, + SimtraceVideoNames, ) from markov.boto.s3.files.checkpoint import Checkpoint from markov.boto.s3.files.model_metadata import ModelMetadata @@ -71,31 +50,52 @@ from markov.boto.s3.files.virtual_event_best_sector_time import VirtualEventBestSectorTime from markov.boto.s3.s3_client import S3Client from markov.boto.s3.utils import get_s3_key -from markov.s3_boto_data_store import S3BotoDataStore, S3BotoDataStoreParameters from markov.boto.sqs.sqs_client import SQSClient -from markov.track_geom.track_data import FiniteDifference, TrackData -from markov.track_geom.utils import get_start_positions, get_hide_positions, euler_to_quaternion -from markov.utils import get_boto_config -from markov.gazebo_utils.model_updater import ModelUpdater +from markov.camera_utils import configure_camera from markov.cameras.camera_manager import CameraManager +from markov.constants import SIMAPP_VERSION_2 +from markov.defaults import reward_function +from markov.environments.constants import LINK_NAMES, STEERING_TOPICS, VELOCITY_TOPICS +from markov.gazebo_utils.model_updater import ModelUpdater +from markov.log_handler.constants import ( + SIMAPP_EVENT_ERROR_CODE_400, + SIMAPP_EVENT_ERROR_CODE_500, + SIMAPP_EVENT_SYSTEM_ERROR, + SIMAPP_EVENT_USER_ERROR, + SIMAPP_SQS_RECEIVE_MESSAGE_EXCEPTION, + SIMAPP_VIRTUAL_EVENT_RACE_EXCEPTION, +) +from markov.log_handler.deepracer_exceptions import GenericNonFatalException +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger +from markov.metrics.constants import MetricsS3Keys +from markov.metrics.s3_metrics import EvalMetrics +from markov.rollout_utils import ( + PhaseObserver, + configure_environment_randomizer, + get_robomaker_profiler_env, + signal_robomaker_markov_package_ready, +) +from markov.rospy_wrappers import ServiceProxyWrapper +from markov.s3_boto_data_store import S3BotoDataStore, S3BotoDataStoreParameters +from markov.sagemaker_graph_manager import get_graph_manager +from markov.track_geom.track_data import FiniteDifference, TrackData +from markov.track_geom.utils import euler_to_quaternion, get_hide_positions, get_start_positions +from markov.utils import get_boto_config, get_racecar_idx from markov.virtual_event.constants import ( - VIRTUAL_EVENT, - RACER_INFO_OBJECT, + MAX_NUM_OF_SQS_ERROR, + MAX_NUM_OF_SQS_MESSAGE, + PAUSE_TIME_BEFORE_START, RACER_INFO_JSON_SCHEMA, + RACER_INFO_OBJECT, SENSOR_MODEL_MAP, - MAX_NUM_OF_SQS_MESSAGE, SQS_WAIT_TIME_SEC, - MAX_NUM_OF_SQS_ERROR, - PAUSE_TIME_BEFORE_START, + VIRTUAL_EVENT, ) from markov.virtual_event.utils import validate_json_input -from deepracer_simulation_environment.srv import ( - VirtualEventVideoEditSrv, - VirtualEventVideoEditSrvRequest, -) +from rl_coach.base_parameters import TaskParameters +from rl_coach.core_types import EnvironmentSteps from std_srvs.srv import Empty, EmptyRequest -from gazebo_msgs.msg import ModelState -import markov.rollout_constants as const LOG = Logger(__name__, logging.INFO).get_logger() diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event_worker.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event_worker.py index ae1a8b44d9..e7036baa2d 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event_worker.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/virtual_event_worker.py @@ -2,23 +2,23 @@ import argparse import logging import os + import rospy from markov import utils -from markov.virtual_event.virtual_event_manager import VirtualEventManager -from markov.virtual_event.constants import DEFAULT_RACE_DURATION -from markov.log_handler.logger import Logger -from markov.log_handler.exception_handler import log_and_exit from markov.log_handler.constants import ( SIMAPP_EVENT_ERROR_CODE_500, SIMAPP_SIMULATION_WORKER_EXCEPTION, ) - +from markov.log_handler.exception_handler import log_and_exit +from markov.log_handler.logger import Logger +from markov.virtual_event.constants import DEFAULT_RACE_DURATION +from markov.virtual_event.virtual_event_manager import VirtualEventManager LOG = Logger(__name__, logging.INFO).get_logger() def main(): - """ Main function for virutal event manager """ + """Main function for virutal event manager""" parser = argparse.ArgumentParser() parser.add_argument( "--queue_url", diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/abs_effect.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/abs_effect.py index 3174062bd6..12f86fb352 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/abs_effect.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/abs_effect.py @@ -1,4 +1,5 @@ import abc + from markov.visual_effects.effect_manager import EffectManager # Python 2 and 3 compatible Abstract class diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/effect_manager.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/effect_manager.py index d0e6014b0a..6b75590f25 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/effect_manager.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/effect_manager.py @@ -1,6 +1,7 @@ -from markov.log_handler.deepracer_exceptions import GenericRolloutException import threading + from markov.gazebo_tracker.abs_tracker import AbstractTracker +from markov.log_handler.deepracer_exceptions import GenericRolloutException class EffectManager(AbstractTracker): diff --git a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/effects/blink_effect.py b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/effects/blink_effect.py index 0d5a1269cd..45b3da236c 100644 --- a/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/effects/blink_effect.py +++ b/reinforcement_learning/rl_deepracer_robomaker_coach_gazebo/src/markov/visual_effects/effects/blink_effect.py @@ -1,14 +1,4 @@ -from markov.cameras.utils import lerp -from markov.rospy_wrappers import ServiceProxyWrapper -from markov.domain_randomizations.constants import GazeboServiceName -from markov.visual_effects.abs_effect import AbstractEffect -from markov.track_geom.track_data import TrackData -from markov.gazebo_tracker.trackers.set_visual_transparency_tracker import ( - SetVisualTransparencyTracker, -) import rospy -from gazebo_msgs.srv import GetModelProperties, GetModelPropertiesRequest - from deepracer_msgs.srv import ( GetVisualNames, GetVisualNamesRequest, @@ -16,6 +6,15 @@ GetVisualsRequest, GetVisualsResponse, ) +from gazebo_msgs.srv import GetModelProperties, GetModelPropertiesRequest +from markov.cameras.utils import lerp +from markov.domain_randomizations.constants import GazeboServiceName +from markov.gazebo_tracker.trackers.set_visual_transparency_tracker import ( + SetVisualTransparencyTracker, +) +from markov.rospy_wrappers import ServiceProxyWrapper +from markov.track_geom.track_data import TrackData +from markov.visual_effects.abs_effect import AbstractEffect class BlinkEffect(AbstractEffect): diff --git a/reinforcement_learning/rl_game_server_autopilot/ap-server/app.py b/reinforcement_learning/rl_game_server_autopilot/ap-server/app.py index c369d75eb0..11dc08b568 100644 --- a/reinforcement_learning/rl_game_server_autopilot/ap-server/app.py +++ b/reinforcement_learning/rl_game_server_autopilot/ap-server/app.py @@ -1,14 +1,15 @@ -from chalice import Chalice import json -import numpy as np -from time import gmtime, strftime +import os +import sys import time from datetime import datetime -from urllib.parse import urlparse, parse_qs +from time import gmtime, strftime +from urllib.parse import parse_qs, urlparse + import boto3 -import sys -import os +import numpy as np import requests +from chalice import Chalice # Global Varibale from model content_type = "application/json" diff --git a/reinforcement_learning/rl_game_server_autopilot/sagemaker/src/gameserver_env.py b/reinforcement_learning/rl_game_server_autopilot/sagemaker/src/gameserver_env.py index bc0964ccec..006bf6e967 100644 --- a/reinforcement_learning/rl_game_server_autopilot/sagemaker/src/gameserver_env.py +++ b/reinforcement_learning/rl_game_server_autopilot/sagemaker/src/gameserver_env.py @@ -1,10 +1,11 @@ import time +from time import gmtime, strftime + import boto3 -import requests import gym import numpy as np -from time import gmtime, strftime -from gym.spaces import Discrete, Box +import requests +from gym.spaces import Box, Discrete cloudwatch_cli = boto3.client("cloudwatch", region_name="us-west-2") diff --git a/reinforcement_learning/rl_game_server_autopilot/sagemaker/src/train_gameserver_ppo.py b/reinforcement_learning/rl_game_server_autopilot/sagemaker/src/train_gameserver_ppo.py index 0acb244382..0b4ad1c464 100644 --- a/reinforcement_learning/rl_game_server_autopilot/sagemaker/src/train_gameserver_ppo.py +++ b/reinforcement_learning/rl_game_server_autopilot/sagemaker/src/train_gameserver_ppo.py @@ -1,11 +1,11 @@ import json import os import sys + import gym import ray from ray.tune import run_experiments from ray.tune.registry import register_env - from sagemaker_rl.ray_launcher import SageMakerRayLauncher env_config = {} diff --git a/reinforcement_learning/rl_hvac_coach_energyplus/src/deploy-mxnet-coach.py b/reinforcement_learning/rl_hvac_coach_energyplus/src/deploy-mxnet-coach.py index 7f0439d92b..5f93328ca8 100644 --- a/reinforcement_learning/rl_hvac_coach_energyplus/src/deploy-mxnet-coach.py +++ b/reinforcement_learning/rl_hvac_coach_energyplus/src/deploy-mxnet-coach.py @@ -1,9 +1,10 @@ -import os import json +import os + import mxnet as mx -from mxnet.contrib import onnx as onnx_mxnet -from mxnet import gluon, nd import numpy as np +from mxnet import gluon, nd +from mxnet.contrib import onnx as onnx_mxnet def model_fn(model_dir): diff --git a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/__init__.py b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/__init__.py index fee47527af..101efed1d8 100644 --- a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/__init__.py +++ b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/__init__.py @@ -1,6 +1,4 @@ -from eplus.envs.large_office_env import LargeOfficeEnv from eplus.envs.data_center_env import DataCenterEnv - -from eplus.envs.pyEp import ep_process -from eplus.envs.pyEp import set_eplus_dir +from eplus.envs.large_office_env import LargeOfficeEnv +from eplus.envs.pyEp import ep_process, set_eplus_dir from eplus.envs.socket_builder import socket_builder diff --git a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/data_center_env.py b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/data_center_env.py index f2ed254707..4f73e5c7b9 100644 --- a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/data_center_env.py +++ b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/data_center_env.py @@ -1,11 +1,12 @@ +import os +import socket + import gym -from gym import error, spaces, utils -from gym.utils import seeding +import numpy as np from eplus.envs import pyEp -import socket from eplus.envs.socket_builder import socket_builder -import numpy as np -import os +from gym import error, spaces, utils +from gym.utils import seeding class DataCenterEnv(gym.Env): diff --git a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/large_office_env.py b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/large_office_env.py index 4332b66346..4bb32ac45a 100644 --- a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/large_office_env.py +++ b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/large_office_env.py @@ -1,11 +1,12 @@ +import os +import socket + import gym -from gym import error, spaces, utils -from gym.utils import seeding +import numpy as np from eplus.envs import pyEp -import socket from eplus.envs.socket_builder import socket_builder -import numpy as np -import os +from gym import error, spaces, utils +from gym.utils import seeding class LargeOfficeEnv(gym.Env): diff --git a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/pyEp.py b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/pyEp.py index bde811066a..691a6d6e04 100644 --- a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/pyEp.py +++ b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/pyEp.py @@ -1,7 +1,8 @@ +import os import socket -import sys import subprocess -import os +import sys + from eplus.envs import pyEpError diff --git a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/pyEpError.py b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/pyEpError.py index 4477eb150f..536a81648d 100644 --- a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/pyEpError.py +++ b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/pyEpError.py @@ -1,12 +1,12 @@ class pyEpError(Exception): - """ Base class for pyEp Errors """ + """Base class for pyEp Errors""" def __init__(self, message): super(pyEpError, self).__init__(message) class VersionError(pyEpError): - """ Error Thrown when E+ Communications protocol is not 2. """ + """Error Thrown when E+ Communications protocol is not 2.""" def __init__(self, message=None): if message is None: @@ -16,7 +16,7 @@ def __init__(self, message=None): class EpWriteError(pyEpError): - """ Error thrown when appempting to write to a closed E+ instance """ + """Error thrown when appempting to write to a closed E+ instance""" def __init__(self, message=None): if message is None: @@ -26,7 +26,7 @@ def __init__(self, message=None): class EpReadError(pyEpError): - """ Error thrown when appempting to read from a closed E+ instance """ + """Error thrown when appempting to read from a closed E+ instance""" def __init__(self, message=None): if message is None: @@ -36,7 +36,7 @@ def __init__(self, message=None): class MissingEpPathError(pyEpError): - """ Error thrown when the path to EnergyPlus is not specified. """ + """Error thrown when the path to EnergyPlus is not specified.""" def __init__(self, message=None): if message is None: diff --git a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/socket_builder.py b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/socket_builder.py index 5b4f5de110..040ee5112e 100644 --- a/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/socket_builder.py +++ b/reinforcement_learning/rl_hvac_coach_energyplus/src/eplus/envs/socket_builder.py @@ -1,6 +1,6 @@ +import os import socket import xml.etree.ElementTree as ET -import os class socket_builder: diff --git a/reinforcement_learning/rl_hvac_coach_energyplus/src/evaluate-baseline.py b/reinforcement_learning/rl_hvac_coach_energyplus/src/evaluate-baseline.py index 519560d057..ec18e33cc5 100644 --- a/reinforcement_learning/rl_hvac_coach_energyplus/src/evaluate-baseline.py +++ b/reinforcement_learning/rl_hvac_coach_energyplus/src/evaluate-baseline.py @@ -1,6 +1,6 @@ -from eplus.envs import DataCenterEnv import platform +from eplus.envs import DataCenterEnv os_type = platform.system() if os_type == "Linux": diff --git a/reinforcement_learning/rl_hvac_coach_energyplus/src/evaluate-coach.py b/reinforcement_learning/rl_hvac_coach_energyplus/src/evaluate-coach.py index 3bf2b48615..0995f9dd59 100644 --- a/reinforcement_learning/rl_hvac_coach_energyplus/src/evaluate-coach.py +++ b/reinforcement_learning/rl_hvac_coach_energyplus/src/evaluate-coach.py @@ -1,9 +1,10 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher, CoachConfigurationList import argparse import os + import rl_coach from rl_coach.base_parameters import Frameworks, TaskParameters from rl_coach.core_types import EnvironmentSteps +from sagemaker_rl.coach_launcher import CoachConfigurationList, SageMakerCoachPresetLauncher def inplace_replace_in_file(filepath, old, new): diff --git a/reinforcement_learning/rl_hvac_coach_energyplus/src/preset-energy-plus-clipped-ppo.py b/reinforcement_learning/rl_hvac_coach_energyplus/src/preset-energy-plus-clipped-ppo.py index 0c6988fb1f..2fcf66e3d4 100644 --- a/reinforcement_learning/rl_hvac_coach_energyplus/src/preset-energy-plus-clipped-ppo.py +++ b/reinforcement_learning/rl_hvac_coach_energyplus/src/preset-energy-plus-clipped-ppo.py @@ -1,11 +1,12 @@ +import eplus from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters from rl_coach.architectures.layers import Dense from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, DistributedCoachSynchronizationType, + PresetValidationParameters, + VisualizationParameters, ) -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps, RunPhase +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, RunPhase, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment, mujoco_v2 from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters from rl_coach.exploration_policies.e_greedy import EGreedyParameters @@ -15,7 +16,6 @@ from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager from rl_coach.graph_managers.graph_manager import ScheduleParameters from rl_coach.schedules import LinearSchedule -import eplus #################### # Graph Scheduling # diff --git a/reinforcement_learning/rl_hvac_ray_energyplus/source/custom/callbacks.py b/reinforcement_learning/rl_hvac_ray_energyplus/source/custom/callbacks.py index 55b83aca82..7f9b7dd7f9 100644 --- a/reinforcement_learning/rl_hvac_ray_energyplus/source/custom/callbacks.py +++ b/reinforcement_learning/rl_hvac_ray_energyplus/source/custom/callbacks.py @@ -1,13 +1,12 @@ from typing import Dict +import numpy as np import ray +from ray.rllib.agents.callbacks import DefaultCallbacks from ray.rllib.env import BaseEnv +from ray.rllib.evaluation import MultiAgentEpisode, RolloutWorker from ray.rllib.policy import Policy from ray.rllib.policy.sample_batch import SampleBatch -from ray.rllib.evaluation import MultiAgentEpisode, RolloutWorker -from ray.rllib.agents.callbacks import DefaultCallbacks - -import numpy as np class CustomCallbacks(DefaultCallbacks): diff --git a/reinforcement_learning/rl_hvac_ray_energyplus/source/eplus/envs/medium_office_env.py b/reinforcement_learning/rl_hvac_ray_energyplus/source/eplus/envs/medium_office_env.py index e3f6609b3c..5e844fd95f 100644 --- a/reinforcement_learning/rl_hvac_ray_energyplus/source/eplus/envs/medium_office_env.py +++ b/reinforcement_learning/rl_hvac_ray_energyplus/source/eplus/envs/medium_office_env.py @@ -1,18 +1,16 @@ -import gym -from gym import error, spaces, utils -from gym.utils import seeding - -from pyenergyplus.api import EnergyPlusAPI - -import numpy as np -import os import itertools -import uuid -import time +import os import queue +import time +import uuid from queue import Queue +from threading import Event, Thread -from threading import Thread, Event +import gym +import numpy as np +from gym import error, spaces, utils +from gym.utils import seeding +from pyenergyplus.api import EnergyPlusAPI class MediumOfficeEnv(gym.Env): diff --git a/reinforcement_learning/rl_hvac_ray_energyplus/source/hvac_ray_launcher.py b/reinforcement_learning/rl_hvac_ray_energyplus/source/hvac_ray_launcher.py index 01a94ec321..bb87aafa4f 100644 --- a/reinforcement_learning/rl_hvac_ray_energyplus/source/hvac_ray_launcher.py +++ b/reinforcement_learning/rl_hvac_ray_energyplus/source/hvac_ray_launcher.py @@ -11,18 +11,15 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. import ast -import os import json +import os import subprocess import ray - -from ray.tune.tune import run_experiments, run, _make_scheduler - +from ray.tune.tune import _make_scheduler, run, run_experiments from sagemaker_rl.ray_launcher import SageMakerRayLauncher from sagemaker_rl.tf_serving_utils import export_tf_serving, natural_keys - TERMINATION_SIGNAL = "JOB_TERMINATED" MODEL_OUTPUT_DIR = "/opt/ml/model" CHECKPOINTS_DIR = "/opt/ml/checkpoints" diff --git a/reinforcement_learning/rl_hvac_ray_energyplus/source/ray_experiment_builder.py b/reinforcement_learning/rl_hvac_ray_energyplus/source/ray_experiment_builder.py index 1ae9da1f98..60adcc53ac 100644 --- a/reinforcement_learning/rl_hvac_ray_energyplus/source/ray_experiment_builder.py +++ b/reinforcement_learning/rl_hvac_ray_energyplus/source/ray_experiment_builder.py @@ -11,14 +11,14 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. -import os -from pathlib import Path import argparse import json -import yaml +import os +from pathlib import Path +import yaml +from ray.tune.resources import json_to_resources, resources_to_json from ray.tune.result import DEFAULT_RESULTS_DIR -from ray.tune.resources import resources_to_json, json_to_resources try: from custom.callbacks import CustomCallbacks diff --git a/reinforcement_learning/rl_hvac_ray_energyplus/source/rollout.py b/reinforcement_learning/rl_hvac_ray_energyplus/source/rollout.py index 344070e21e..0e2f24713b 100644 --- a/reinforcement_learning/rl_hvac_ray_energyplus/source/rollout.py +++ b/reinforcement_learning/rl_hvac_ray_energyplus/source/rollout.py @@ -5,9 +5,9 @@ import copy import json import os -from pathlib import Path import pickle import shelve +from pathlib import Path import gym import gym.wrappers @@ -24,10 +24,10 @@ from ray.rllib.evaluation.worker_set import WorkerSet from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID from ray.rllib.utils.deprecation import deprecation_warning -from ray.tune.utils import merge_dicts from ray.tune.registry import get_trainable_cls +from ray.tune.utils import merge_dicts -from utils.loader import load_envs, load_models, load_algorithms, load_preprocessors +from utils.loader import load_algorithms, load_envs, load_models, load_preprocessors """ Note : This script has been adapted from : diff --git a/reinforcement_learning/rl_hvac_ray_energyplus/source/train-sagemaker-hvac.py b/reinforcement_learning/rl_hvac_ray_energyplus/source/train-sagemaker-hvac.py index 0097bfd8db..46e3208ec5 100644 --- a/reinforcement_learning/rl_hvac_ray_energyplus/source/train-sagemaker-hvac.py +++ b/reinforcement_learning/rl_hvac_ray_energyplus/source/train-sagemaker-hvac.py @@ -11,17 +11,16 @@ # express or implied. See the License for the specific language governing # permissions and limitations under the License. -import os +import argparse import json import multiprocessing +import os import subprocess -import argparse - -from ray.tune.registry import register_env +from eplus.envs.medium_office_env import MediumOfficeEnv from hvac_ray_launcher import HVACSageMakerRayLauncher +from ray.tune.registry import register_env from ray_experiment_builder import RayExperimentBuilder -from eplus.envs.medium_office_env import MediumOfficeEnv class MyLauncher(HVACSageMakerRayLauncher): diff --git a/reinforcement_learning/rl_knapsack_coach_custom/src/evaluate-coach.py b/reinforcement_learning/rl_knapsack_coach_custom/src/evaluate-coach.py index 7a7e55e285..e76f18b7ec 100644 --- a/reinforcement_learning/rl_knapsack_coach_custom/src/evaluate-coach.py +++ b/reinforcement_learning/rl_knapsack_coach_custom/src/evaluate-coach.py @@ -1,9 +1,10 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher, CoachConfigurationList import argparse import os + import rl_coach from rl_coach.base_parameters import Frameworks, TaskParameters from rl_coach.core_types import EnvironmentSteps +from sagemaker_rl.coach_launcher import CoachConfigurationList, SageMakerCoachPresetLauncher def inplace_replace_in_file(filepath, old, new): diff --git a/reinforcement_learning/rl_knapsack_coach_custom/src/knapsack_baseline.py b/reinforcement_learning/rl_knapsack_coach_custom/src/knapsack_baseline.py index bad4c3c49b..bc4d39d25f 100644 --- a/reinforcement_learning/rl_knapsack_coach_custom/src/knapsack_baseline.py +++ b/reinforcement_learning/rl_knapsack_coach_custom/src/knapsack_baseline.py @@ -1,6 +1,7 @@ -import numpy as np -import random import queue +import random + +import numpy as np def get_knapsack_solution_simple(weights, values, c_max, penalty, debug=False): diff --git a/reinforcement_learning/rl_knapsack_coach_custom/src/knapsack_env.py b/reinforcement_learning/rl_knapsack_coach_custom/src/knapsack_env.py index 1e08760470..3c3ffbe64f 100644 --- a/reinforcement_learning/rl_knapsack_coach_custom/src/knapsack_env.py +++ b/reinforcement_learning/rl_knapsack_coach_custom/src/knapsack_env.py @@ -1,14 +1,16 @@ -import gym -from gym import spaces -import numpy as np -import queue import csv +import queue +import gym +import numpy as np from capacity import Capacity +from gym import spaces from item import Item -from knapsack_baseline import get_knapsack_solution_simple -from knapsack_baseline import get_knapsack_solution_medium -from knapsack_baseline import get_knapsack_benchmark_sol_hard_greedy_heuristic +from knapsack_baseline import ( + get_knapsack_benchmark_sol_hard_greedy_heuristic, + get_knapsack_solution_medium, + get_knapsack_solution_simple, +) from knapsack_view_2D import KnapsackView2D diff --git a/reinforcement_learning/rl_knapsack_coach_custom/src/preset-knapsack-clippedppo.py b/reinforcement_learning/rl_knapsack_coach_custom/src/preset-knapsack-clippedppo.py index 3203628afc..44ce60f6f9 100644 --- a/reinforcement_learning/rl_knapsack_coach_custom/src/preset-knapsack-clippedppo.py +++ b/reinforcement_learning/rl_knapsack_coach_custom/src/preset-knapsack-clippedppo.py @@ -1,11 +1,11 @@ from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters from rl_coach.architectures.layers import Dense from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, DistributedCoachSynchronizationType, + PresetValidationParameters, + VisualizationParameters, ) -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps, RunPhase +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, RunPhase, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment, mujoco_v2 from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters from rl_coach.exploration_policies.e_greedy import EGreedyParameters diff --git a/reinforcement_learning/rl_knapsack_coach_custom/src/train-coach.py b/reinforcement_learning/rl_knapsack_coach_custom/src/train-coach.py index 631d77d4c4..14c15bbeab 100644 --- a/reinforcement_learning/rl_knapsack_coach_custom/src/train-coach.py +++ b/reinforcement_learning/rl_knapsack_coach_custom/src/train-coach.py @@ -1,6 +1,5 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher - import knapsack_env +from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher class MyLauncher(SageMakerCoachPresetLauncher): diff --git a/reinforcement_learning/rl_managed_spot_cartpole_coach/src/deploy-mxnet-coach.py b/reinforcement_learning/rl_managed_spot_cartpole_coach/src/deploy-mxnet-coach.py index 7f0439d92b..5f93328ca8 100644 --- a/reinforcement_learning/rl_managed_spot_cartpole_coach/src/deploy-mxnet-coach.py +++ b/reinforcement_learning/rl_managed_spot_cartpole_coach/src/deploy-mxnet-coach.py @@ -1,9 +1,10 @@ -import os import json +import os + import mxnet as mx -from mxnet.contrib import onnx as onnx_mxnet -from mxnet import gluon, nd import numpy as np +from mxnet import gluon, nd +from mxnet.contrib import onnx as onnx_mxnet def model_fn(model_dir): diff --git a/reinforcement_learning/rl_managed_spot_cartpole_coach/src/evaluate-coach.py b/reinforcement_learning/rl_managed_spot_cartpole_coach/src/evaluate-coach.py index 3bf2b48615..0995f9dd59 100644 --- a/reinforcement_learning/rl_managed_spot_cartpole_coach/src/evaluate-coach.py +++ b/reinforcement_learning/rl_managed_spot_cartpole_coach/src/evaluate-coach.py @@ -1,9 +1,10 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher, CoachConfigurationList import argparse import os + import rl_coach from rl_coach.base_parameters import Frameworks, TaskParameters from rl_coach.core_types import EnvironmentSteps +from sagemaker_rl.coach_launcher import CoachConfigurationList, SageMakerCoachPresetLauncher def inplace_replace_in_file(filepath, old, new): diff --git a/reinforcement_learning/rl_managed_spot_cartpole_coach/src/preset-cartpole-clippedppo.py b/reinforcement_learning/rl_managed_spot_cartpole_coach/src/preset-cartpole-clippedppo.py index b80e7fbdb2..bfa308765c 100644 --- a/reinforcement_learning/rl_managed_spot_cartpole_coach/src/preset-cartpole-clippedppo.py +++ b/reinforcement_learning/rl_managed_spot_cartpole_coach/src/preset-cartpole-clippedppo.py @@ -1,11 +1,11 @@ from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters from rl_coach.architectures.layers import Dense from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, DistributedCoachSynchronizationType, + PresetValidationParameters, + VisualizationParameters, ) -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps, RunPhase +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, RunPhase, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment, mujoco_v2 from rl_coach.exploration_policies.additive_noise import AdditiveNoiseParameters from rl_coach.exploration_policies.e_greedy import EGreedyParameters diff --git a/reinforcement_learning/rl_managed_spot_cartpole_coach/src/preset-cartpole-dqn.py b/reinforcement_learning/rl_managed_spot_cartpole_coach/src/preset-cartpole-dqn.py index 1f0dcc92fe..4a92448e38 100644 --- a/reinforcement_learning/rl_managed_spot_cartpole_coach/src/preset-cartpole-dqn.py +++ b/reinforcement_learning/rl_managed_spot_cartpole_coach/src/preset-cartpole-dqn.py @@ -1,10 +1,10 @@ from rl_coach.agents.dqn_agent import DQNAgentParameters from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, DistributedCoachSynchronizationType, + PresetValidationParameters, + VisualizationParameters, ) -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager from rl_coach.graph_managers.graph_manager import ScheduleParameters diff --git a/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/deploy-mxnet-coach.py b/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/deploy-mxnet-coach.py index 7f0439d92b..5f93328ca8 100644 --- a/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/deploy-mxnet-coach.py +++ b/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/deploy-mxnet-coach.py @@ -1,9 +1,10 @@ -import os import json +import os + import mxnet as mx -from mxnet.contrib import onnx as onnx_mxnet -from mxnet import gluon, nd import numpy as np +from mxnet import gluon, nd +from mxnet.contrib import onnx as onnx_mxnet def model_fn(model_dir): diff --git a/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/evaluate-coach.py b/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/evaluate-coach.py index 3bf2b48615..0995f9dd59 100644 --- a/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/evaluate-coach.py +++ b/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/evaluate-coach.py @@ -1,9 +1,10 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher, CoachConfigurationList import argparse import os + import rl_coach from rl_coach.base_parameters import Frameworks, TaskParameters from rl_coach.core_types import EnvironmentSteps +from sagemaker_rl.coach_launcher import CoachConfigurationList, SageMakerCoachPresetLauncher def inplace_replace_in_file(filepath, old, new): diff --git a/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/patient_envs.py b/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/patient_envs.py index ab9a77213f..3a4eca3af5 100644 --- a/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/patient_envs.py +++ b/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/patient_envs.py @@ -1,5 +1,5 @@ -from gym.envs.classic_control.mountain_car import MountainCarEnv from gym.envs.classic_control.continuous_mountain_car import Continuous_MountainCarEnv +from gym.envs.classic_control.mountain_car import MountainCarEnv from gym.wrappers.time_limit import TimeLimit diff --git a/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/preset-mountain-car-continuous-clipped-ppo.py b/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/preset-mountain-car-continuous-clipped-ppo.py index c13a71a0ef..563e4b61ed 100644 --- a/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/preset-mountain-car-continuous-clipped-ppo.py +++ b/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/preset-mountain-car-continuous-clipped-ppo.py @@ -1,13 +1,13 @@ from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters from rl_coach.architectures.layers import Dense -from rl_coach.base_parameters import VisualizationParameters, PresetValidationParameters +from rl_coach.base_parameters import PresetValidationParameters, VisualizationParameters from rl_coach.core_types import ( - TrainingSteps, EnvironmentEpisodes, EnvironmentSteps, + MaxDumpFilter, RunPhase, SelectedPhaseOnlyDumpFilter, - MaxDumpFilter, + TrainingSteps, ) from rl_coach.environments.gym_environment import GymVectorEnvironment from rl_coach.exploration_policies.e_greedy import EGreedyParameters diff --git a/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/preset-mountain-car-dqn.py b/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/preset-mountain-car-dqn.py index 2f7072a096..33f5335533 100644 --- a/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/preset-mountain-car-dqn.py +++ b/reinforcement_learning/rl_mountain_car_coach_gymEnv/src/preset-mountain-car-dqn.py @@ -1,17 +1,17 @@ from rl_coach.agents.dqn_agent import DQNAgentParameters from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, EmbedderScheme, MiddlewareScheme, + PresetValidationParameters, + VisualizationParameters, ) from rl_coach.core_types import ( - TrainingSteps, EnvironmentEpisodes, EnvironmentSteps, + MaxDumpFilter, RunPhase, SelectedPhaseOnlyDumpFilter, - MaxDumpFilter, + TrainingSteps, ) from rl_coach.environments.gym_environment import GymVectorEnvironment from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/environment.py b/reinforcement_learning/rl_network_compression_ray_custom/src/environment.py index 38358d95fc..a7a2621170 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/environment.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/environment.py @@ -1,16 +1,12 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + +from tempfile import mkstemp import gym import numpy as np -from gym.spaces import Box -from gym.spaces import MultiBinary - +from gym.spaces import Box, MultiBinary from tensorflow_resnet import NetworkCompression -from tempfile import mkstemp - class Compression(gym.Env): """Example of a network compression env which simulates layer removal and shrinkage. diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/__init__.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/__init__.py index 344c887235..4aae2caee1 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/__init__.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/__init__.py @@ -1,6 +1,5 @@ -from .core import Module -from .core import ModeKeys -from .train import tensorflow_train +from .core import ModeKeys, Module from .resnet import ResNet18Model +from .train import tensorflow_train __all__ = ["Module", "ModeKeys", "ResNet", "tensorflow_train"] diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/core/__init__.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/core/__init__.py index a8ae4fcb4c..b1ea4cf0ce 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/core/__init__.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/core/__init__.py @@ -1,5 +1,5 @@ -from .module import Module, Layer -from .modekeys import ModeKeys from .fake_ops import Fake +from .modekeys import ModeKeys +from .module import Layer, Module __all__ = ["Module", "Layer", "ModeKeys", "Fake"] diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/core/module.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/core/module.py index c629c33984..c6b40044f8 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/core/module.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/core/module.py @@ -1,5 +1,5 @@ -from collections import OrderedDict import logging +from collections import OrderedDict class Module(object): @@ -18,7 +18,7 @@ def __init__(self, name): self.description = [] def get_params(self): - """ Returns the parameters of the module, including all the sub parameters.""" + """Returns the parameters of the module, including all the sub parameters.""" if not self._leaf: params = {} for module in self._modules.values(): @@ -43,15 +43,15 @@ def add_module(self, module, name): self.output = module.output def get_name(self): - """ Returns the modeule name """ + """Returns the modeule name""" return self._name def get_desc(self): - """ Retruns the module description """ + """Retruns the module description""" return self.description def pretty_print(self, prefix=""): - """ Prints module detail in context with all sub modules """ + """Prints module detail in context with all sub modules""" if self._leaf: logging.info(prefix + "------------------------------") logging.info(prefix + "Module Name: " + self.get_name()) @@ -65,7 +65,7 @@ def pretty_print(self, prefix=""): self._modules[module].pretty_print(prefix + " ") def get_sub_module(self, mod_name): - """ Returns the sub module of some name """ + """Returns the sub module of some name""" if not isinstance(mod_name, str): mod_name = str(mod_name) if mod_name in self._modules.keys(): diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/__init__.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/__init__.py index 3789a62235..14789ee8be 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/__init__.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/__init__.py @@ -1,9 +1,9 @@ +from .activation import ReLU from .batch_norm import BatchNorm from .conv import Conv2DFixedPadding as Conv from .dense import Dense +from .descriptions import LayerState from .pooling import Pool -from .activation import ReLU from .renset_blocks import BuildingBlock -from .descriptions import LayerState __all__ = ["BatchNorm", "Conv", "Dense", "Pool", "ReLU", "BuildingBlock", "LayerDescriptions"] diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/activation.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/activation.py index 980b09bcfe..fc4a8920ad 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/activation.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/activation.py @@ -1,5 +1,6 @@ import tensorflow as tf -from ..core import Layer, Fake + +from ..core import Fake, Layer class ReLU(Layer): @@ -25,7 +26,7 @@ def __init__(self, name, inputs, start=None, end=None, fake=False): self.description.append(self.get_memory_footprint()) def get_params(self): - """ Activation layers do not have parameters to return. """ + """Activation layers do not have parameters to return.""" return [] def get_memory_footprint(self): diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/batch_norm.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/batch_norm.py index f031988197..edec212c81 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/batch_norm.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/batch_norm.py @@ -1,8 +1,8 @@ # This file puts a wrapper on top of all tf batch norm layer. -import tensorflow as tf import numpy as np +import tensorflow as tf -from ..core import Layer, Fake +from ..core import Fake, Layer from .ops import get_param_from_name, load_pkl_obj _BATCH_NORM_DECAY = 0.997 @@ -85,7 +85,7 @@ def __init__( self.description.append(self.get_memory_footprint()) def _get_params_real(self): - """ Returns a list of [gamma, beta] """ + """Returns a list of [gamma, beta]""" gamma = get_param_from_name(self._tf_name + "/batch_normalization/gamma:0") beta = get_param_from_name(self._tf_name + "/batch_normalization/beta:0") moving_mean = get_param_from_name(self._tf_name + "/batch_normalization/moving_mean:0") @@ -95,7 +95,7 @@ def _get_params_real(self): return {self.get_name(): [gamma, beta, moving_mean, moving_variance]} def _get_memory_footprint_real(self): - """ Returns the number of parameters in the layer """ + """Returns the number of parameters in the layer""" params = self.get_params()[self.get_name()] gamma = params[0] beta = params[1] diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/conv.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/conv.py index 6685b74499..2e46d64813 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/conv.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/conv.py @@ -1,8 +1,9 @@ -import tensorflow as tf -import numpy as np import logging -from ..core import Layer, Fake +import numpy as np +import tensorflow as tf + +from ..core import Fake, Layer from .ops import get_param_from_name, load_pkl_obj @@ -113,11 +114,11 @@ def __init__( self.description.append(self.get_memory_footprint()) def _get_params_real(self): - """ Returns the kernel node """ + """Returns the kernel node""" return {self.get_name(): get_param_from_name(self._tf_name + "/conv2d/kernel:0")} def _get_memory_footprint_real(self): - """ Number of parameters in the layer """ + """Number of parameters in the layer""" params = self.get_params()[self.get_name()] return int(np.prod(params.shape)) diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/dense.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/dense.py index 36ca7a5514..f0c72d0e34 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/dense.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/dense.py @@ -1,7 +1,7 @@ -import tensorflow as tf import numpy as np +import tensorflow as tf -from ..core import Layer, Fake +from ..core import Fake, Layer from .ops import get_param_from_name, load_pkl_obj @@ -56,13 +56,13 @@ def __init__( self.description.append(self.get_memory_footprint()) def _get_params_real(self): - """ Returns the kernel node """ + """Returns the kernel node""" kernel = get_param_from_name(self._tf_name + "/dense/kernel:0") bias = get_param_from_name(self._tf_name + "/dense/bias:0") return {self.get_name(): [kernel, bias]} def _get_memory_footprint_real(self): - """ Returns the number of paramters """ + """Returns the number of paramters""" params = self.get_params()[self.get_name()] return int(np.prod(params[0].shape)) + int(np.prod(params[1].shape)) diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/ops.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/ops.py index 02efc70b9d..8c80cf81b7 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/ops.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/ops.py @@ -1,10 +1,11 @@ -import tensorflow as tf -from collections import OrderedDict import pickle +from collections import OrderedDict + +import tensorflow as tf def get_tf_vars_dict(scope=None): - """ Returns all trainable variables in the session in a dictionary form """ + """Returns all trainable variables in the session in a dictionary form""" all_trainable_vars = get_tf_vars_list(scope) vars_dict = OrderedDict() for var in all_trainable_vars: @@ -13,12 +14,12 @@ def get_tf_vars_dict(scope=None): def get_param_from_name(name, scope=None): - """ Returns a particular parameter as a tf element given its name """ + """Returns a particular parameter as a tf element given its name""" return get_global_vars_dict(scope)[name] def load_meta_model_as_np(infile, import_scope="imported"): - """ This will load the meta file into numpy arrays.""" + """This will load the meta file into numpy arrays.""" with tf.Session() as sess: restorer = tf.train.import_meta_graph(infile + ".meta", import_scope=import_scope) restorer.restore(sess, infile) @@ -31,23 +32,23 @@ def load_meta_model_as_np(infile, import_scope="imported"): def load_pkl_obj(name): - """ Loads a pickle model weights for when weights are supplied as initializers to layers """ + """Loads a pickle model weights for when weights are supplied as initializers to layers""" with open(name + ".pkl", "rb") as f: return pickle.load(f) def get_tf_vars_list(scope=None): - """ Returns all the trainable varialbes in the scope as a trainable dictionary. """ + """Returns all the trainable varialbes in the scope as a trainable dictionary.""" return tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope) def get_global_vars_list(scope=None): - """ Returns all the varialbes in the scope as a trainable dictionary. """ + """Returns all the varialbes in the scope as a trainable dictionary.""" return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope) def get_global_vars_dict(scope=None): - """ Returns all variables in the session in a dictionary form """ + """Returns all variables in the session in a dictionary form""" all_vars = get_global_vars_list(scope) vars_dict = OrderedDict() for var in all_vars: diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/pooling.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/pooling.py index fe12d0996f..678f0c1abc 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/pooling.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/pooling.py @@ -1,5 +1,6 @@ import tensorflow as tf -from ..core import Layer, Fake + +from ..core import Fake, Layer class Pool(Layer): @@ -50,9 +51,9 @@ def __init__( self.description.append(self.get_memory_footprint()) def get_params(self): - """ Pooling layers do not have parameters to return. """ + """Pooling layers do not have parameters to return.""" return [] def get_memory_footprint(self): - """ Pooling layer has no memory """ + """Pooling layer has no memory""" return 0 diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/renset_blocks.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/renset_blocks.py index 0817b98e21..821994ce77 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/renset_blocks.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/layers/renset_blocks.py @@ -1,9 +1,10 @@ -from . import Conv, BatchNorm, ReLU -from ..core import Module - import logging + import numpy as np +from ..core import Module +from . import BatchNorm, Conv, ReLU + class BuildingBlock(Module): """A single block for ResNet v2, without a bottleneck. diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/resnet.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/resnet.py index d519d8befe..31625377a5 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/resnet.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/resnet.py @@ -1,14 +1,14 @@ -import sys, os -import tensorflow as tf -import numpy as np import logging +import os +import sys -from . import Module -from . import layers -from . import ModeKeys +import numpy as np +import tensorflow as tf + +from . import ModeKeys, Module, layers +from .core import Fake from .core.model import Model from .layers.ops import get_tf_vars_list -from .core import Fake class ResNetXXModel(Model): @@ -359,7 +359,7 @@ def _dense_grad_filter(gvs): @staticmethod def create_distillation_loss(logit_1, logit_2, temperature): - """ Creates a distilaltion loss """ + """Creates a distilaltion loss""" knowledge_1 = ResNetXXModel.temperature_softmax(logit_1, temperature) knowledge_2 = ResNetXXModel.temperature_softmax(logit_2, temperature) return ResNetXXModel.L2(knowledge_1, knowledge_2) @@ -378,7 +378,7 @@ def temperature_softmax(input, temperature=1): @staticmethod def L2(node1, node2): - """ Returns a symbol thats the 2-norm error """ + """Returns a symbol thats the 2-norm error""" return tf.reduce_mean(tf.squared_difference(node1, node2)) @staticmethod @@ -556,7 +556,7 @@ def __init__( self.name = name def pretty_print(self): - """ Prints important characters of the class """ + """Prints important characters of the class""" logging.info("Num Classes: " + str(self.num_classes)) logging.info("Num Filters: " + str(self.num_filters)) logging.info("Kernel Size: " + str(self.kernel_size)) @@ -569,7 +569,7 @@ def pretty_print(self): logging.info("Resnet Size: " + str(self.resnet_size)) def _model_variable_scope(self): - """ Returns the scope of the model """ + """Returns the scope of the model""" return tf.variable_scope(self.name) def initialize(params): diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/tf_interface.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/tf_interface.py index a7226fc687..7aa15ca496 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/tf_interface.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/tf_interface.py @@ -1,9 +1,10 @@ import tensorflow as tf + from . import tensorflow_train class TensorflowInterface: - """ This is a base class that contains all the hidden tensorflow methods """ + """This is a base class that contains all the hidden tensorflow methods""" ### Static methods that interface with tensorflow.### @staticmethod diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/train.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/train.py index afa30a8795..9ebcb89749 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/train.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/compressor/train.py @@ -1,6 +1,7 @@ -import tensorflow as tf -import math import logging +import math + +import tensorflow as tf def tensorflow_train( diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/dataset/cifar10.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/dataset/cifar10.py index 482cc83344..5b7d0e9232 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/dataset/cifar10.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/dataset/cifar10.py @@ -1,4 +1,5 @@ import os + import tensorflow as tf _HEIGHT = 32 diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/dataset/cifar10_download_and_extract.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/dataset/cifar10_download_and_extract.py index adec207bbc..d36782d4f0 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/dataset/cifar10_download_and_extract.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/dataset/cifar10_download_and_extract.py @@ -15,18 +15,16 @@ """Downloads and extracts the binary version of the CIFAR-10 dataset.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import argparse import os import sys import tarfile -from six.moves import urllib import tensorflow as tf from cifar10 import _DATA_DIR +from six.moves import urllib DATA_URL = "https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz" diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/network_compression.py b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/network_compression.py index f302258228..90369174a8 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/network_compression.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/tensorflow_resnet/network_compression.py @@ -1,20 +1,18 @@ -import numpy as np +import heapq +import json import logging -import shutil import os -import heapq +import shutil import sys +import numpy as np from tensorflow_resnet.compressor import ModeKeys +from tensorflow_resnet.compressor.core import Fake from tensorflow_resnet.compressor.layers import LayerState from tensorflow_resnet.compressor.layers.ops import get_tf_vars_dict +from tensorflow_resnet.compressor.resnet import ResNet18Model from tensorflow_resnet.compressor.tf_interface import TensorflowInterface as tf -from tensorflow_resnet.compressor.core import Fake - from tensorflow_resnet.dataset import Cifar10 -from tensorflow_resnet.compressor.resnet import ResNet18Model - -import json _MASTER_MODEL_NAME = "master" _MODEL_DIR = "cifar10_model" @@ -88,7 +86,7 @@ def _reset(self): ) def _perform_actions(self, epochs=5, epochs_between_evals=1): - """ This is a sub-method of the perform actions, that is focussed only on the action itself.""" + """This is a sub-method of the perform actions, that is focussed only on the action itself.""" logging.info("Performing action: " + self.params["name"]) self._reset() estimator = tf.estimator_builder( @@ -287,15 +285,15 @@ def __init__( self.reward = 0.5 def get_action_space(self): - """ Returns the number of layers in the network """ + """Returns the number of layers in the network""" return self.model_class.LAYER_COUNTS[self.model_class.size] def get_observation_space(self): - """ Returns a tuple of layer number and length of each layer descirptions """ + """Returns a tuple of layer number and length of each layer descirptions""" return (self.get_action_space(), LayerState.LAYER_STATE_LENGTH) def get_current_pos(self): - """ Returns the observation of the master network """ + """Returns the observation of the master network""" obs = self._get_observation(None, "master") pos = np.zeros(self.get_observation_space()) for k in obs.keys(): @@ -371,11 +369,11 @@ def perform_actions(self, actions, epochs=5): return True def get_reward(self): - """ Returns the value of the current reward. """ + """Returns the value of the current reward.""" return self.reward def reset_params(self, in_params): - """ Accept the params and reset them .""" + """Accept the params and reset them .""" for k in in_params.keys(): self.params[k] = in_params[k] diff --git a/reinforcement_learning/rl_network_compression_ray_custom/src/train-ray.py b/reinforcement_learning/rl_network_compression_ray_custom/src/train-ray.py index 6ace884fa6..2bedbd2091 100644 --- a/reinforcement_learning/rl_network_compression_ray_custom/src/train-ray.py +++ b/reinforcement_learning/rl_network_compression_ray_custom/src/train-ray.py @@ -3,12 +3,11 @@ import gym import ray -from ray.tune import run_experiments import ray.rllib.agents.a3c as a3c import ray.rllib.agents.ppo as ppo -from ray.tune.registry import register_env from environment import NetworkCompression - +from ray.tune import run_experiments +from ray.tune.registry import register_env from sagemaker_rl.ray_launcher import SageMakerRayLauncher diff --git a/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/config.py b/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/config.py index f44c58f226..7ebd99fd20 100644 --- a/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/config.py +++ b/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/config.py @@ -1,5 +1,5 @@ -import os import datetime +import os START_DATE = "2012-08-13" END_DATE = "2017-08-11" diff --git a/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/evaluate-coach.py b/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/evaluate-coach.py index 950aad9949..f1bad7ee5c 100644 --- a/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/evaluate-coach.py +++ b/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/evaluate-coach.py @@ -1,9 +1,10 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher, CoachConfigurationList import argparse import os + import rl_coach from rl_coach.base_parameters import Frameworks, TaskParameters from rl_coach.core_types import EnvironmentSteps +from sagemaker_rl.coach_launcher import CoachConfigurationList, SageMakerCoachPresetLauncher def inplace_replace_in_file(filepath, old, new): diff --git a/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/portfolio_env.py b/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/portfolio_env.py index ccc5f18c92..facffc4c93 100644 --- a/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/portfolio_env.py +++ b/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/portfolio_env.py @@ -1,14 +1,16 @@ """ Modified from https://github.com/vermouth1992/drl-portfolio-management/blob/master/src/environment/portfolio.py """ +import csv +from pprint import pprint + import gym import gym.spaces -from pprint import pprint import numpy as np -from utils import * -import csv from config import * +from utils import * + class PortfolioEnv(gym.Env): """ diff --git a/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/preset-portfolio-management-clippedppo.py b/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/preset-portfolio-management-clippedppo.py index 0c3b2a7fcb..d35acc2a4c 100644 --- a/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/preset-portfolio-management-clippedppo.py +++ b/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/preset-portfolio-management-clippedppo.py @@ -1,8 +1,12 @@ from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters -from rl_coach.architectures.layers import Dense, Conv2d -from rl_coach.base_parameters import VisualizationParameters, PresetValidationParameters -from rl_coach.base_parameters import MiddlewareScheme, DistributedCoachSynchronizationType -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps, RunPhase +from rl_coach.architectures.layers import Conv2d, Dense +from rl_coach.base_parameters import ( + DistributedCoachSynchronizationType, + MiddlewareScheme, + PresetValidationParameters, + VisualizationParameters, +) +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, RunPhase, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment, ObservationSpaceType from rl_coach.exploration_policies.e_greedy import EGreedyParameters from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager diff --git a/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/utils.py b/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/utils.py index 0602689365..8850f7b835 100644 --- a/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/utils.py +++ b/reinforcement_learning/rl_portfolio_management_coach_customEnv/src/utils.py @@ -1,6 +1,7 @@ -import h5py import datetime -from config import START_DATE, DATE_FORMAT, START_DATETIME + +import h5py +from config import DATE_FORMAT, START_DATE, START_DATETIME def read_stock_history(filepath): diff --git a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/autoscalesim.py b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/autoscalesim.py index 430e46a692..4420c70d5b 100644 --- a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/autoscalesim.py +++ b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/autoscalesim.py @@ -1,7 +1,7 @@ -import numpy as np import gym -from gym.spaces import Discrete, Box +import numpy as np from gym.envs.registration import EnvSpec +from gym.spaces import Box, Discrete from gymhelper import box_space_from_description diff --git a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/evaluate-coach.py b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/evaluate-coach.py index ff41671fdc..f89887dd16 100644 --- a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/evaluate-coach.py +++ b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/evaluate-coach.py @@ -1,9 +1,10 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher, CoachConfigurationList import argparse import os + import rl_coach from rl_coach.base_parameters import Frameworks, TaskParameters from rl_coach.core_types import EnvironmentSteps +from sagemaker_rl.coach_launcher import CoachConfigurationList, SageMakerCoachPresetLauncher def inplace_replace_in_file(filepath, old, new): diff --git a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/gymhelper.py b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/gymhelper.py index 41230cc1f6..ce2537a035 100644 --- a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/gymhelper.py +++ b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/gymhelper.py @@ -1,5 +1,5 @@ import numpy as np -from gym.spaces import Discrete, Box +from gym.spaces import Box, Discrete def box_space_from_description(dimension_list): diff --git a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/preset-autoscale-a3c.py b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/preset-autoscale-a3c.py index 10956ef4cc..c35df4bc9e 100644 --- a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/preset-autoscale-a3c.py +++ b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/preset-autoscale-a3c.py @@ -1,18 +1,18 @@ from rl_coach.agents.actor_critic_agent import ActorCriticAgentParameters +from rl_coach.agents.policy_optimization_agent import PolicyGradientRescaler from rl_coach.architectures.layers import Dense from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, DistributedCoachSynchronizationType, + PresetValidationParameters, + VisualizationParameters, ) -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps, RunPhase +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, RunPhase, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment, mujoco_v2 -from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager -from rl_coach.graph_managers.graph_manager import ScheduleParameters -from rl_coach.agents.policy_optimization_agent import PolicyGradientRescaler +from rl_coach.exploration_policies.e_greedy import EGreedyParameters from rl_coach.filters.filter import InputFilter from rl_coach.filters.reward.reward_rescale_filter import RewardRescaleFilter -from rl_coach.exploration_policies.e_greedy import EGreedyParameters +from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager +from rl_coach.graph_managers.graph_manager import ScheduleParameters from rl_coach.schedules import LinearSchedule #################### diff --git a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/preset-autoscale-ppo.py b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/preset-autoscale-ppo.py index f67afbe7a4..c309fdfa4b 100644 --- a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/preset-autoscale-ppo.py +++ b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/preset-autoscale-ppo.py @@ -1,11 +1,11 @@ from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters from rl_coach.architectures.layers import Dense from rl_coach.base_parameters import ( - VisualizationParameters, - PresetValidationParameters, DistributedCoachSynchronizationType, + PresetValidationParameters, + VisualizationParameters, ) -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps, RunPhase +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, RunPhase, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment, mujoco_v2 from rl_coach.exploration_policies.e_greedy import EGreedyParameters from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager diff --git a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/train-coach.py b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/train-coach.py index 9f3364d69e..828511f8a4 100644 --- a/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/train-coach.py +++ b/reinforcement_learning/rl_predictive_autoscaling_coach_customEnv/src/train-coach.py @@ -1,7 +1,8 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher -import tensorflow as tf import shutil +import tensorflow as tf +from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher + class MyLauncher(SageMakerCoachPresetLauncher): def default_preset_name(self): diff --git a/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/news_vendor_env.py b/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/news_vendor_env.py index 293a34adc8..80aa8e8c7f 100644 --- a/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/news_vendor_env.py +++ b/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/news_vendor_env.py @@ -1,6 +1,4 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import gym import numpy as np diff --git a/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_bin_packing.py b/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_bin_packing.py index 65a631f5e9..4e7f8aaa15 100644 --- a/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_bin_packing.py +++ b/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_bin_packing.py @@ -1,6 +1,5 @@ -from ray.tune.registry import register_env - from model import register_actor_mask_model +from ray.tune.registry import register_env from sagemaker_rl.ray_launcher import SageMakerRayLauncher register_actor_mask_model() diff --git a/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_news_vendor.py b/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_news_vendor.py index 0d31330127..7dcdde114c 100644 --- a/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_news_vendor.py +++ b/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_news_vendor.py @@ -1,5 +1,4 @@ from ray.tune.registry import register_env - from sagemaker_rl.ray_launcher import SageMakerRayLauncher diff --git a/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_vehicle_routing_problem.py b/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_vehicle_routing_problem.py index 9f5d617e32..9640b5c9c4 100644 --- a/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_vehicle_routing_problem.py +++ b/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/train_vehicle_routing_problem.py @@ -1,6 +1,5 @@ -from ray.tune.registry import register_env - from model import register_actor_mask_model +from ray.tune.registry import register_env from sagemaker_rl.ray_launcher import SageMakerRayLauncher register_actor_mask_model() diff --git a/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/vrp_env.py b/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/vrp_env.py index e4835cee45..a5817f182a 100644 --- a/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/vrp_env.py +++ b/reinforcement_learning/rl_resource_allocation_ray_customEnv/src/vrp_env.py @@ -2,7 +2,7 @@ import gym import numpy as np -from gym.spaces import Box, Discrete, Dict +from gym.spaces import Box, Dict, Discrete from scipy.stats import truncnorm from utils import vrp_action_go_from_a_to_b diff --git a/reinforcement_learning/rl_roboschool_ray/src/evaluate-ray.py b/reinforcement_learning/rl_roboschool_ray/src/evaluate-ray.py index 4de49a4272..31ea2b297c 100644 --- a/reinforcement_learning/rl_roboschool_ray/src/evaluate-ray.py +++ b/reinforcement_learning/rl_roboschool_ray/src/evaluate-ray.py @@ -1,16 +1,13 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import argparse import json import os -import numpy as np - import gym -from gym import wrappers +import numpy as np import ray +from gym import wrappers from ray.rllib.models import ModelCatalog from ray.tune.registry import register_env diff --git a/reinforcement_learning/rl_roboschool_ray/src/train-hopper.py b/reinforcement_learning/rl_roboschool_ray/src/train-hopper.py index 88f5560938..fd709062e5 100644 --- a/reinforcement_learning/rl_roboschool_ray/src/train-hopper.py +++ b/reinforcement_learning/rl_roboschool_ray/src/train-hopper.py @@ -3,10 +3,9 @@ import gym import ray +import roboschool from ray.tune import run_experiments from ray.tune.registry import register_env -import roboschool - from sagemaker_rl.ray_launcher import SageMakerRayLauncher diff --git a/reinforcement_learning/rl_roboschool_ray/src/train-humanoid.py b/reinforcement_learning/rl_roboschool_ray/src/train-humanoid.py index d79cda4b40..772e20c733 100644 --- a/reinforcement_learning/rl_roboschool_ray/src/train-humanoid.py +++ b/reinforcement_learning/rl_roboschool_ray/src/train-humanoid.py @@ -3,10 +3,9 @@ import gym import ray +import roboschool from ray.tune import run_experiments from ray.tune.registry import register_env -import roboschool - from sagemaker_rl.ray_launcher import SageMakerRayLauncher diff --git a/reinforcement_learning/rl_roboschool_ray/src/train-reacher.py b/reinforcement_learning/rl_roboschool_ray/src/train-reacher.py index 732afbf8af..438cb69e6a 100644 --- a/reinforcement_learning/rl_roboschool_ray/src/train-reacher.py +++ b/reinforcement_learning/rl_roboschool_ray/src/train-reacher.py @@ -3,10 +3,9 @@ import gym import ray +import roboschool from ray.tune import run_experiments from ray.tune.registry import register_env -import roboschool - from sagemaker_rl.ray_launcher import SageMakerRayLauncher diff --git a/reinforcement_learning/rl_stock_trading_coach_customEnv/src/data.py b/reinforcement_learning/rl_stock_trading_coach_customEnv/src/data.py index 1115a76170..1a66f525e2 100644 --- a/reinforcement_learning/rl_stock_trading_coach_customEnv/src/data.py +++ b/reinforcement_learning/rl_stock_trading_coach_customEnv/src/data.py @@ -1,8 +1,9 @@ -import os +import collections import csv import glob +import os + import numpy as np -import collections Prices = collections.namedtuple("Prices", field_names=["open", "high", "low", "close", "volume"]) diff --git a/reinforcement_learning/rl_stock_trading_coach_customEnv/src/evaluate-coach.py b/reinforcement_learning/rl_stock_trading_coach_customEnv/src/evaluate-coach.py index 210fc22dca..e033c97923 100644 --- a/reinforcement_learning/rl_stock_trading_coach_customEnv/src/evaluate-coach.py +++ b/reinforcement_learning/rl_stock_trading_coach_customEnv/src/evaluate-coach.py @@ -1,9 +1,10 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher, CoachConfigurationList import argparse import os + import rl_coach from rl_coach.base_parameters import Frameworks, TaskParameters from rl_coach.core_types import EnvironmentSteps +from sagemaker_rl.coach_launcher import CoachConfigurationList, SageMakerCoachPresetLauncher def inplace_replace_in_file(filepath, old, new): diff --git a/reinforcement_learning/rl_stock_trading_coach_customEnv/src/preset-stock-trading-ddqn.py b/reinforcement_learning/rl_stock_trading_coach_customEnv/src/preset-stock-trading-ddqn.py index d7e23a656c..6a238adba1 100644 --- a/reinforcement_learning/rl_stock_trading_coach_customEnv/src/preset-stock-trading-ddqn.py +++ b/reinforcement_learning/rl_stock_trading_coach_customEnv/src/preset-stock-trading-ddqn.py @@ -2,13 +2,13 @@ from rl_coach.agents.ddqn_agent import DDQNAgentParameters from rl_coach.architectures.head_parameters import DuelingQHeadParameters from rl_coach.architectures.layers import Dense -from rl_coach.base_parameters import VisualizationParameters, PresetValidationParameters -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps +from rl_coach.base_parameters import PresetValidationParameters, VisualizationParameters +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager from rl_coach.graph_managers.graph_manager import ScheduleParameters -from rl_coach.schedules import LinearSchedule from rl_coach.memories.memory import MemoryGranularity +from rl_coach.schedules import LinearSchedule ################# # Graph Scheduling diff --git a/reinforcement_learning/rl_stock_trading_coach_customEnv/src/trading_env.py b/reinforcement_learning/rl_stock_trading_coach_customEnv/src/trading_env.py index 0f3a747af4..dc86e5ce41 100644 --- a/reinforcement_learning/rl_stock_trading_coach_customEnv/src/trading_env.py +++ b/reinforcement_learning/rl_stock_trading_coach_customEnv/src/trading_env.py @@ -1,14 +1,14 @@ """ Modified from https://github.com/PacktPublishing/Deep-Reinforcement-Learning-Hands-On/blob/master/Chapter08/lib/environ.py """ +import enum + import gym import gym.spaces -from gym.utils import seeding -import enum import numpy as np - -from data import * from config import * +from data import * +from gym.utils import seeding class TradingEnv(gym.Env): diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/docker_utils.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/docker_utils.py index 5487032802..f5c3486dfa 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/docker_utils.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/docker_utils.py @@ -15,16 +15,16 @@ import base64 import contextlib +import json import os -import time import shlex import shutil import subprocess import sys import tempfile +import time import boto3 -import json IMAGE_TEMPLATE = "{account}.dkr.ecr.{region}.amazonaws.com/{image_name}:{version}" diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/misc.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/misc.py index 8adda51671..5cec938528 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/misc.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/misc.py @@ -15,16 +15,16 @@ import base64 import contextlib +import json import os -import time import shlex import shutil import subprocess import sys import tempfile +import time import boto3 -import json def wait_for_s3_object( diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/coach_launcher.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/coach_launcher.py index ce9891cc35..dd71e0e189 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/coach_launcher.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/coach_launcher.py @@ -1,24 +1,25 @@ -from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters -from rl_coach.agents.policy_gradients_agent import PolicyGradientsAgentParameters -from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager -from rl_coach.graph_managers.graph_manager import ScheduleParameters -from rl_coach.base_parameters import VisualizationParameters, TaskParameters, Frameworks -from rl_coach.utils import short_dynamic_import -from rl_coach.core_types import SelectedPhaseOnlyDumpFilter, MaxDumpFilter, RunPhase -import rl_coach.core_types -from rl_coach import logger -from rl_coach.logger import screen import argparse import copy +import glob import logging import os -import sys -import shutil -import glob import re +import shutil +import sys -from .configuration_list import ConfigurationList +import rl_coach.core_types +from rl_coach import logger +from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters +from rl_coach.agents.policy_gradients_agent import PolicyGradientsAgentParameters +from rl_coach.base_parameters import Frameworks, TaskParameters, VisualizationParameters from rl_coach.coach import CoachLauncher +from rl_coach.core_types import MaxDumpFilter, RunPhase, SelectedPhaseOnlyDumpFilter +from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager +from rl_coach.graph_managers.graph_manager import ScheduleParameters +from rl_coach.logger import screen +from rl_coach.utils import short_dynamic_import + +from .configuration_list import ConfigurationList screen.set_use_colors(False) # Simple text logging so it looks good in CloudWatch diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/mpi_launcher.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/mpi_launcher.py index e42ca5ad30..94ef785fa5 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/mpi_launcher.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/mpi_launcher.py @@ -1,19 +1,15 @@ -import stat - +import os import shlex +import signal import socket +import stat +import subprocess import sys - import textwrap -from contextlib import contextmanager -import signal - -import os -import subprocess import time +from contextlib import contextmanager import sagemaker_containers - from retrying import retry from sagemaker_containers import _logging from sagemaker_containers.beta import framework @@ -204,7 +200,7 @@ def is_master(self, hosts, current_host): class MPIWorker(object): - """ MPI Worker""" + """MPI Worker""" @retry( stop_max_delay=30000 * 1000, wait_fixed=1000, retry_on_result=lambda result: result is False diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/onnx_utils.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/onnx_utils.py index 120eecb51c..1ade41a494 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/onnx_utils.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/onnx_utils.py @@ -2,7 +2,7 @@ ONNX Utils to support multiple output heads in agent networks, until future releases of MXNet support this. """ import onnx -from onnx import helper, checker, TensorProto +from onnx import TensorProto, checker, helper def get_correct_outputs(model): diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/ray_launcher.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/ray_launcher.py index bb9f3ed981..b88c792300 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/ray_launcher.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/ray_launcher.py @@ -6,13 +6,12 @@ from enum import Enum import boto3 - import ray from ray.tune import run_experiments from .configuration_list import ConfigurationList -from .sage_cluster_communicator import SageClusterCommunicator from .docker_utils import get_ip_from_host +from .sage_cluster_communicator import SageClusterCommunicator TERMINATION_SIGNAL = "JOB_TERMINATED" diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/sage_cluster_communicator.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/sage_cluster_communicator.py index 7ff115e616..558b38bec2 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/sage_cluster_communicator.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/sage_cluster_communicator.py @@ -1,9 +1,10 @@ -import boto3 -import os import io import json +import os import time +import boto3 + class SageClusterCommunicator: def __init__(self): diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/stable_baselines_launcher.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/stable_baselines_launcher.py index ef9bf72f13..76d6eb9564 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/stable_baselines_launcher.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/common/sagemaker_rl/stable_baselines_launcher.py @@ -1,14 +1,13 @@ -import gym -import roboschool import os +import gym +import roboschool from gym.wrappers.monitoring.video_recorder import VideoRecorder -from stable_baselines.ppo1 import PPO1 -from stable_baselines.common import set_global_seeds +from mpi4py import MPI from stable_baselines.bench import Monitor -from stable_baselines.common import tf_util +from stable_baselines.common import set_global_seeds, tf_util from stable_baselines.common.policies import MlpPolicy -from mpi4py import MPI +from stable_baselines.ppo1 import PPO1 class RewScale(gym.RewardWrapper): diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/deploy-coach.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/deploy-coach.py index 7f0439d92b..5f93328ca8 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/deploy-coach.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/deploy-coach.py @@ -1,9 +1,10 @@ -import os import json +import os + import mxnet as mx -from mxnet.contrib import onnx as onnx_mxnet -from mxnet import gluon, nd import numpy as np +from mxnet import gluon, nd +from mxnet.contrib import onnx as onnx_mxnet def model_fn(model_dir): diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/preset.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/preset.py index 59743e83da..2b4c496e0f 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/preset.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/preset.py @@ -1,6 +1,6 @@ from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters -from rl_coach.base_parameters import VisualizationParameters, PresetValidationParameters -from rl_coach.core_types import TrainingSteps, EnvironmentEpisodes, EnvironmentSteps +from rl_coach.base_parameters import PresetValidationParameters, VisualizationParameters +from rl_coach.core_types import EnvironmentEpisodes, EnvironmentSteps, TrainingSteps from rl_coach.environments.gym_environment import GymVectorEnvironment from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager from rl_coach.graph_managers.graph_manager import ScheduleParameters diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/tic_tac_toe.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/tic_tac_toe.py index 0b11600569..f9d8e68d57 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/tic_tac_toe.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/src/tic_tac_toe.py @@ -1,9 +1,10 @@ -import gym -from gym import spaces -import numpy as np import os import time +import gym +import numpy as np +from gym import spaces + class TicTacToeEnv(gym.Env): def __init__(self, opponent="moderate"): diff --git a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/tic_tac_toe_game.py b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/tic_tac_toe_game.py index d89deb13a0..f1aee77c74 100644 --- a/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/tic_tac_toe_game.py +++ b/reinforcement_learning/rl_tic_tac_toe_coach_customEnv/tic_tac_toe_game.py @@ -1,7 +1,8 @@ -from ipywidgets import widgets, HBox, VBox, Layout -from IPython.display import display from functools import partial + import numpy as np +from IPython.display import display +from ipywidgets import HBox, Layout, VBox, widgets class TicTacToeGame(object): diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_baseline.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_baseline.py index 45ef714472..0f1367a87d 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_baseline.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_baseline.py @@ -1,7 +1,6 @@ from time import sleep import numpy as np - from TSP_baseline_utils import tsp_action_go_from_a_to_b, tsp_dp_approx_sol, tsp_dp_opt_sol from TSP_env import TSPEasyEnv, TSPMediumEnv diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_env.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_env.py index 2d08ef0ae6..b7d5d2b8fa 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_env.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_env.py @@ -1,7 +1,6 @@ import gym import numpy as np from gym import spaces - from TSP_view_2D import TSPView2D diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_view_2D.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_view_2D.py index 544c61051f..db690a0f45 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_view_2D.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/TSP_view_2D.py @@ -1,5 +1,5 @@ -import pygame import numpy as np +import pygame class TSPView2D: diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_abstract_env.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_abstract_env.py index f66563ce62..de07d0598b 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_abstract_env.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_abstract_env.py @@ -1,9 +1,8 @@ import gym import numpy as np from gym import spaces - -from VRP_view_2D import VRPView2D from VRP_baseline_utils import vrp_action_go_from_a_to_b +from VRP_view_2D import VRPView2D """ diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_baseline_easy.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_baseline_easy.py index ea5b1c2a16..1936b4e4e6 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_baseline_easy.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_baseline_easy.py @@ -1,8 +1,7 @@ +import copy from time import sleep -import copy import numpy as np - from VRP_baseline_utils import decide_action from VRP_env import VRPEasyEnv, VRPMediumEnv diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_baseline_utils.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_baseline_utils.py index 9610e93e02..889c17b99d 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_baseline_utils.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_baseline_utils.py @@ -1,6 +1,7 @@ -import numpy as np -import itertools import copy +import itertools + +import numpy as np def decide_action(prev_o_status, env, visit_stops): diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_env.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_env.py index f06e76bfbc..9c31323962 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_env.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_env.py @@ -1,7 +1,6 @@ import gym import numpy as np from gym import spaces - from rl_operations_research_baselines.VRP.VRP_view_2D import VRPView2D """ diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_view_2D.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_view_2D.py index b5e1bb4310..bd1157065f 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_view_2D.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/VRP_view_2D.py @@ -1,5 +1,5 @@ -import pygame import numpy as np +import pygame class VRPView2D: diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/evaluate-coach.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/evaluate-coach.py index 18461bc13c..1c2bdda975 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/evaluate-coach.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/evaluate-coach.py @@ -1,9 +1,10 @@ -from sagemaker_rl.coach_launcher import SageMakerCoachPresetLauncher, CoachConfigurationList import argparse import os + import rl_coach from rl_coach.base_parameters import Frameworks, TaskParameters from rl_coach.core_types import EnvironmentSteps +from sagemaker_rl.coach_launcher import CoachConfigurationList, SageMakerCoachPresetLauncher def inplace_replace_in_file(filepath, old, new): diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-tsp-easy.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-tsp-easy.py index 2416137f52..fa84fbf0c0 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-tsp-easy.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-tsp-easy.py @@ -1,13 +1,11 @@ +# from rl_coach.environments.environment import SelectedPhaseOnlyDumpMethod, MaxDumpMethod +from rl_coach import logger from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters +from rl_coach.base_parameters import TaskParameters, VisualizationParameters +from rl_coach.core_types import * from rl_coach.environments.gym_environment import GymVectorEnvironment from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager from rl_coach.graph_managers.graph_manager import SimpleSchedule -from rl_coach.core_types import * - -# from rl_coach.environments.environment import SelectedPhaseOnlyDumpMethod, MaxDumpMethod -from rl_coach import logger -from rl_coach.base_parameters import TaskParameters -from rl_coach.base_parameters import VisualizationParameters ################ # Environment # diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-tsp-medium.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-tsp-medium.py index 1712745ee2..97c27c44c8 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-tsp-medium.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-tsp-medium.py @@ -1,13 +1,11 @@ +# from rl_coach.environments.environment import SelectedPhaseOnlyDumpMethod, MaxDumpMethod +from rl_coach import logger from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters +from rl_coach.base_parameters import TaskParameters, VisualizationParameters +from rl_coach.core_types import * from rl_coach.environments.gym_environment import GymVectorEnvironment from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager from rl_coach.graph_managers.graph_manager import SimpleSchedule -from rl_coach.core_types import * - -# from rl_coach.environments.environment import SelectedPhaseOnlyDumpMethod, MaxDumpMethod -from rl_coach import logger -from rl_coach.base_parameters import TaskParameters -from rl_coach.base_parameters import VisualizationParameters ################ # Environment # diff --git a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-vrp-easy.py b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-vrp-easy.py index cf7bd856a7..57d7a2d08e 100644 --- a/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-vrp-easy.py +++ b/reinforcement_learning/rl_traveling_salesman_vehicle_routing_coach/src/preset-vrp-easy.py @@ -1,13 +1,11 @@ +# from rl_coach.environments.environment import SelectedPhaseOnlyDumpMethod, MaxDumpMethod +from rl_coach import logger from rl_coach.agents.clipped_ppo_agent import ClippedPPOAgentParameters +from rl_coach.base_parameters import TaskParameters, VisualizationParameters +from rl_coach.core_types import * from rl_coach.environments.gym_environment import GymVectorEnvironment from rl_coach.graph_managers.basic_rl_graph_manager import BasicRLGraphManager from rl_coach.graph_managers.graph_manager import SimpleSchedule -from rl_coach.core_types import * - -# from rl_coach.environments.environment import SelectedPhaseOnlyDumpMethod, MaxDumpMethod -from rl_coach import logger -from rl_coach.base_parameters import TaskParameters -from rl_coach.base_parameters import VisualizationParameters ################ # Environment # diff --git a/reinforcement_learning/rl_unity_ray/src/evaluate-unity.py b/reinforcement_learning/rl_unity_ray/src/evaluate-unity.py index 8a0941ccb3..3fa67dee25 100644 --- a/reinforcement_learning/rl_unity_ray/src/evaluate-unity.py +++ b/reinforcement_learning/rl_unity_ray/src/evaluate-unity.py @@ -1,25 +1,20 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import argparse import json import os import subprocess -import numpy as np -import os - import gym -from gym import wrappers +import numpy as np import ray -from ray.rllib.models import ModelCatalog -from ray.tune.registry import register_env +from gym import wrappers +from gym_unity.envs import UnityToGymWrapper from mlagents_envs.environment import UnityEnvironment from mlagents_envs.exception import UnityWorkerInUseException from mlagents_envs.registry import default_registry -from gym_unity.envs import UnityToGymWrapper - +from ray.rllib.models import ModelCatalog +from ray.tune.registry import register_env OUTPUT_DIR = "/opt/ml/output/intermediate" diff --git a/reinforcement_learning/rl_unity_ray/src/train-unity.py b/reinforcement_learning/rl_unity_ray/src/train-unity.py index 3ab419ac5a..61936d3c71 100644 --- a/reinforcement_learning/rl_unity_ray/src/train-unity.py +++ b/reinforcement_learning/rl_unity_ray/src/train-unity.py @@ -3,14 +3,13 @@ import gym import ray -from ray.tune import run_experiments -from ray.tune.registry import register_env - -from sagemaker_rl.ray_launcher import SageMakerRayLauncher +from gym_unity.envs import UnityToGymWrapper from mlagents_envs.environment import UnityEnvironment from mlagents_envs.exception import UnityWorkerInUseException from mlagents_envs.registry import default_registry -from gym_unity.envs import UnityToGymWrapper +from ray.tune import run_experiments +from ray.tune.registry import register_env +from sagemaker_rl.ray_launcher import SageMakerRayLauncher class UnityEnvWrapper(gym.Env): diff --git a/sagemaker-debugger/build_your_own_container_with_debugger/docker/tf_keras_resnet_byoc.py b/sagemaker-debugger/build_your_own_container_with_debugger/docker/tf_keras_resnet_byoc.py index 9434266b7c..1ed9ab6b04 100644 --- a/sagemaker-debugger/build_your_own_container_with_debugger/docker/tf_keras_resnet_byoc.py +++ b/sagemaker-debugger/build_your_own_container_with_debugger/docker/tf_keras_resnet_byoc.py @@ -14,14 +14,14 @@ # Third Party import numpy as np + +# smdebug modification: Import smdebug support for Tensorflow +import smdebug.tensorflow as smd import tensorflow.compat.v2 as tf from tensorflow.keras.applications.resnet50 import ResNet50 from tensorflow.keras.datasets import cifar10 from tensorflow.keras.utils import to_categorical -# smdebug modification: Import smdebug support for Tensorflow -import smdebug.tensorflow as smd - def train(batch_size, epoch, model, hook): (X_train, y_train), (X_valid, y_valid) = cifar10.load_data() diff --git a/sagemaker-debugger/debugger_interactive_analysis_profiling/train_tf.py b/sagemaker-debugger/debugger_interactive_analysis_profiling/train_tf.py index 41c2bf4ccf..3a40c0b1b0 100644 --- a/sagemaker-debugger/debugger_interactive_analysis_profiling/train_tf.py +++ b/sagemaker-debugger/debugger_interactive_analysis_profiling/train_tf.py @@ -5,14 +5,14 @@ # Third Party import numpy as np import tensorflow.compat.v2 as tf + +# First Party +from smdebug.tensorflow import KerasHook from tensorflow.keras.applications.resnet50 import ResNet50 from tensorflow.keras.datasets import cifar10 from tensorflow.keras.preprocessing.image import ImageDataGenerator from tensorflow.keras.utils import to_categorical -# First Party -from smdebug.tensorflow import KerasHook - def between_steps_bottleneck(): time.sleep(1) diff --git a/sagemaker-debugger/mnist_tensor_analysis/src/mnist.py b/sagemaker-debugger/mnist_tensor_analysis/src/mnist.py index 877a565895..bffbcce401 100644 --- a/sagemaker-debugger/mnist_tensor_analysis/src/mnist.py +++ b/sagemaker-debugger/mnist_tensor_analysis/src/mnist.py @@ -1,5 +1,7 @@ # Standard Library import argparse +import os +import time # Third Party import mxnet as mx @@ -7,8 +9,6 @@ from mxnet import autograd, gluon, init from mxnet.gluon import nn from mxnet.gluon.data.vision import datasets, transforms -import os -import time def parse_args(): diff --git a/sagemaker-debugger/mnist_tensor_analysis/utils.py b/sagemaker-debugger/mnist_tensor_analysis/utils.py index 887a58d43c..adf661beec 100644 --- a/sagemaker-debugger/mnist_tensor_analysis/utils.py +++ b/sagemaker-debugger/mnist_tensor_analysis/utils.py @@ -1,6 +1,7 @@ -import numpy as np import math + import matplotlib.pyplot as plt +import numpy as np from matplotlib.animation import FuncAnimation plt.rcParams.update({"font.size": 8}) diff --git a/sagemaker-debugger/model_specific_realtime_analysis/autoencoder_mnist/src/autoencoder_mnist.py b/sagemaker-debugger/model_specific_realtime_analysis/autoencoder_mnist/src/autoencoder_mnist.py index 24ca9b234a..d877e3b847 100644 --- a/sagemaker-debugger/model_specific_realtime_analysis/autoencoder_mnist/src/autoencoder_mnist.py +++ b/sagemaker-debugger/model_specific_realtime_analysis/autoencoder_mnist/src/autoencoder_mnist.py @@ -1,7 +1,8 @@ -import mxnet as mx -from mxnet import autograd, gluon import argparse + +import mxnet as mx import smdebug.mxnet as smd +from mxnet import autograd, gluon from smdebug.mxnet import modes diff --git a/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/entry_point/data.py b/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/entry_point/data.py index 5129900ffb..4d40ca40d1 100644 --- a/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/entry_point/data.py +++ b/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/entry_point/data.py @@ -18,9 +18,9 @@ import time from functools import partial -from mxnet.gluon.data import SimpleDataset -from gluonnlp.data.utils import whitespace_splitter import numpy as np +from gluonnlp.data.utils import whitespace_splitter +from mxnet.gluon.data import SimpleDataset __all__ = ["SQuADTransform", "\rocess_dataset"] diff --git a/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/entry_point/train.py b/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/entry_point/train.py index 940b039bf6..c6d6be060b 100644 --- a/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/entry_point/train.py +++ b/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/entry_point/train.py @@ -1,14 +1,13 @@ import argparse import time -import numpy as np -import mxnet as mx import gluonnlp as nlp -from gluonnlp.data import SQuAD -from model import BertForQALoss, BertForQA -from data import SQuADTransform, preprocess_dataset - +import mxnet as mx +import numpy as np import smdebug.mxnet as smd +from data import SQuADTransform, preprocess_dataset +from gluonnlp.data import SQuAD +from model import BertForQA, BertForQALoss from smdebug import modes diff --git a/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/utils/attention_head_view.py b/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/utils/attention_head_view.py index 6c8b35271b..38b4c429ce 100644 --- a/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/utils/attention_head_view.py +++ b/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/utils/attention_head_view.py @@ -1,9 +1,9 @@ -from bokeh.plotting import show, figure -from bokeh.models.annotations import Title +import numpy as np +from bokeh.io import output_notebook, push_notebook, show from bokeh.models import ColumnDataSource, Label, Range1d -from bokeh.io import show, output_notebook, push_notebook +from bokeh.models.annotations import Title from bokeh.models.glyphs import Line -import numpy as np +from bokeh.plotting import figure, show output_notebook() diff --git a/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/utils/neuron_view.py b/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/utils/neuron_view.py index 0e4f270ab6..8dfb52ced9 100644 --- a/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/utils/neuron_view.py +++ b/sagemaker-debugger/model_specific_realtime_analysis/bert_attention_head_view/utils/neuron_view.py @@ -1,10 +1,10 @@ -from bokeh.plotting import show, figure, gridplot -from bokeh.models.annotations import Title +import numpy as np +from bokeh.io import output_notebook, push_notebook, show from bokeh.models import ColumnDataSource, Label, Range1d -from bokeh.io import show, output_notebook, push_notebook -from bokeh.models.glyphs import Line, Image +from bokeh.models.annotations import Title +from bokeh.models.glyphs import Image, Line from bokeh.models.mappers import LinearColorMapper -import numpy as np +from bokeh.plotting import figure, gridplot, show output_notebook() diff --git a/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/custom_hook.py b/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/custom_hook.py index 3858379a4d..1c445eb15f 100644 --- a/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/custom_hook.py +++ b/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/custom_hook.py @@ -1,6 +1,6 @@ import smdebug.pytorch as smd -from smdebug.core.modes import ModeKeys import torch +from smdebug.core.modes import ModeKeys class CustomHook(smd.Hook): diff --git a/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/train.py b/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/train.py index 1793d93c80..c5e57b9429 100644 --- a/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/train.py +++ b/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/train.py @@ -1,16 +1,16 @@ -from __future__ import print_function, division +from __future__ import division, print_function import argparse -import torch -import torch.nn as nn -import torch.optim as optim -from torchvision import datasets, models, transforms import os +import custom_hook import smdebug.pytorch as smd +import torch +import torch.nn as nn +import torch.optim as optim from smdebug import modes from smdebug.core.modes import ModeKeys -import custom_hook +from torchvision import datasets, models, transforms def get_dataloaders(batch_size_train, batch_size_val): diff --git a/sagemaker-debugger/mxnet_spot_training/mxnet_gluon_spot_training.py b/sagemaker-debugger/mxnet_spot_training/mxnet_gluon_spot_training.py index 5314eb05db..8fc486ef78 100644 --- a/sagemaker-debugger/mxnet_spot_training/mxnet_gluon_spot_training.py +++ b/sagemaker-debugger/mxnet_spot_training/mxnet_gluon_spot_training.py @@ -1,5 +1,6 @@ # Standard Library import argparse +import os import random # Third Party @@ -7,7 +8,6 @@ import numpy as np from mxnet import autograd, gluon from mxnet.gluon import nn -import os def parse_args(): @@ -116,7 +116,8 @@ def create_gluon_model(): def validate(): - import os, json + import json + import os with open("/opt/ml/input/config/debughookconfig.json") as jsondata: configs = json.load(jsondata) diff --git a/sagemaker-debugger/pytorch_custom_container/scripts/pytorch_mnist.py b/sagemaker-debugger/pytorch_custom_container/scripts/pytorch_mnist.py index 44f586d0cf..55aa870abc 100644 --- a/sagemaker-debugger/pytorch_custom_container/scripts/pytorch_mnist.py +++ b/sagemaker-debugger/pytorch_custom_container/scripts/pytorch_mnist.py @@ -11,13 +11,19 @@ from __future__ import absolute_import + import argparse import logging import os +import random import sys import cv2 as cv +import numpy as np import sagemaker_containers + +# SageMaker Debugger: Import the package +import smdebug.pytorch as smd import torch import torch.nn as nn import torch.nn.functional as F @@ -25,12 +31,6 @@ import torch.utils.data from torchvision import datasets, transforms -# SageMaker Debugger: Import the package -import smdebug.pytorch as smd - -import numpy as np -import random - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) diff --git a/sagemaker-debugger/pytorch_iterative_model_pruning/custom_rule/check_accuracy.py b/sagemaker-debugger/pytorch_iterative_model_pruning/custom_rule/check_accuracy.py index 466aefb36f..2dd322eda0 100644 --- a/sagemaker-debugger/pytorch_iterative_model_pruning/custom_rule/check_accuracy.py +++ b/sagemaker-debugger/pytorch_iterative_model_pruning/custom_rule/check_accuracy.py @@ -1,6 +1,6 @@ -from smdebug.rules.rule import Rule -from smdebug import modes import numpy as np +from smdebug import modes +from smdebug.rules.rule import Rule class check_accuracy(Rule): diff --git a/sagemaker-debugger/pytorch_iterative_model_pruning/model_alexnet.py b/sagemaker-debugger/pytorch_iterative_model_pruning/model_alexnet.py index 32e7428978..27d0115095 100644 --- a/sagemaker-debugger/pytorch_iterative_model_pruning/model_alexnet.py +++ b/sagemaker-debugger/pytorch_iterative_model_pruning/model_alexnet.py @@ -1,10 +1,10 @@ -import torch -from torchvision import models -import torchvision -import torch.nn as nn import numpy as np import smdebug +import torch +import torch.nn as nn +import torchvision from smdebug import modes +from torchvision import models # list of ordered tensor names activation_outputs = [ diff --git a/sagemaker-debugger/pytorch_iterative_model_pruning/model_resnet.py b/sagemaker-debugger/pytorch_iterative_model_pruning/model_resnet.py index ba3c47944d..1c2f10bfd8 100644 --- a/sagemaker-debugger/pytorch_iterative_model_pruning/model_resnet.py +++ b/sagemaker-debugger/pytorch_iterative_model_pruning/model_resnet.py @@ -1,10 +1,10 @@ -import torch -from torchvision import models -import torchvision -import torch.nn as nn import numpy as np import smdebug +import torch +import torch.nn as nn +import torchvision from smdebug import modes +from torchvision import models # list of ordered tensor names activation_outputs = [ diff --git a/sagemaker-debugger/pytorch_iterative_model_pruning/src/train.py b/sagemaker-debugger/pytorch_iterative_model_pruning/src/train.py index 640221e15b..9f6e9e2c70 100644 --- a/sagemaker-debugger/pytorch_iterative_model_pruning/src/train.py +++ b/sagemaker-debugger/pytorch_iterative_model_pruning/src/train.py @@ -1,16 +1,17 @@ +import argparse +import logging +import os +import sys + +import custom_hook +import numpy as np +import smdebug.pytorch as smd import torch -from torchvision import models +import torch.optim as optim import torch.utils.data as data import torchvision.datasets as datasets import torchvision.transforms as transforms -import torch.optim as optim -import custom_hook -import smdebug.pytorch as smd -import numpy as np -import argparse -import logging -import sys -import os +from torchvision import models logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/sagemaker-debugger/pytorch_profiling/entry_point/distributed_launch.py b/sagemaker-debugger/pytorch_profiling/entry_point/distributed_launch.py index e43db460f1..177104fca0 100644 --- a/sagemaker-debugger/pytorch_profiling/entry_point/distributed_launch.py +++ b/sagemaker-debugger/pytorch_profiling/entry_point/distributed_launch.py @@ -7,7 +7,6 @@ import sys from argparse import REMAINDER, ArgumentParser - r""" `torch.distributed.launch` is a module that spawns up multiple distributed training processes on each of the training nodes. diff --git a/sagemaker-debugger/tensorflow2/tensorflow2_keras_custom_container/src/tf_keras_resnet_byoc.py b/sagemaker-debugger/tensorflow2/tensorflow2_keras_custom_container/src/tf_keras_resnet_byoc.py index 4533182ab3..56cc889d70 100644 --- a/sagemaker-debugger/tensorflow2/tensorflow2_keras_custom_container/src/tf_keras_resnet_byoc.py +++ b/sagemaker-debugger/tensorflow2/tensorflow2_keras_custom_container/src/tf_keras_resnet_byoc.py @@ -14,14 +14,14 @@ # Third Party import numpy as np + +# smdebug modification: Import smdebug support for Tensorflow +import smdebug.tensorflow as smd import tensorflow.compat.v2 as tf from tensorflow.keras.applications.resnet50 import ResNet50 from tensorflow.keras.datasets import cifar10 from tensorflow.keras.utils import to_categorical -# smdebug modification: Import smdebug support for Tensorflow -import smdebug.tensorflow as smd - def train(batch_size, epoch, model, hook): (X_train, y_train), (X_valid, y_valid) = cifar10.load_data() diff --git a/sagemaker-debugger/tensorflow2/tensorflow2_zero_code_change/src/keras_fit_zero_code_change.py b/sagemaker-debugger/tensorflow2/tensorflow2_zero_code_change/src/keras_fit_zero_code_change.py index 52bf3a291b..416374a7a7 100644 --- a/sagemaker-debugger/tensorflow2/tensorflow2_zero_code_change/src/keras_fit_zero_code_change.py +++ b/sagemaker-debugger/tensorflow2/tensorflow2_zero_code_change/src/keras_fit_zero_code_change.py @@ -8,14 +8,14 @@ # Standard Library import argparse -import time import os +import time # Third Party import numpy as np from tensorflow.keras.applications.resnet50 import ResNet50 -from tensorflow.keras.datasets import cifar10 from tensorflow.keras.callbacks import Callback +from tensorflow.keras.datasets import cifar10 from tensorflow.keras.optimizers import Adam from tensorflow.keras.utils import to_categorical diff --git a/sagemaker-debugger/tensorflow_action_on_rule/src/mnist_zerocodechange.py b/sagemaker-debugger/tensorflow_action_on_rule/src/mnist_zerocodechange.py index e091dea52b..1b09760cad 100644 --- a/sagemaker-debugger/tensorflow_action_on_rule/src/mnist_zerocodechange.py +++ b/sagemaker-debugger/tensorflow_action_on_rule/src/mnist_zerocodechange.py @@ -8,14 +8,13 @@ # Standard Library import argparse +import logging import random # Third Party import numpy as np import tensorflow as tf -import logging - logging.getLogger().setLevel(logging.INFO) parser = argparse.ArgumentParser() diff --git a/sagemaker-debugger/tensorflow_builtin_rule/src/tf-resnet50-cifar10.py b/sagemaker-debugger/tensorflow_builtin_rule/src/tf-resnet50-cifar10.py index 187bd321e6..781ba694d0 100644 --- a/sagemaker-debugger/tensorflow_builtin_rule/src/tf-resnet50-cifar10.py +++ b/sagemaker-debugger/tensorflow_builtin_rule/src/tf-resnet50-cifar10.py @@ -1,7 +1,7 @@ # Standard Library import argparse -import time import os +import time # Third Party import numpy as np diff --git a/sagemaker-debugger/tensorflow_keras_custom_rule/src/tf_keras_resnet_byoc.py b/sagemaker-debugger/tensorflow_keras_custom_rule/src/tf_keras_resnet_byoc.py index 57dd1a6a49..2d70d368e8 100644 --- a/sagemaker-debugger/tensorflow_keras_custom_rule/src/tf_keras_resnet_byoc.py +++ b/sagemaker-debugger/tensorflow_keras_custom_rule/src/tf_keras_resnet_byoc.py @@ -14,11 +14,11 @@ # Third Party import numpy as np +import smdebug.tensorflow as smd import tensorflow as tf from tensorflow.keras.applications.resnet50 import ResNet50 from tensorflow.keras.datasets import cifar10 from tensorflow.keras.utils import to_categorical -import smdebug.tensorflow as smd def train(batch_size, epoch, model): diff --git a/sagemaker-debugger/tensorflow_keras_custom_rule/src/tf_keras_resnet_zerocodechange.py b/sagemaker-debugger/tensorflow_keras_custom_rule/src/tf_keras_resnet_zerocodechange.py index 543dc573a7..1e9bfd63de 100644 --- a/sagemaker-debugger/tensorflow_keras_custom_rule/src/tf_keras_resnet_zerocodechange.py +++ b/sagemaker-debugger/tensorflow_keras_custom_rule/src/tf_keras_resnet_zerocodechange.py @@ -14,8 +14,8 @@ # Third Party import numpy as np from tensorflow.keras.applications.resnet50 import ResNet50 -from tensorflow.keras.datasets import cifar10 from tensorflow.keras.callbacks import Callback +from tensorflow.keras.datasets import cifar10 from tensorflow.keras.optimizers import Adam from tensorflow.keras.utils import to_categorical diff --git a/sagemaker-debugger/tensorflow_nlp_sentiment_analysis/tf-sentiment-script-mode/sentiment-distributed.py b/sagemaker-debugger/tensorflow_nlp_sentiment_analysis/tf-sentiment-script-mode/sentiment-distributed.py index 9f42aa36ea..7dbf115e79 100644 --- a/sagemaker-debugger/tensorflow_nlp_sentiment_analysis/tf-sentiment-script-mode/sentiment-distributed.py +++ b/sagemaker-debugger/tensorflow_nlp_sentiment_analysis/tf-sentiment-script-mode/sentiment-distributed.py @@ -1,9 +1,9 @@ import argparse -import numpy as np import os -import tensorflow.compat.v2 as tf -import horovod.tensorflow.keras as hvd +import horovod.tensorflow.keras as hvd +import numpy as np +import tensorflow.compat.v2 as tf max_features = 20000 maxlen = 400 diff --git a/sagemaker-debugger/tensorflow_profiling/demo/generate_cifar10_tfrecords.py b/sagemaker-debugger/tensorflow_profiling/demo/generate_cifar10_tfrecords.py index 405b5efc87..67eccef4da 100644 --- a/sagemaker-debugger/tensorflow_profiling/demo/generate_cifar10_tfrecords.py +++ b/sagemaker-debugger/tensorflow_profiling/demo/generate_cifar10_tfrecords.py @@ -14,11 +14,11 @@ import argparse import os import sys - import tarfile + +import tensorflow as tf from six.moves import cPickle as pickle from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow as tf # import tensorflow_datasets as tfds diff --git a/sagemaker-debugger/tensorflow_profiling/demo/generate_large_tfrecords.py b/sagemaker-debugger/tensorflow_profiling/demo/generate_large_tfrecords.py index 125314335e..51f289425b 100644 --- a/sagemaker-debugger/tensorflow_profiling/demo/generate_large_tfrecords.py +++ b/sagemaker-debugger/tensorflow_profiling/demo/generate_large_tfrecords.py @@ -14,11 +14,11 @@ import argparse import os import sys - import tarfile + +import tensorflow as tf from six.moves import cPickle as pickle from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow as tf # import tensorflow_datasets as tfds diff --git a/sagemaker-debugger/tensorflow_profiling/entry_point/tf-TensorLike.py b/sagemaker-debugger/tensorflow_profiling/entry_point/tf-TensorLike.py index 6a225d51a2..25c4fe3d3d 100644 --- a/sagemaker-debugger/tensorflow_profiling/entry_point/tf-TensorLike.py +++ b/sagemaker-debugger/tensorflow_profiling/entry_point/tf-TensorLike.py @@ -5,14 +5,14 @@ # Third Party import numpy as np import tensorflow.compat.v2 as tf + +# First Party +from smdebug.tensorflow import KerasHook from tensorflow.keras.applications.resnet50 import ResNet50 from tensorflow.keras.datasets import cifar10 from tensorflow.keras.preprocessing.image import ImageDataGenerator from tensorflow.keras.utils import to_categorical -# First Party -from smdebug.tensorflow import KerasHook - def between_steps_bottleneck(): time.sleep(1) diff --git a/sagemaker-debugger/tensorflow_profiling/entry_point/train_tf.py b/sagemaker-debugger/tensorflow_profiling/entry_point/train_tf.py index 41c2bf4ccf..3a40c0b1b0 100644 --- a/sagemaker-debugger/tensorflow_profiling/entry_point/train_tf.py +++ b/sagemaker-debugger/tensorflow_profiling/entry_point/train_tf.py @@ -5,14 +5,14 @@ # Third Party import numpy as np import tensorflow.compat.v2 as tf + +# First Party +from smdebug.tensorflow import KerasHook from tensorflow.keras.applications.resnet50 import ResNet50 from tensorflow.keras.datasets import cifar10 from tensorflow.keras.preprocessing.image import ImageDataGenerator from tensorflow.keras.utils import to_categorical -# First Party -from smdebug.tensorflow import KerasHook - def between_steps_bottleneck(): time.sleep(1) diff --git a/sagemaker-experiments/mnist-handwritten-digits-classification-experiment/mnist.py b/sagemaker-experiments/mnist-handwritten-digits-classification-experiment/mnist.py index 24cdc64188..5cd799b574 100644 --- a/sagemaker-experiments/mnist-handwritten-digits-classification-experiment/mnist.py +++ b/sagemaker-experiments/mnist-handwritten-digits-classification-experiment/mnist.py @@ -2,9 +2,12 @@ import json import logging import os +import sys +import time from os.path import join + +import boto3 import sagemaker_containers -import sys import torch import torch.distributed as dist import torch.nn as nn @@ -14,10 +17,6 @@ import torch.utils.data.distributed from torchvision import datasets, transforms -import boto3 - -import time - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) diff --git a/sagemaker-experiments/track-an-airflow-workflow/code/data_util.py b/sagemaker-experiments/track-an-airflow-workflow/code/data_util.py index 18cb89ef50..cb139d9851 100644 --- a/sagemaker-experiments/track-an-airflow-workflow/code/data_util.py +++ b/sagemaker-experiments/track-an-airflow-workflow/code/data_util.py @@ -11,8 +11,9 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import os import gzip +import os + import numpy as np diff --git a/sagemaker-experiments/track-an-airflow-workflow/code/inference.py b/sagemaker-experiments/track-an-airflow-workflow/code/inference.py index db6a1291f6..ad5dfb8d16 100644 --- a/sagemaker-experiments/track-an-airflow-workflow/code/inference.py +++ b/sagemaker-experiments/track-an-airflow-workflow/code/inference.py @@ -13,6 +13,7 @@ import io import json + import numpy as np diff --git a/sagemaker-experiments/track-an-airflow-workflow/code/train.py b/sagemaker-experiments/track-an-airflow-workflow/code/train.py index 495d7b120d..498eeb53e6 100644 --- a/sagemaker-experiments/track-an-airflow-workflow/code/train.py +++ b/sagemaker-experiments/track-an-airflow-workflow/code/train.py @@ -11,16 +11,16 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import os import argparse import logging +import os + import numpy as np import tensorflow as tf -from data_util import load_train_dataset, load_test_dataset -from sklearn.model_selection import train_test_split -from model import get_model - import tensorflow.keras.backend as K +from data_util import load_test_dataset, load_train_dataset +from model import get_model +from sklearn.model_selection import train_test_split logging.getLogger().setLevel(logging.INFO) tf.logging.set_verbosity(tf.logging.ERROR) diff --git a/sagemaker-fundamentals/create-training-job/container/local_test/test_container.py b/sagemaker-fundamentals/create-training-job/container/local_test/test_container.py index faf0815281..cc5a60a913 100644 --- a/sagemaker-fundamentals/create-training-job/container/local_test/test_container.py +++ b/sagemaker-fundamentals/create-training-job/container/local_test/test_container.py @@ -2,9 +2,10 @@ # on SageMaker infrastructure. It mimics how SageMaker provides # training info to your container and how it executes it. -import docker import os +import docker + dirname = os.path.dirname(os.path.realpath(__file__)) client = docker.from_env() diff --git a/sagemaker-fundamentals/create-training-job/container_hyperparameter_injection/local_test/test_container.py b/sagemaker-fundamentals/create-training-job/container_hyperparameter_injection/local_test/test_container.py index faf0815281..cc5a60a913 100644 --- a/sagemaker-fundamentals/create-training-job/container_hyperparameter_injection/local_test/test_container.py +++ b/sagemaker-fundamentals/create-training-job/container_hyperparameter_injection/local_test/test_container.py @@ -2,9 +2,10 @@ # on SageMaker infrastructure. It mimics how SageMaker provides # training info to your container and how it executes it. -import docker import os +import docker + dirname = os.path.dirname(os.path.realpath(__file__)) client = docker.from_env() diff --git a/sagemaker-fundamentals/create-training-job/container_hyperparameter_injection/train.py b/sagemaker-fundamentals/create-training-job/container_hyperparameter_injection/train.py index 797a0461f0..b0cedf731c 100644 --- a/sagemaker-fundamentals/create-training-job/container_hyperparameter_injection/train.py +++ b/sagemaker-fundamentals/create-training-job/container_hyperparameter_injection/train.py @@ -6,10 +6,9 @@ from __future__ import print_function -import os import json +import os import pickle - import pprint # print out dictionary in a nice way diff --git a/sagemaker-fundamentals/create-training-job/container_kms/local_test/test_container.py b/sagemaker-fundamentals/create-training-job/container_kms/local_test/test_container.py index 41c378b22a..ef6c630c6d 100644 --- a/sagemaker-fundamentals/create-training-job/container_kms/local_test/test_container.py +++ b/sagemaker-fundamentals/create-training-job/container_kms/local_test/test_container.py @@ -2,9 +2,10 @@ # on SageMaker infrastructure. It mimics how SageMaker provides # training info to your container and how it executes it. -import docker import os +import docker + dirname = os.path.dirname(os.path.realpath(__file__)) client = docker.from_env() diff --git a/sagemaker-fundamentals/create-training-job/container_kms/train.py b/sagemaker-fundamentals/create-training-job/container_kms/train.py index 397654bf75..c74bb9613f 100644 --- a/sagemaker-fundamentals/create-training-job/container_kms/train.py +++ b/sagemaker-fundamentals/create-training-job/container_kms/train.py @@ -1,15 +1,14 @@ #!/usr/bin/env python -import boto3 - import base64 -from cryptography.fernet import Fernet -import os import json +import os import pickle - import pprint +import boto3 +from cryptography.fernet import Fernet + # init a boto3 session sess = boto3.Session( aws_access_key_id=os.environ["ACCESS_KEY"], diff --git a/sagemaker-fundamentals/execution-role/iam_helpers.py b/sagemaker-fundamentals/execution-role/iam_helpers.py index edb13287e6..d7d04dc5cd 100644 --- a/sagemaker-fundamentals/execution-role/iam_helpers.py +++ b/sagemaker-fundamentals/execution-role/iam_helpers.py @@ -1,8 +1,9 @@ -import boto3 # your python gate way to all aws services -import pprint # print readable dictionary import json +import pprint # print readable dictionary import time +import boto3 # your python gate way to all aws services + pp = pprint.PrettyPrinter(indent=1) iam = boto3.client("iam") diff --git a/sagemaker-pipelines/tabular/customizing_build_train_deploy_project/modelbuild/pipelines/customer_churn/evaluate.py b/sagemaker-pipelines/tabular/customizing_build_train_deploy_project/modelbuild/pipelines/customer_churn/evaluate.py index 15c1e632c0..f5e57fdc77 100644 --- a/sagemaker-pipelines/tabular/customizing_build_train_deploy_project/modelbuild/pipelines/customer_churn/evaluate.py +++ b/sagemaker-pipelines/tabular/customizing_build_train_deploy_project/modelbuild/pipelines/customer_churn/evaluate.py @@ -13,10 +13,10 @@ """Evaluation script for measuring model accuracy.""" import json -import os -import tarfile import logging +import os import pickle +import tarfile import pandas as pd import xgboost @@ -27,8 +27,7 @@ # May need to import additional metrics depending on what you are measuring. # See https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor-model-quality-metrics.html -from sklearn.metrics import classification_report, roc_auc_score, accuracy_score - +from sklearn.metrics import accuracy_score, classification_report, roc_auc_score if __name__ == "__main__": model_path = "/opt/ml/processing/model/model.tar.gz" diff --git a/sagemaker-pipelines/tabular/customizing_build_train_deploy_project/modelbuild/pipelines/customer_churn/pipeline.py b/sagemaker-pipelines/tabular/customizing_build_train_deploy_project/modelbuild/pipelines/customer_churn/pipeline.py index c7c17a280c..78418d5d07 100644 --- a/sagemaker-pipelines/tabular/customizing_build_train_deploy_project/modelbuild/pipelines/customer_churn/pipeline.py +++ b/sagemaker-pipelines/tabular/customizing_build_train_deploy_project/modelbuild/pipelines/customer_churn/pipeline.py @@ -26,38 +26,18 @@ import boto3 import sagemaker import sagemaker.session - from sagemaker.estimator import Estimator from sagemaker.inputs import TrainingInput -from sagemaker.processing import ( - ProcessingInput, - ProcessingOutput, - ScriptProcessor, -) +from sagemaker.model_metrics import MetricsSource, ModelMetrics +from sagemaker.processing import ProcessingInput, ProcessingOutput, ScriptProcessor from sagemaker.sklearn.processing import SKLearnProcessor -from sagemaker.workflow.conditions import ( - ConditionGreaterThanOrEqualTo, -) -from sagemaker.workflow.condition_step import ( - ConditionStep, - JsonGet, -) -from sagemaker.model_metrics import ( - MetricsSource, - ModelMetrics, -) -from sagemaker.workflow.parameters import ( - ParameterInteger, - ParameterString, -) +from sagemaker.workflow.condition_step import ConditionStep, JsonGet +from sagemaker.workflow.conditions import ConditionGreaterThanOrEqualTo +from sagemaker.workflow.parameters import ParameterInteger, ParameterString from sagemaker.workflow.pipeline import Pipeline from sagemaker.workflow.properties import PropertyFile -from sagemaker.workflow.steps import ( - ProcessingStep, - TrainingStep, -) from sagemaker.workflow.step_collections import RegisterModel - +from sagemaker.workflow.steps import ProcessingStep, TrainingStep BASE_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/sagemaker-python-sdk/chainer_cifar10/s3_util.py b/sagemaker-python-sdk/chainer_cifar10/s3_util.py index c838d84bb6..fc7391694a 100644 --- a/sagemaker-python-sdk/chainer_cifar10/s3_util.py +++ b/sagemaker-python-sdk/chainer_cifar10/s3_util.py @@ -11,10 +11,11 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import boto3 +import os import tarfile from urllib.parse import urlparse -import os + +import boto3 def retrieve_output_from_s3(s3_url, output_dir): diff --git a/sagemaker-python-sdk/chainer_cifar10/src/chainer_cifar_vgg_distributed.py b/sagemaker-python-sdk/chainer_cifar10/src/chainer_cifar_vgg_distributed.py index cb680be39a..09ebde3919 100644 --- a/sagemaker-python-sdk/chainer_cifar10/src/chainer_cifar_vgg_distributed.py +++ b/sagemaker-python-sdk/chainer_cifar10/src/chainer_cifar_vgg_distributed.py @@ -11,24 +11,19 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function import argparse import os -import numpy as np - import chainer import chainer.functions as F import chainer.links as L import chainermn -from chainer import initializers -from chainer import serializers -from chainer import training -from chainer.training import extensions - import net - +import numpy as np +from chainer import initializers, serializers, training +from chainer.training import extensions if __name__ == "__main__": diff --git a/sagemaker-python-sdk/chainer_cifar10/src/chainer_cifar_vgg_single_machine.py b/sagemaker-python-sdk/chainer_cifar10/src/chainer_cifar_vgg_single_machine.py index 42d37677ee..06e0183924 100644 --- a/sagemaker-python-sdk/chainer_cifar10/src/chainer_cifar_vgg_single_machine.py +++ b/sagemaker-python-sdk/chainer_cifar10/src/chainer_cifar_vgg_single_machine.py @@ -11,21 +11,18 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function import argparse import os -import numpy as np - import chainer import chainer.functions as F import chainer.links as L -from chainer import training -from chainer import serializers -from chainer.training import extensions - import net +import numpy as np +from chainer import serializers, training +from chainer.training import extensions if __name__ == "__main__": diff --git a/sagemaker-python-sdk/chainer_mnist/chainer_mnist_single_machine.py b/sagemaker-python-sdk/chainer_mnist/chainer_mnist_single_machine.py index 4fc28a3547..e75c51a8c1 100644 --- a/sagemaker-python-sdk/chainer_mnist/chainer_mnist_single_machine.py +++ b/sagemaker-python-sdk/chainer_mnist/chainer_mnist_single_machine.py @@ -16,13 +16,14 @@ import argparse import os -import numpy as np import chainer import chainer.functions as F import chainer.links as L -from chainer import training, serializers -from chainer.training import extensions +import numpy as np +from chainer import serializers, training from chainer.datasets import tuple_dataset +from chainer.training import extensions + # Define the network to train MNIST class MLP(chainer.Chain): diff --git a/sagemaker-python-sdk/chainer_sentiment_analysis/dataset.py b/sagemaker-python-sdk/chainer_sentiment_analysis/dataset.py index 1015b70035..3c70f2e92f 100644 --- a/sagemaker-python-sdk/chainer_sentiment_analysis/dataset.py +++ b/sagemaker-python-sdk/chainer_sentiment_analysis/dataset.py @@ -19,10 +19,9 @@ import tarfile import tempfile -import numpy import chainer - -from src.nlp_utils import transform_to_array, split_text, normalize_text, make_vocab +import numpy +from src.nlp_utils import make_vocab, normalize_text, split_text, transform_to_array URL_STSA_BASE = "https://raw.githubusercontent.com/harvardnlp/sent-conv-torch/master/data/" diff --git a/sagemaker-python-sdk/chainer_sentiment_analysis/s3_util.py b/sagemaker-python-sdk/chainer_sentiment_analysis/s3_util.py index c838d84bb6..fc7391694a 100644 --- a/sagemaker-python-sdk/chainer_sentiment_analysis/s3_util.py +++ b/sagemaker-python-sdk/chainer_sentiment_analysis/s3_util.py @@ -11,10 +11,11 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. -import boto3 +import os import tarfile from urllib.parse import urlparse -import os + +import boto3 def retrieve_output_from_s3(s3_url, output_dir): diff --git a/sagemaker-python-sdk/chainer_sentiment_analysis/src/nets.py b/sagemaker-python-sdk/chainer_sentiment_analysis/src/nets.py index c3127b9ed3..4d2840ec62 100644 --- a/sagemaker-python-sdk/chainer_sentiment_analysis/src/nets.py +++ b/sagemaker-python-sdk/chainer_sentiment_analysis/src/nets.py @@ -1,8 +1,7 @@ -import numpy - import chainer import chainer.functions as F import chainer.links as L +import numpy from chainer import reporter embed_init = chainer.initializers.Uniform(0.25) diff --git a/sagemaker-python-sdk/chainer_sentiment_analysis/src/nlp_utils.py b/sagemaker-python-sdk/chainer_sentiment_analysis/src/nlp_utils.py index e52f8a749a..d20641d908 100644 --- a/sagemaker-python-sdk/chainer_sentiment_analysis/src/nlp_utils.py +++ b/sagemaker-python-sdk/chainer_sentiment_analysis/src/nlp_utils.py @@ -13,9 +13,8 @@ import collections -import numpy - import chainer +import numpy from chainer.backends import cuda diff --git a/sagemaker-python-sdk/chainer_sentiment_analysis/src/sentiment_analysis.py b/sagemaker-python-sdk/chainer_sentiment_analysis/src/sentiment_analysis.py index 0d1173c1e6..bdc0730219 100644 --- a/sagemaker-python-sdk/chainer_sentiment_analysis/src/sentiment_analysis.py +++ b/sagemaker-python-sdk/chainer_sentiment_analysis/src/sentiment_analysis.py @@ -12,19 +12,16 @@ # language governing permissions and limitations under the License. import argparse -import os import json +import os -import numpy as np -from six import BytesIO import chainer -from chainer import training -from chainer import serializers -from chainer.training import extensions - import nets -from nlp_utils import convert_seq, split_text, normalize_text, transform_to_array - +import numpy as np +from chainer import serializers, training +from chainer.training import extensions +from nlp_utils import convert_seq, normalize_text, split_text, transform_to_array +from six import BytesIO if __name__ == "__main__": diff --git a/sagemaker-python-sdk/dgl_gcmc/data.py b/sagemaker-python-sdk/dgl_gcmc/data.py index 0b9099f730..c91b7dbe2d 100644 --- a/sagemaker-python-sdk/dgl_gcmc/data.py +++ b/sagemaker-python-sdk/dgl_gcmc/data.py @@ -1,13 +1,13 @@ """MovieLens dataset""" -import numpy as np import os import re -import pandas as pd -import scipy.sparse as sp -import gluonnlp as nlp -import mxnet as mx import dgl +import gluonnlp as nlp +import mxnet as mx +import numpy as np +import pandas as pd +import scipy.sparse as sp from dgl.data.utils import download, extract_archive, get_download_dir _urls = { diff --git a/sagemaker-python-sdk/dgl_gcmc/model.py b/sagemaker-python-sdk/dgl_gcmc/model.py index 7c21f01f89..f5454117d1 100644 --- a/sagemaker-python-sdk/dgl_gcmc/model.py +++ b/sagemaker-python-sdk/dgl_gcmc/model.py @@ -1,11 +1,11 @@ """NN modules""" import math -import numpy as np +import dgl.function as fn import mxnet as mx import mxnet.ndarray as F -from mxnet.gluon import nn, Block -import dgl.function as fn +import numpy as np +from mxnet.gluon import Block, nn from utils import get_activation diff --git a/sagemaker-python-sdk/dgl_gcmc/train.py b/sagemaker-python-sdk/dgl_gcmc/train.py index dd56a7bc47..13be9c0804 100644 --- a/sagemaker-python-sdk/dgl_gcmc/train.py +++ b/sagemaker-python-sdk/dgl_gcmc/train.py @@ -1,24 +1,27 @@ """Training script""" -import os, time import argparse +import json import logging +import os import random import string -import json -import numpy as np +import time + import mxnet as mx -from mxnet import gluon +import numpy as np from data import MovieLens -from model import GCMCLayer, BiDecoder +from model import BiDecoder, GCMCLayer +from mxnet import gluon +from mxnet.gluon import Block + from utils import ( + MetricLogger, get_activation, - parse_ctx, gluon_net_info, gluon_total_param_num, params_clip_global_norm, - MetricLogger, + parse_ctx, ) -from mxnet.gluon import Block class Net(Block): diff --git a/sagemaker-python-sdk/dgl_gcmc/utils.py b/sagemaker-python-sdk/dgl_gcmc/utils.py index 07b853a31e..ae5930dc58 100644 --- a/sagemaker-python-sdk/dgl_gcmc/utils.py +++ b/sagemaker-python-sdk/dgl_gcmc/utils.py @@ -1,15 +1,16 @@ import ast -import os import csv import inspect import logging +import os import re +from collections import OrderedDict + +import mxnet as mx import mxnet.ndarray as nd +import numpy as np from mxnet import gluon from mxnet.gluon import nn -import mxnet as mx -import numpy as np -from collections import OrderedDict class MetricLogger(object): diff --git a/sagemaker-python-sdk/dgl_gcn/mxnet_gcn.py b/sagemaker-python-sdk/dgl_gcn/mxnet_gcn.py index f4a2ebbc92..02d5c17723 100644 --- a/sagemaker-python-sdk/dgl_gcn/mxnet_gcn.py +++ b/sagemaker-python-sdk/dgl_gcn/mxnet_gcn.py @@ -7,22 +7,19 @@ - Paper: https://arxiv.org/abs/1609.02907 - Code: https://github.com/tkipf/gcn """ -import mxnet as mx -from mxnet import gluon -import os import argparse -import dgl -from dgl.nn.mxnet import GraphConv - -import time +import collections import json -import numpy as np -from mxnet import gluon +import os +import time +import dgl +import mxnet as mx +import numpy as np from dgl import DGLGraph -from dgl.data import register_data_args, load_data - -import collections +from dgl.data import load_data, register_data_args +from dgl.nn.mxnet import GraphConv +from mxnet import gluon class GCN(gluon.Block): diff --git a/sagemaker-python-sdk/dgl_gcn/pytorch_gcn.py b/sagemaker-python-sdk/dgl_gcn/pytorch_gcn.py index fd295748c0..54165879ba 100644 --- a/sagemaker-python-sdk/dgl_gcn/pytorch_gcn.py +++ b/sagemaker-python-sdk/dgl_gcn/pytorch_gcn.py @@ -1,15 +1,16 @@ -import torch -import torch.nn as nn -from dgl.nn.pytorch import GraphConv - +import argparse +import json import os import time -import json -import argparse + import numpy as np +import torch +import torch.nn as nn import torch.nn.functional as F from dgl import DGLGraph -from dgl.data import register_data_args, load_data +from dgl.data import load_data, register_data_args +from dgl.nn.pytorch import GraphConv + # define GCN layer class GCN(nn.Module): diff --git a/sagemaker-python-sdk/dgl_gcn_tox21/main.py b/sagemaker-python-sdk/dgl_gcn_tox21/main.py index dbede49cd4..db0aeaeba5 100644 --- a/sagemaker-python-sdk/dgl_gcn_tox21/main.py +++ b/sagemaker-python-sdk/dgl_gcn_tox21/main.py @@ -1,12 +1,12 @@ import argparse -import dgl import json -import numpy as np import os import random -import torch - from datetime import datetime + +import dgl +import numpy as np +import torch from dgl import model_zoo from dgl.data.chem import Tox21 from dgl.data.utils import split_dataset diff --git a/sagemaker-python-sdk/dgl_kge/dataloader/KGDataset.py b/sagemaker-python-sdk/dgl_kge/dataloader/KGDataset.py index 7faa8fa411..595f6d23be 100644 --- a/sagemaker-python-sdk/dgl_kge/dataloader/KGDataset.py +++ b/sagemaker-python-sdk/dgl_kge/dataloader/KGDataset.py @@ -2,9 +2,11 @@ def _download_and_extract(url, path, filename): - import shutil, zipfile - from tqdm import tqdm + import shutil + import zipfile + import requests + from tqdm import tqdm fn = os.path.join(path, filename) diff --git a/sagemaker-python-sdk/dgl_kge/dataloader/sampler.py b/sagemaker-python-sdk/dgl_kge/dataloader/sampler.py index e6cb4de663..a0bafa697c 100644 --- a/sagemaker-python-sdk/dgl_kge/dataloader/sampler.py +++ b/sagemaker-python-sdk/dgl_kge/dataloader/sampler.py @@ -1,12 +1,14 @@ import math -import numpy as np -import scipy as sp -import dgl.backend as F -import dgl import os import pickle import time +import dgl +import dgl.backend as F +import numpy as np +import scipy as sp + + # This partitions a list of edges based on relations to make sure # each partition has roughly the same number of edges and relations. def RelationPartition(edges, n): diff --git a/sagemaker-python-sdk/dgl_kge/eval.py b/sagemaker-python-sdk/dgl_kge/eval.py index df66d9a6a6..6a54062e52 100644 --- a/sagemaker-python-sdk/dgl_kge/eval.py +++ b/sagemaker-python-sdk/dgl_kge/eval.py @@ -1,20 +1,17 @@ -from dataloader import EvalDataset, TrainDataset -from dataloader import get_dataset - import argparse -import torch.multiprocessing as mp -import os import logging -import time +import os import pickle +import time + +import torch.multiprocessing as mp +from dataloader import EvalDataset, TrainDataset, get_dataset backend = os.environ.get("DGLBACKEND") if backend.lower() == "mxnet": - from train_mxnet import load_model_from_checkpoint - from train_mxnet import test + from train_mxnet import load_model_from_checkpoint, test else: - from train_pytorch import load_model_from_checkpoint - from train_pytorch import test + from train_pytorch import load_model_from_checkpoint, test class ArgParser(argparse.ArgumentParser): diff --git a/sagemaker-python-sdk/dgl_kge/models/general_models.py b/sagemaker-python-sdk/dgl_kge/models/general_models.py index 5f77c40be1..ceb500b5f4 100644 --- a/sagemaker-python-sdk/dgl_kge/models/general_models.py +++ b/sagemaker-python-sdk/dgl_kge/models/general_models.py @@ -1,26 +1,31 @@ import os -import numpy as np + import dgl.backend as F +import numpy as np backend = os.environ.get("DGLBACKEND") if backend and backend.lower() == "mxnet": - from .mxnet.tensor_models import logsigmoid - from .mxnet.tensor_models import get_device - from .mxnet.tensor_models import norm - from .mxnet.tensor_models import get_scalar - from .mxnet.tensor_models import reshape - from .mxnet.tensor_models import cuda - from .mxnet.tensor_models import ExternalEmbedding from .mxnet.score_fun import * + from .mxnet.tensor_models import ( + ExternalEmbedding, + cuda, + get_device, + get_scalar, + logsigmoid, + norm, + reshape, + ) else: - from .pytorch.tensor_models import logsigmoid - from .pytorch.tensor_models import get_device - from .pytorch.tensor_models import norm - from .pytorch.tensor_models import get_scalar - from .pytorch.tensor_models import reshape - from .pytorch.tensor_models import cuda - from .pytorch.tensor_models import ExternalEmbedding from .pytorch.score_fun import * + from .pytorch.tensor_models import ( + ExternalEmbedding, + cuda, + get_device, + get_scalar, + logsigmoid, + norm, + reshape, + ) class KEModel(object): diff --git a/sagemaker-python-sdk/dgl_kge/models/mxnet/score_fun.py b/sagemaker-python-sdk/dgl_kge/models/mxnet/score_fun.py index 193129e8e8..6830b6f035 100644 --- a/sagemaker-python-sdk/dgl_kge/models/mxnet/score_fun.py +++ b/sagemaker-python-sdk/dgl_kge/models/mxnet/score_fun.py @@ -1,8 +1,8 @@ -import numpy as np import mxnet as mx +import numpy as np from mxnet import gluon -from mxnet.gluon import nn from mxnet import ndarray as nd +from mxnet.gluon import nn class TransEScore(nn.Block): diff --git a/sagemaker-python-sdk/dgl_kge/models/mxnet/tensor_models.py b/sagemaker-python-sdk/dgl_kge/models/mxnet/tensor_models.py index c3683c60e6..1cccb89569 100644 --- a/sagemaker-python-sdk/dgl_kge/models/mxnet/tensor_models.py +++ b/sagemaker-python-sdk/dgl_kge/models/mxnet/tensor_models.py @@ -1,11 +1,12 @@ import os -import numpy as np + import mxnet as mx +import numpy as np from mxnet import gluon from mxnet import ndarray as nd -from .score_fun import * from .. import * +from .score_fun import * def logsigmoid(val): diff --git a/sagemaker-python-sdk/dgl_kge/models/pytorch/score_fun.py b/sagemaker-python-sdk/dgl_kge/models/pytorch/score_fun.py index 63af196f3c..94d53004a2 100644 --- a/sagemaker-python-sdk/dgl_kge/models/pytorch/score_fun.py +++ b/sagemaker-python-sdk/dgl_kge/models/pytorch/score_fun.py @@ -1,8 +1,8 @@ +import numpy as np import torch as th import torch.nn as nn import torch.nn.functional as functional import torch.nn.init as INIT -import numpy as np class TransEScore(nn.Module): diff --git a/sagemaker-python-sdk/dgl_kge/models/pytorch/tensor_models.py b/sagemaker-python-sdk/dgl_kge/models/pytorch/tensor_models.py index 0486e58bda..920631702e 100644 --- a/sagemaker-python-sdk/dgl_kge/models/pytorch/tensor_models.py +++ b/sagemaker-python-sdk/dgl_kge/models/pytorch/tensor_models.py @@ -11,8 +11,8 @@ 9. RESCAL """ import os -import numpy as np +import numpy as np import torch as th import torch.nn as nn import torch.nn.functional as functional diff --git a/sagemaker-python-sdk/dgl_kge/train.py b/sagemaker-python-sdk/dgl_kge/train.py index 6441a98e10..4df41ec5ab 100644 --- a/sagemaker-python-sdk/dgl_kge/train.py +++ b/sagemaker-python-sdk/dgl_kge/train.py @@ -1,22 +1,18 @@ -from dataloader import EvalDataset, TrainDataset, NewBidirectionalOneShotIterator -from dataloader import get_dataset - import argparse -import os import logging +import os import time +from dataloader import EvalDataset, NewBidirectionalOneShotIterator, TrainDataset, get_dataset + backend = os.environ.get("DGLBACKEND") if backend and backend.lower() == "mxnet": import multiprocessing as mp - from train_mxnet import load_model - from train_mxnet import train - from train_mxnet import test + + from train_mxnet import load_model, test, train else: import torch.multiprocessing as mp - from train_pytorch import load_model - from train_pytorch import train - from train_pytorch import test + from train_pytorch import load_model, test, train class ArgParser(argparse.ArgumentParser): diff --git a/sagemaker-python-sdk/dgl_kge/train_mxnet.py b/sagemaker-python-sdk/dgl_kge/train_mxnet.py index 94d32c87bb..9fef0153b9 100644 --- a/sagemaker-python-sdk/dgl_kge/train_mxnet.py +++ b/sagemaker-python-sdk/dgl_kge/train_mxnet.py @@ -1,14 +1,13 @@ -from models import KEModel +import json +import logging +import os +import time import mxnet as mx +from models import KEModel from mxnet import gluon from mxnet import ndarray as nd -import os -import logging -import time -import json - def load_model(logger, args, n_entities, n_relations, ckpt=None): model = KEModel( diff --git a/sagemaker-python-sdk/dgl_kge/train_pytorch.py b/sagemaker-python-sdk/dgl_kge/train_pytorch.py index b9be436d05..bf5d65aba1 100644 --- a/sagemaker-python-sdk/dgl_kge/train_pytorch.py +++ b/sagemaker-python-sdk/dgl_kge/train_pytorch.py @@ -1,18 +1,17 @@ -from models import KEModel +from distutils.version import LooseVersion -from torch.utils.data import DataLoader -import torch.optim as optim import torch as th import torch.multiprocessing as mp - -from distutils.version import LooseVersion +import torch.optim as optim +from models import KEModel +from torch.utils.data import DataLoader TH_VERSION = LooseVersion(th.__version__) if TH_VERSION.version[0] == 1 and TH_VERSION.version[1] < 2: raise Exception("DGL-ke has to work with Pytorch version >= 1.2") -import os import logging +import os import time diff --git a/sagemaker-python-sdk/keras_script_mode_pipe_mode_horovod/generate_tensorboard_command.py b/sagemaker-python-sdk/keras_script_mode_pipe_mode_horovod/generate_tensorboard_command.py index aee5eadab6..018426dc89 100644 --- a/sagemaker-python-sdk/keras_script_mode_pipe_mode_horovod/generate_tensorboard_command.py +++ b/sagemaker-python-sdk/keras_script_mode_pipe_mode_horovod/generate_tensorboard_command.py @@ -10,9 +10,10 @@ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either # express or implied. See the License for the specific language governing # permissions and limitations under the License. -import boto3 -from datetime import datetime, timedelta import re +from datetime import datetime, timedelta + +import boto3 client = boto3.client("sagemaker") running_jobs = client.list_training_jobs(CreationTimeAfter=datetime.utcnow() - timedelta(hours=1)) diff --git a/sagemaker-python-sdk/keras_script_mode_pipe_mode_horovod/source_dir/cifar10_keras_main.py b/sagemaker-python-sdk/keras_script_mode_pipe_mode_horovod/source_dir/cifar10_keras_main.py index 97f18be795..4b42f67caf 100644 --- a/sagemaker-python-sdk/keras_script_mode_pipe_mode_horovod/source_dir/cifar10_keras_main.py +++ b/sagemaker-python-sdk/keras_script_mode_pipe_mode_horovod/source_dir/cifar10_keras_main.py @@ -15,24 +15,24 @@ import argparse import json import logging -import re import os +import re import keras import tensorflow as tf from keras import backend as K -from keras.callbacks import TensorBoard, ModelCheckpoint +from keras.callbacks import ModelCheckpoint, TensorBoard from keras.layers import ( Activation, + BatchNormalization, Conv2D, Dense, Dropout, Flatten, MaxPooling2D, - BatchNormalization, ) from keras.models import Sequential -from keras.optimizers import Adam, SGD, RMSprop +from keras.optimizers import SGD, Adam, RMSprop logging.getLogger().setLevel(logging.INFO) tf.logging.set_verbosity(tf.logging.INFO) diff --git a/sagemaker-python-sdk/managed_spot_training_mxnet/mnist.py b/sagemaker-python-sdk/managed_spot_training_mxnet/mnist.py index 2b2580745b..6ba22a38fd 100644 --- a/sagemaker-python-sdk/managed_spot_training_mxnet/mnist.py +++ b/sagemaker-python-sdk/managed_spot_training_mxnet/mnist.py @@ -133,9 +133,10 @@ def parse_args(): ### NOTE: this function cannot use MXNet def neo_preprocess(payload, content_type): + import io import logging + import numpy as np - import io logging.info("Invoking user-defined pre-processing function") @@ -148,9 +149,10 @@ def neo_preprocess(payload, content_type): ### NOTE: this function cannot use MXNet def neo_postprocess(result): + import json import logging + import numpy as np - import json logging.info("Invoking user-defined post-processing function") diff --git a/sagemaker-python-sdk/managed_spot_training_tensorflow_estimator/iris_dnn_classifier.py b/sagemaker-python-sdk/managed_spot_training_tensorflow_estimator/iris_dnn_classifier.py index b15f5a46e5..423fdd0ada 100644 --- a/sagemaker-python-sdk/managed_spot_training_tensorflow_estimator/iris_dnn_classifier.py +++ b/sagemaker-python-sdk/managed_spot_training_tensorflow_estimator/iris_dnn_classifier.py @@ -1,5 +1,6 @@ -import numpy as np import os + +import numpy as np import tensorflow as tf INPUT_TENSOR_NAME = "inputs" diff --git a/sagemaker-python-sdk/mxnet_gluon_mnist/mnist.py b/sagemaker-python-sdk/mxnet_gluon_mnist/mnist.py index 38a035737d..5e8730ba18 100644 --- a/sagemaker-python-sdk/mxnet_gluon_mnist/mnist.py +++ b/sagemaker-python-sdk/mxnet_gluon_mnist/mnist.py @@ -1,16 +1,15 @@ from __future__ import print_function import argparse +import json import logging import os +import time import mxnet as mx -from mxnet import gluon, autograd -from mxnet.gluon import nn import numpy as np -import json -import time - +from mxnet import autograd, gluon +from mxnet.gluon import nn logging.basicConfig(level=logging.DEBUG) diff --git a/sagemaker-python-sdk/mxnet_gluon_sentiment/sentiment.py b/sagemaker-python-sdk/mxnet_gluon_sentiment/sentiment.py index 686f312bd8..01de986498 100644 --- a/sagemaker-python-sdk/mxnet_gluon_sentiment/sentiment.py +++ b/sagemaker-python-sdk/mxnet_gluon_sentiment/sentiment.py @@ -12,8 +12,8 @@ import mxnet as mx import numpy as np -from mxnet import gluon, autograd, nd -from mxnet.io import DataIter, DataBatch, DataDesc +from mxnet import autograd, gluon, nd +from mxnet.io import DataBatch, DataDesc, DataIter logging.basicConfig(level=logging.DEBUG) diff --git a/sagemaker-python-sdk/mxnet_horovod_fasterrcnn/source/train_faster_rcnn.py b/sagemaker-python-sdk/mxnet_horovod_fasterrcnn/source/train_faster_rcnn.py index 0bf105a8f5..c33456e18f 100644 --- a/sagemaker-python-sdk/mxnet_horovod_fasterrcnn/source/train_faster_rcnn.py +++ b/sagemaker-python-sdk/mxnet_horovod_fasterrcnn/source/train_faster_rcnn.py @@ -42,31 +42,32 @@ import logging import time -import numpy as np + +import gluoncv as gcv import mxnet as mx +import numpy as np from mxnet import gluon from mxnet.contrib import amp -import gluoncv as gcv gcv.utils.check_version("0.7.0") from gluoncv import data as gdata from gluoncv import utils as gutils -from gluoncv.model_zoo import get_model -from gluoncv.data.batchify import FasterRCNNTrainBatchify, Tuple, Append +from gluoncv.data.batchify import Append, FasterRCNNTrainBatchify, Tuple from gluoncv.data.transforms.presets.rcnn import ( FasterRCNNDefaultTrainTransform, FasterRCNNDefaultValTransform, ) -from gluoncv.utils.metrics.voc_detection import VOC07MApMetric +from gluoncv.model_zoo import get_model +from gluoncv.model_zoo.rcnn.faster_rcnn.data_parallel import ForwardBackwardTask from gluoncv.utils.metrics.coco_detection import COCODetectionMetric -from gluoncv.utils.parallel import Parallel from gluoncv.utils.metrics.rcnn import ( - RPNAccMetric, - RPNL1LossMetric, RCNNAccMetric, RCNNL1LossMetric, + RPNAccMetric, + RPNL1LossMetric, ) -from gluoncv.model_zoo.rcnn.faster_rcnn.data_parallel import ForwardBackwardTask +from gluoncv.utils.metrics.voc_detection import VOC07MApMetric +from gluoncv.utils.parallel import Parallel try: import horovod.mxnet as hvd diff --git a/sagemaker-python-sdk/mxnet_horovod_maskrcnn/source/train_mask_rcnn.py b/sagemaker-python-sdk/mxnet_horovod_maskrcnn/source/train_mask_rcnn.py index 71350b845d..7acce8475f 100644 --- a/sagemaker-python-sdk/mxnet_horovod_maskrcnn/source/train_mask_rcnn.py +++ b/sagemaker-python-sdk/mxnet_horovod_maskrcnn/source/train_mask_rcnn.py @@ -40,34 +40,35 @@ import logging import time -import numpy as np + +import gluoncv as gcv import mxnet as mx +import numpy as np from mxnet import gluon from mxnet.contrib import amp -import gluoncv as gcv gcv.utils.check_version("0.7.0") +from multiprocessing import Process + from gluoncv import data as gdata from gluoncv import utils as gutils -from gluoncv.model_zoo import get_model -from gluoncv.data import batchify +from gluoncv.data import COCODetection, VOCDetection, batchify from gluoncv.data.transforms.presets.rcnn import ( MaskRCNNDefaultTrainTransform, MaskRCNNDefaultValTransform, ) +from gluoncv.model_zoo import get_model +from gluoncv.model_zoo.rcnn.mask_rcnn.data_parallel import ForwardBackwardTask from gluoncv.utils.metrics.coco_instance import COCOInstanceMetric from gluoncv.utils.metrics.rcnn import ( - RPNAccMetric, - RPNL1LossMetric, - RCNNAccMetric, - RCNNL1LossMetric, MaskAccMetric, MaskFGAccMetric, + RCNNAccMetric, + RCNNL1LossMetric, + RPNAccMetric, + RPNL1LossMetric, ) from gluoncv.utils.parallel import Parallel -from gluoncv.data import COCODetection, VOCDetection -from multiprocessing import Process -from gluoncv.model_zoo.rcnn.mask_rcnn.data_parallel import ForwardBackwardTask try: import horovod.mxnet as hvd diff --git a/sagemaker-python-sdk/mxnet_horovod_mnist/code/horovod_mnist.py b/sagemaker-python-sdk/mxnet_horovod_mnist/code/horovod_mnist.py index d7e55c289e..0df69e19fa 100644 --- a/sagemaker-python-sdk/mxnet_horovod_mnist/code/horovod_mnist.py +++ b/sagemaker-python-sdk/mxnet_horovod_mnist/code/horovod_mnist.py @@ -13,11 +13,11 @@ import argparse import logging import os -import zipfile import time +import zipfile -import mxnet as mx import horovod.mxnet as hvd +import mxnet as mx from mxnet import autograd, gluon, nd from mxnet.test_utils import download diff --git a/sagemaker-python-sdk/mxnet_horovod_mnist/code/test_train.py b/sagemaker-python-sdk/mxnet_horovod_mnist/code/test_train.py index 467adaad84..bb86fa408b 100644 --- a/sagemaker-python-sdk/mxnet_horovod_mnist/code/test_train.py +++ b/sagemaker-python-sdk/mxnet_horovod_mnist/code/test_train.py @@ -1,9 +1,9 @@ -from train import train, parse_args - -import sys +import json import os +import sys + import boto3 -import json +from train import parse_args, train dirname = os.path.dirname(os.path.abspath(__file__)) diff --git a/sagemaker-python-sdk/mxnet_mnist/mnist.py b/sagemaker-python-sdk/mxnet_mnist/mnist.py index d09aed57d8..3e27e425c5 100644 --- a/sagemaker-python-sdk/mxnet_mnist/mnist.py +++ b/sagemaker-python-sdk/mxnet_mnist/mnist.py @@ -134,9 +134,10 @@ def parse_args(): ### NOTE: this function cannot use MXNet def neo_preprocess(payload, content_type): + import io import logging + import numpy as np - import io logging.info("Invoking user-defined pre-processing function") @@ -149,9 +150,10 @@ def neo_preprocess(payload, content_type): ### NOTE: this function cannot use MXNet def neo_postprocess(result): + import json import logging + import numpy as np - import json logging.info("Invoking user-defined post-processing function") diff --git a/sagemaker-python-sdk/mxnet_mnist/mnist_ei.py b/sagemaker-python-sdk/mxnet_mnist/mnist_ei.py index 1fe88abfa2..61a13f5367 100644 --- a/sagemaker-python-sdk/mxnet_mnist/mnist_ei.py +++ b/sagemaker-python-sdk/mxnet_mnist/mnist_ei.py @@ -171,9 +171,10 @@ def parse_args(): ### NOTE: this function cannot use MXNet def neo_preprocess(payload, content_type): + import io import logging + import numpy as np - import io logging.info("Invoking user-defined pre-processing function") @@ -186,9 +187,10 @@ def neo_preprocess(payload, content_type): ### NOTE: this function cannot use MXNet def neo_postprocess(result): + import json import logging + import numpy as np - import json logging.info("Invoking user-defined post-processing function") diff --git a/sagemaker-python-sdk/mxnet_onnx_eia/resnet152.py b/sagemaker-python-sdk/mxnet_onnx_eia/resnet152.py index d5f35c17c8..2faafae329 100644 --- a/sagemaker-python-sdk/mxnet_onnx_eia/resnet152.py +++ b/sagemaker-python-sdk/mxnet_onnx_eia/resnet152.py @@ -2,6 +2,7 @@ import json import logging + import mxnet as mx import mxnet.contrib.onnx as onnx_mxnet import numpy as np diff --git a/sagemaker-python-sdk/mxnet_onnx_export/mnist.py b/sagemaker-python-sdk/mxnet_onnx_export/mnist.py index 2f2a851d81..084942bdaa 100644 --- a/sagemaker-python-sdk/mxnet_onnx_export/mnist.py +++ b/sagemaker-python-sdk/mxnet_onnx_export/mnist.py @@ -3,14 +3,13 @@ import json import logging import os -import tempfile import shutil import struct +import tempfile import mxnet as mx -from mxnet.contrib import onnx as onnx_mxnet import numpy as np - +from mxnet.contrib import onnx as onnx_mxnet from sagemaker_mxnet_container.training_utils import scheduler_host diff --git a/sagemaker-python-sdk/mxnet_onnx_superresolution/super_resolution.py b/sagemaker-python-sdk/mxnet_onnx_superresolution/super_resolution.py index 2bb361ae25..e814fc827d 100644 --- a/sagemaker-python-sdk/mxnet_onnx_superresolution/super_resolution.py +++ b/sagemaker-python-sdk/mxnet_onnx_superresolution/super_resolution.py @@ -3,18 +3,18 @@ import bisect import json import logging -import time import random import re +import time from collections import Counter, namedtuple from itertools import chain, islice import mxnet as mx import mxnet.contrib.onnx as onnx_mxnet import numpy as np -from mxnet import gluon, autograd, nd -from mxnet.io import DataIter, DataBatch, DataDesc +from mxnet import autograd, gluon, nd from mxnet.gluon import nn +from mxnet.io import DataBatch, DataDesc, DataIter logging.basicConfig(level=logging.DEBUG) diff --git a/sagemaker-python-sdk/pytorch_cnn_cifar10/source/cifar10.py b/sagemaker-python-sdk/pytorch_cnn_cifar10/source/cifar10.py index 171c970d40..b6ebd5a295 100644 --- a/sagemaker-python-sdk/pytorch_cnn_cifar10/source/cifar10.py +++ b/sagemaker-python-sdk/pytorch_cnn_cifar10/source/cifar10.py @@ -1,11 +1,11 @@ import argparse import logging - import os import torch import torch.distributed as dist import torch.nn as nn +import torch.nn.functional as F import torch.nn.parallel import torch.optim import torch.utils.data @@ -13,7 +13,6 @@ import torchvision import torchvision.models import torchvision.transforms as transforms -import torch.nn.functional as F try: from sagemaker_inference import environment diff --git a/sagemaker-python-sdk/pytorch_cnn_cifar10/utils_cifar.py b/sagemaker-python-sdk/pytorch_cnn_cifar10/utils_cifar.py index db0e909f43..8afb272d34 100644 --- a/sagemaker-python-sdk/pytorch_cnn_cifar10/utils_cifar.py +++ b/sagemaker-python-sdk/pytorch_cnn_cifar10/utils_cifar.py @@ -1,8 +1,8 @@ +import matplotlib.pyplot as plt +import numpy as np import torch import torchvision import torchvision.transforms as transforms -import matplotlib.pyplot as plt -import numpy as np classes = ("plane", "car", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck") diff --git a/sagemaker-python-sdk/pytorch_horovod_mnist/code/inference.py b/sagemaker-python-sdk/pytorch_horovod_mnist/code/inference.py index 879923c71f..29204784b6 100644 --- a/sagemaker-python-sdk/pytorch_horovod_mnist/code/inference.py +++ b/sagemaker-python-sdk/pytorch_horovod_mnist/code/inference.py @@ -1,7 +1,8 @@ from __future__ import print_function + import os -import torch +import torch from model_def import Net diff --git a/sagemaker-python-sdk/pytorch_horovod_mnist/code/mnist.py b/sagemaker-python-sdk/pytorch_horovod_mnist/code/mnist.py index 84f5ba4d7a..d7d3f474ce 100644 --- a/sagemaker-python-sdk/pytorch_horovod_mnist/code/mnist.py +++ b/sagemaker-python-sdk/pytorch_horovod_mnist/code/mnist.py @@ -1,16 +1,17 @@ from __future__ import print_function + +import argparse import json import logging import os import sys -import argparse + import horovod.torch as hvd import torch.nn.functional as F import torch.optim as optim import torch.utils.data.distributed - -from torchvision import datasets, transforms from model_def import Net +from torchvision import datasets, transforms logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/sagemaker-python-sdk/pytorch_horovod_mnist/code/model_def.py b/sagemaker-python-sdk/pytorch_horovod_mnist/code/model_def.py index b148f5ecb4..3c4e1aa2cf 100644 --- a/sagemaker-python-sdk/pytorch_horovod_mnist/code/model_def.py +++ b/sagemaker-python-sdk/pytorch_horovod_mnist/code/model_def.py @@ -1,4 +1,5 @@ from __future__ import print_function + import torch.nn as nn import torch.nn.functional as F diff --git a/sagemaker-python-sdk/pytorch_mnist/code/deploy_ei.py b/sagemaker-python-sdk/pytorch_mnist/code/deploy_ei.py index abdf3ab712..968c66592a 100644 --- a/sagemaker-python-sdk/pytorch_mnist/code/deploy_ei.py +++ b/sagemaker-python-sdk/pytorch_mnist/code/deploy_ei.py @@ -11,6 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from __future__ import absolute_import + import logging import os import sys diff --git a/sagemaker-python-sdk/pytorch_mnist/mnist.py b/sagemaker-python-sdk/pytorch_mnist/mnist.py index 461162f054..d1c4f3b369 100644 --- a/sagemaker-python-sdk/pytorch_mnist/mnist.py +++ b/sagemaker-python-sdk/pytorch_mnist/mnist.py @@ -2,8 +2,9 @@ import json import logging import os -import sagemaker_containers import sys + +import sagemaker_containers import torch import torch.distributed as dist import torch.nn as nn diff --git a/sagemaker-python-sdk/scikit_learn_inference_pipeline/sklearn_abalone_featurizer.py b/sagemaker-python-sdk/scikit_learn_inference_pipeline/sklearn_abalone_featurizer.py index 11caa9deba..a749a5a481 100644 --- a/sagemaker-python-sdk/scikit_learn_inference_pipeline/sklearn_abalone_featurizer.py +++ b/sagemaker-python-sdk/scikit_learn_inference_pipeline/sklearn_abalone_featurizer.py @@ -1,23 +1,17 @@ from __future__ import print_function -import time -import sys -from io import StringIO -import os -import shutil - import argparse import csv import json +import os +import shutil +import sys +import time +from io import StringIO + import joblib import numpy as np import pandas as pd - -from sklearn.compose import ColumnTransformer, make_column_selector -from sklearn.impute import SimpleImputer -from sklearn.pipeline import make_pipeline -from sklearn.preprocessing import Binarizer, StandardScaler, OneHotEncoder - from sagemaker_containers.beta.framework import ( content_types, encoders, @@ -26,6 +20,10 @@ transformer, worker, ) +from sklearn.compose import ColumnTransformer, make_column_selector +from sklearn.impute import SimpleImputer +from sklearn.pipeline import make_pipeline +from sklearn.preprocessing import Binarizer, OneHotEncoder, StandardScaler # Since we get a headerless CSV file we specify the column names here. feature_columns_names = [ diff --git a/sagemaker-python-sdk/scikit_learn_iris/scikit_learn_iris.py b/sagemaker-python-sdk/scikit_learn_iris/scikit_learn_iris.py index 21e55d86fd..b6246fba9f 100644 --- a/sagemaker-python-sdk/scikit_learn_iris/scikit_learn_iris.py +++ b/sagemaker-python-sdk/scikit_learn_iris/scikit_learn_iris.py @@ -14,13 +14,12 @@ from __future__ import print_function import argparse -import joblib import os -import pandas as pd +import joblib +import pandas as pd from sklearn import tree - if __name__ == "__main__": parser = argparse.ArgumentParser() diff --git a/sagemaker-python-sdk/tensorboard_keras/source_dir/tensorboard_keras_cifar10.py b/sagemaker-python-sdk/tensorboard_keras/source_dir/tensorboard_keras_cifar10.py index dc9ddd6142..994eb3dd9c 100644 --- a/sagemaker-python-sdk/tensorboard_keras/source_dir/tensorboard_keras_cifar10.py +++ b/sagemaker-python-sdk/tensorboard_keras/source_dir/tensorboard_keras_cifar10.py @@ -12,33 +12,33 @@ # permissions and limitations under the License. import argparse +import io +import itertools import json import logging import os import re +import matplotlib.pyplot as plt +import numpy as np +import sklearn.metrics import tensorflow as tf +from tensorboard.plugins.hparams import api as hp from tensorflow import keras from tensorflow.keras import backend as K +from tensorflow.keras.callbacks import TensorBoard +from tensorflow.keras.constraints import max_norm from tensorflow.keras.layers import ( Activation, + BatchNormalization, Conv2D, Dense, Dropout, Flatten, MaxPooling2D, - BatchNormalization, ) from tensorflow.keras.models import Sequential -from tensorflow.keras.optimizers import Adam, SGD, RMSprop -from tensorflow.keras.constraints import max_norm -from tensorflow.keras.callbacks import TensorBoard - -import matplotlib.pyplot as plt -import numpy as np -import itertools, io -import sklearn.metrics -from tensorboard.plugins.hparams import api as hp +from tensorflow.keras.optimizers import SGD, Adam, RMSprop logging.getLogger().setLevel(logging.INFO) tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO) diff --git a/sagemaker-python-sdk/tensorflow-eager-script-mode/train_model/train.py b/sagemaker-python-sdk/tensorflow-eager-script-mode/train_model/train.py index a99d408608..49dbe03a2e 100644 --- a/sagemaker-python-sdk/tensorflow-eager-script-mode/train_model/train.py +++ b/sagemaker-python-sdk/tensorflow-eager-script-mode/train_model/train.py @@ -1,11 +1,10 @@ import argparse -import numpy as np import os -import tensorflow as tf -from tensorflow.contrib.eager.python import tfe +import numpy as np +import tensorflow as tf from model_def import get_model - +from tensorflow.contrib.eager.python import tfe tf.enable_eager_execution() tf.set_random_seed(0) diff --git a/sagemaker-python-sdk/tensorflow_moving_from_framework_mode_to_script_mode/mnist.py b/sagemaker-python-sdk/tensorflow_moving_from_framework_mode_to_script_mode/mnist.py index ed45ebca9e..39c95038ef 100644 --- a/sagemaker-python-sdk/tensorflow_moving_from_framework_mode_to_script_mode/mnist.py +++ b/sagemaker-python-sdk/tensorflow_moving_from_framework_mode_to_script_mode/mnist.py @@ -1,4 +1,5 @@ import os + import tensorflow as tf from tensorflow.python.estimator.model_fn import ModeKeys as Modes @@ -110,10 +111,11 @@ def _input_fn(training_dir, training_filename, batch_size=100): def neo_preprocess(payload, content_type): + import io import logging + import numpy as np import PIL.Image # Training container doesn't have this package - import io logging.info("Invoking user-defined pre-processing function") @@ -133,9 +135,10 @@ def neo_preprocess(payload, content_type): ### NOTE: this function cannot use MXNet def neo_postprocess(result): + import json import logging + import numpy as np - import json logging.info("Invoking user-defined post-processing function") diff --git a/sagemaker-python-sdk/tensorflow_moving_from_framework_mode_to_script_mode/utils.py b/sagemaker-python-sdk/tensorflow_moving_from_framework_mode_to_script_mode/utils.py index cf8af1b2b1..c883d56685 100644 --- a/sagemaker-python-sdk/tensorflow_moving_from_framework_mode_to_script_mode/utils.py +++ b/sagemaker-python-sdk/tensorflow_moving_from_framework_mode_to_script_mode/utils.py @@ -1,5 +1,6 @@ """Converts MNIST data to TFRecords file format with Example protos.""" import os + import tensorflow as tf diff --git a/sagemaker-python-sdk/tensorflow_script_mode_horovod/mnist_hvd.py b/sagemaker-python-sdk/tensorflow_script_mode_horovod/mnist_hvd.py index 675e6a46ff..89ad734311 100644 --- a/sagemaker-python-sdk/tensorflow_script_mode_horovod/mnist_hvd.py +++ b/sagemaker-python-sdk/tensorflow_script_mode_horovod/mnist_hvd.py @@ -14,19 +14,18 @@ from __future__ import print_function import argparse +import math import os +import horovod.tensorflow.keras as hvd import numpy as np +import tensorflow as tf +from tensorflow.keras import backend as K # import tensorflow.keras as keras from tensorflow.keras.datasets import mnist +from tensorflow.keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D from tensorflow.keras.models import Sequential -from tensorflow.keras.layers import Dense, Dropout, Flatten -from tensorflow.keras.layers import Conv2D, MaxPooling2D -from tensorflow.keras import backend as K -import math -import tensorflow as tf -import horovod.tensorflow.keras as hvd if __name__ == "__main__": diff --git a/sagemaker-python-sdk/tensorflow_script_mode_pipe_mode/pipemode.py b/sagemaker-python-sdk/tensorflow_script_mode_pipe_mode/pipemode.py index d1d8eea4ec..5ddbb2b76f 100644 --- a/sagemaker-python-sdk/tensorflow_script_mode_pipe_mode/pipemode.py +++ b/sagemaker-python-sdk/tensorflow_script_mode_pipe_mode/pipemode.py @@ -1,9 +1,9 @@ import argparse import json import os + import tensorflow as tf from sagemaker_tensorflow import PipeModeDataset - from tensorflow.contrib.data import map_and_batch PREFETCH_SIZE = 10 diff --git a/sagemaker-python-sdk/tensorflow_script_mode_training_and_serving/mnist-2.py b/sagemaker-python-sdk/tensorflow_script_mode_training_and_serving/mnist-2.py index 4aeb6f902a..c86f1b14db 100644 --- a/sagemaker-python-sdk/tensorflow_script_mode_training_and_serving/mnist-2.py +++ b/sagemaker-python-sdk/tensorflow_script_mode_training_and_serving/mnist-2.py @@ -11,11 +11,12 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License.import tensorflow as tf -import tensorflow as tf import argparse +import json import os + import numpy as np -import json +import tensorflow as tf def model(x_train, y_train, x_test, y_test): diff --git a/sagemaker-python-sdk/tensorflow_script_mode_training_and_serving/mnist.py b/sagemaker-python-sdk/tensorflow_script_mode_training_and_serving/mnist.py index 6fc5154069..460d12f48f 100644 --- a/sagemaker-python-sdk/tensorflow_script_mode_training_and_serving/mnist.py +++ b/sagemaker-python-sdk/tensorflow_script_mode_training_and_serving/mnist.py @@ -12,19 +12,18 @@ # language governing permissions and limitations under the License. """Convolutional Neural Network Estimator for MNIST, built with tf.layers.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -import numpy as np -import tensorflow as tf -import os -import json import argparse -from tensorflow.python.platform import tf_logging +import json import logging as _logging +import os import sys as _sys +import numpy as np +import tensorflow as tf +from tensorflow.python.platform import tf_logging + def cnn_model_fn(features, labels, mode): """Model function for CNN.""" diff --git a/sagemaker-python-sdk/tensorflow_serving_container/sample_utils.py b/sagemaker-python-sdk/tensorflow_serving_container/sample_utils.py index 748d502be8..d13d9db8d2 100644 --- a/sagemaker-python-sdk/tensorflow_serving_container/sample_utils.py +++ b/sagemaker-python-sdk/tensorflow_serving_container/sample_utils.py @@ -1,7 +1,7 @@ +import cv2 +import numpy as np import tensorflow as tf import tensorflow_hub as hub -import numpy as np -import cv2 def tfhub_to_savedmodel( diff --git a/sagemaker_batch_transform/pytorch_mnist_batch_transform/mnist.py b/sagemaker_batch_transform/pytorch_mnist_batch_transform/mnist.py index 41d8a02f61..f68bff873d 100644 --- a/sagemaker_batch_transform/pytorch_mnist_batch_transform/mnist.py +++ b/sagemaker_batch_transform/pytorch_mnist_batch_transform/mnist.py @@ -1,10 +1,9 @@ import argparse +import io import json import logging import os import sys -import io -from PIL import Image import torch import torch.distributed as dist @@ -13,6 +12,7 @@ import torch.optim as optim import torch.utils.data import torch.utils.data.distributed +from PIL import Image from torchvision import datasets, transforms from torchvision.transforms import ToTensor diff --git a/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/inference.py b/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/inference.py index d15503d886..80091966f9 100755 --- a/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/inference.py +++ b/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/inference.py @@ -13,8 +13,9 @@ import io import json -import numpy as np from collections import namedtuple + +import numpy as np from PIL import Image Context = namedtuple( diff --git a/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/model_def.py b/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/model_def.py index 22c80b4a8a..f98eb88872 100755 --- a/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/model_def.py +++ b/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/model_def.py @@ -1,15 +1,15 @@ import tensorflow as tf from tensorflow.keras.layers import ( Activation, + BatchNormalization, Conv2D, Dense, Dropout, Flatten, MaxPooling2D, - BatchNormalization, ) from tensorflow.keras.models import Sequential -from tensorflow.keras.optimizers import Adam, SGD, RMSprop +from tensorflow.keras.optimizers import SGD, Adam, RMSprop HEIGHT = 32 WIDTH = 32 diff --git a/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/train.py b/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/train.py index 3b7c2bfdda..8969dd4128 100755 --- a/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/train.py +++ b/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/train.py @@ -2,18 +2,16 @@ import codecs import json import logging -import numpy as np import os import re +import numpy as np import tensorflow as tf import tensorflow.keras.backend as K -from tensorflow.keras.callbacks import TensorBoard, ModelCheckpoint - -from model_def import get_model, HEIGHT, WIDTH, DEPTH, NUM_CLASSES +from model_def import DEPTH, HEIGHT, NUM_CLASSES, WIDTH, get_model +from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard from utilities import process_input - logging.getLogger().setLevel(logging.INFO) tf.logging.set_verbosity(tf.logging.ERROR) diff --git a/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/utilities.py b/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/utilities.py index 83752b42e8..8ae0135e29 100755 --- a/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/utilities.py +++ b/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/code/utilities.py @@ -1,9 +1,8 @@ import logging import os -import tensorflow as tf - -from model_def import HEIGHT, WIDTH, DEPTH, NUM_CLASSES +import tensorflow as tf +from model_def import DEPTH, HEIGHT, NUM_CLASSES, WIDTH NUM_DATA_BATCHES = 5 NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 10000 * NUM_DATA_BATCHES diff --git a/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/generate_cifar10_tfrecords.py b/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/generate_cifar10_tfrecords.py index 677f36be9c..d67637ebdb 100755 --- a/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/generate_cifar10_tfrecords.py +++ b/sagemaker_batch_transform/tensorflow_cifar-10_with_inference_script/generate_cifar10_tfrecords.py @@ -19,18 +19,16 @@ https://www.cs.toronto.edu/~kriz/cifar.html. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import argparse import os import sys - import tarfile + +import tensorflow as tf from six.moves import cPickle as pickle from six.moves import xrange # pylint: disable=redefined-builtin -import tensorflow as tf tf.logging.set_verbosity(tf.logging.ERROR) if type(tf.contrib) != type(tf): diff --git a/sagemaker_batch_transform/tensorflow_open-images_jpg/code/inference.py b/sagemaker_batch_transform/tensorflow_open-images_jpg/code/inference.py index 529de7f524..8ac41f73c0 100755 --- a/sagemaker_batch_transform/tensorflow_open-images_jpg/code/inference.py +++ b/sagemaker_batch_transform/tensorflow_open-images_jpg/code/inference.py @@ -14,6 +14,7 @@ import base64 import io import json + import requests diff --git a/sagemaker_batch_transform/tensorflow_open-images_tfrecord/code/inference.py b/sagemaker_batch_transform/tensorflow_open-images_tfrecord/code/inference.py index 356604a59a..c2f7cb8eac 100755 --- a/sagemaker_batch_transform/tensorflow_open-images_tfrecord/code/inference.py +++ b/sagemaker_batch_transform/tensorflow_open-images_tfrecord/code/inference.py @@ -14,10 +14,11 @@ import base64 import io import json +from string import whitespace + import requests import tensorflow as tf from google.protobuf.json_format import MessageToDict -from string import whitespace def input_handler(data, context): diff --git a/sagemaker_batch_transform/working_with_tfrecords/dnn-classifier/train.py b/sagemaker_batch_transform/working_with_tfrecords/dnn-classifier/train.py index f05f23d117..b0596b895e 100644 --- a/sagemaker_batch_transform/working_with_tfrecords/dnn-classifier/train.py +++ b/sagemaker_batch_transform/working_with_tfrecords/dnn-classifier/train.py @@ -20,7 +20,6 @@ import tensorflow as tf - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) # Data and model checkpoints directories parser.add_argument( @@ -71,7 +70,7 @@ def input_fn_train(): def serving_input_receiver_fn(): - """A serving input receiver that expects features encoded as JSON """ + """A serving input receiver that expects features encoded as JSON""" features = { "sepal_length": tf.placeholder(tf.float32, [None, 1]), "sepal_width": tf.placeholder(tf.float32, [None, 1]), diff --git a/sagemaker_batch_transform/working_with_tfrecords/tfrecord-transformer-container/app/main.py b/sagemaker_batch_transform/working_with_tfrecords/tfrecord-transformer-container/app/main.py index f425698b04..15bd1e6e53 100644 --- a/sagemaker_batch_transform/working_with_tfrecords/tfrecord-transformer-container/app/main.py +++ b/sagemaker_batch_transform/working_with_tfrecords/tfrecord-transformer-container/app/main.py @@ -1,11 +1,11 @@ import codecs -import crcmod import io import json import logging import struct -import tensorflow as tf +import crcmod +import tensorflow as tf from flask import Flask, Response, request from google.protobuf.json_format import MessageToDict diff --git a/sagemaker_model_monitor/introduction/preprocessor.py b/sagemaker_model_monitor/introduction/preprocessor.py index 5205b8b62d..6eb63f25b1 100644 --- a/sagemaker_model_monitor/introduction/preprocessor.py +++ b/sagemaker_model_monitor/introduction/preprocessor.py @@ -1,6 +1,7 @@ import json import random + # sample preprocess_handler (to be implemented by customer) # This is a trivial example, where we simply generate random values # But customers can read the data from inference_record and trasnform it into diff --git a/sagemaker_model_monitor/tensorflow/monitoringjob_utils.py b/sagemaker_model_monitor/tensorflow/monitoringjob_utils.py index 599b47f279..f1a2e0d6cd 100644 --- a/sagemaker_model_monitor/tensorflow/monitoringjob_utils.py +++ b/sagemaker_model_monitor/tensorflow/monitoringjob_utils.py @@ -1,6 +1,8 @@ -import os, sys +import os +import sys from urllib.parse import urlparse -from sagemaker.processing import Processor, ProcessingInput, ProcessingOutput + +from sagemaker.processing import ProcessingInput, ProcessingOutput, Processor def get_model_monitor_container_uri(region): diff --git a/sagemaker_model_monitor/visualization/utils.py b/sagemaker_model_monitor/visualization/utils.py index 54dad8ca32..e08ab5445d 100644 --- a/sagemaker_model_monitor/visualization/utils.py +++ b/sagemaker_model_monitor/visualization/utils.py @@ -1,6 +1,7 @@ -from enum import Enum -import warnings import re +import warnings +from enum import Enum + import numpy as np import pandas as pd from IPython.display import HTML, display diff --git a/sagemaker_neo_compilation_jobs/deploy_mxnet_model_on_Inf1_instance/get_input.py b/sagemaker_neo_compilation_jobs/deploy_mxnet_model_on_Inf1_instance/get_input.py index d2a221b1e9..68b4d55cca 100644 --- a/sagemaker_neo_compilation_jobs/deploy_mxnet_model_on_Inf1_instance/get_input.py +++ b/sagemaker_neo_compilation_jobs/deploy_mxnet_model_on_Inf1_instance/get_input.py @@ -1,8 +1,9 @@ import math -import numpy as np from tkinter import * from tkinter import messagebox +import numpy as np + # track welcome status so we only display the welcome message once welcome = False diff --git a/sagemaker_neo_compilation_jobs/deploy_mxnet_model_on_Inf1_instance/mnist.py b/sagemaker_neo_compilation_jobs/deploy_mxnet_model_on_Inf1_instance/mnist.py index a2a68ba8b6..dc59c78ed6 100644 --- a/sagemaker_neo_compilation_jobs/deploy_mxnet_model_on_Inf1_instance/mnist.py +++ b/sagemaker_neo_compilation_jobs/deploy_mxnet_model_on_Inf1_instance/mnist.py @@ -1,13 +1,14 @@ import argparse import gzip +import io import json import logging import os -import io import struct +from collections import namedtuple + import mxnet as mx import numpy as np -from collections import namedtuple def load_data(path): diff --git a/sagemaker_neo_compilation_jobs/deploy_pytorch_model_on_Inf1_instance/resnet18.py b/sagemaker_neo_compilation_jobs/deploy_pytorch_model_on_Inf1_instance/resnet18.py index a39f7d0aab..291c4b3c08 100644 --- a/sagemaker_neo_compilation_jobs/deploy_pytorch_model_on_Inf1_instance/resnet18.py +++ b/sagemaker_neo_compilation_jobs/deploy_pytorch_model_on_Inf1_instance/resnet18.py @@ -1,8 +1,9 @@ def input_fn(request_body, request_content_type): + import io + import torch import torchvision.transforms as transforms from PIL import Image - import io f = io.BytesIO(request_body) input_image = Image.open(f).convert("RGB") diff --git a/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/mnist.py b/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/mnist.py index fd32806c33..2622565a6c 100644 --- a/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/mnist.py +++ b/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/mnist.py @@ -1,4 +1,5 @@ import os + import tensorflow as tf from tensorflow.python.estimator.model_fn import ModeKeys as Modes @@ -110,9 +111,10 @@ def _input_fn(training_dir, training_filename, batch_size=100): def neo_preprocess(payload, content_type): + import io import logging + import numpy as np - import io logging.info("Invoking user-defined pre-processing function") @@ -132,9 +134,10 @@ def neo_preprocess(payload, content_type): ### NOTE: this function cannot use MXNet def neo_postprocess(result): + import json import logging + import numpy as np - import json logging.info("Invoking user-defined post-processing function") diff --git a/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/utils.py b/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/utils.py index cf8af1b2b1..c883d56685 100644 --- a/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/utils.py +++ b/sagemaker_neo_compilation_jobs/deploy_tensorflow_model_on_Inf1_instance/utils.py @@ -1,5 +1,6 @@ """Converts MNIST data to TFRecords file format with Example protos.""" import os + import tensorflow as tf diff --git a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/ssd_entry_point.py b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/ssd_entry_point.py index a47d035468..bfb27dc713 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/ssd_entry_point.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/ssd_entry_point.py @@ -1,9 +1,10 @@ import io -import os -import PIL.Image import json import logging +import os + import numpy as np +import PIL.Image logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -12,14 +13,13 @@ # Training methods # # ------------------------------------------------------------ # +import argparse import glob import time -import argparse import warnings + import mxnet as mx -from mxnet import nd -from mxnet import gluon -from mxnet import autograd +from mxnet import autograd, gluon, nd def parse_args(): @@ -74,7 +74,7 @@ def get_dataloader(net, data_shape, batch_size, num_workers, ctx): """Get dataloader.""" from gluoncv import data as gdata - from gluoncv.data.batchify import Tuple, Stack, Pad + from gluoncv.data.batchify import Pad, Stack, Tuple from gluoncv.data.transforms.presets.ssd import SSDDefaultTrainTransform width, height = data_shape, data_shape diff --git a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/concat_db.py b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/concat_db.py index 9b11085b6a..51ff2adcd3 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/concat_db.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/concat_db.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -from imdb import Imdb import random +from imdb import Imdb + class ConcatDB(Imdb): """ diff --git a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/im2rec.py b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/im2rec.py index d6ad10aceb..ecb8657259 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/im2rec.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/im2rec.py @@ -18,18 +18,20 @@ # under the License. from __future__ import print_function + import os import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "../python")) -import mxnet as mx -import random import argparse -import cv2 +import random import time import traceback +import cv2 +import mxnet as mx + try: import multiprocessing except ImportError: diff --git a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/imdb.py b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/imdb.py index b10d32aec3..c9414326a4 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/imdb.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/imdb.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -import numpy as np import os.path as osp +import numpy as np + class Imdb(object): """ diff --git a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/pascal_voc.py b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/pascal_voc.py index ae3d2a054a..d2b1826fef 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/pascal_voc.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/pascal_voc.py @@ -16,11 +16,13 @@ # under the License. from __future__ import print_function + import os -import numpy as np -from imdb import Imdb import xml.etree.ElementTree as ET + import cv2 +import numpy as np +from imdb import Imdb class PascalVoc(Imdb): diff --git a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/prepare_dataset.py b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/prepare_dataset.py index 2ccdefee80..6acff43479 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/prepare_dataset.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_ssd_mobilenet/tools/prepare_dataset.py @@ -19,14 +19,16 @@ # under the License. from __future__ import print_function -import sys, os + import argparse +import os import subprocess +import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "..")) -from pascal_voc import PascalVoc from concat_db import ConcatDB +from pascal_voc import PascalVoc def load_pascal(image_set, year, devkit_path, shuffle=False): diff --git a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/concat_db.py b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/concat_db.py index 9b11085b6a..51ff2adcd3 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/concat_db.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/concat_db.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -from imdb import Imdb import random +from imdb import Imdb + class ConcatDB(Imdb): """ diff --git a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/im2rec.py b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/im2rec.py index d6ad10aceb..ecb8657259 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/im2rec.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/im2rec.py @@ -18,18 +18,20 @@ # under the License. from __future__ import print_function + import os import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "../python")) -import mxnet as mx -import random import argparse -import cv2 +import random import time import traceback +import cv2 +import mxnet as mx + try: import multiprocessing except ImportError: diff --git a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/imdb.py b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/imdb.py index b10d32aec3..c9414326a4 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/imdb.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/imdb.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. -import numpy as np import os.path as osp +import numpy as np + class Imdb(object): """ diff --git a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/pascal_voc.py b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/pascal_voc.py index ae3d2a054a..d2b1826fef 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/pascal_voc.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/pascal_voc.py @@ -16,11 +16,13 @@ # under the License. from __future__ import print_function + import os -import numpy as np -from imdb import Imdb import xml.etree.ElementTree as ET + import cv2 +import numpy as np +from imdb import Imdb class PascalVoc(Imdb): diff --git a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/prepare_dataset.py b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/prepare_dataset.py index 2ccdefee80..6acff43479 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/prepare_dataset.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/tools/prepare_dataset.py @@ -19,14 +19,16 @@ # under the License. from __future__ import print_function -import sys, os + import argparse +import os import subprocess +import sys curr_path = os.path.abspath(os.path.dirname(__file__)) sys.path.append(os.path.join(curr_path, "..")) -from pascal_voc import PascalVoc from concat_db import ConcatDB +from pascal_voc import PascalVoc def load_pascal(image_set, year, devkit_path, shuffle=False): diff --git a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/train_yolo.py b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/train_yolo.py index 877fb5f96e..e156ae35f1 100644 --- a/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/train_yolo.py +++ b/sagemaker_neo_compilation_jobs/gluoncv_yolo_darknet/train_yolo.py @@ -1,8 +1,9 @@ import io -import PIL.Image import json import logging + import numpy as np +import PIL.Image logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -13,15 +14,14 @@ """Train YOLOv3 with random shapes.""" import argparse -import os import logging +import os import time import warnings -import numpy as np + import mxnet as mx -from mxnet import nd -from mxnet import gluon -from mxnet import autograd +import numpy as np +from mxnet import autograd, gluon, nd # from mxnet.contrib import amp try: @@ -213,14 +213,16 @@ def get_dataloader(net, train_dataset, val_dataset, data_shape, batch_size, num_ gcv.utils.check_version("0.6.0") from gluoncv import data as gdata from gluoncv import utils as gutils - from gluoncv.model_zoo import get_model - from gluoncv.data.batchify import Tuple, Stack, Pad - from gluoncv.data.transforms.presets.yolo import YOLO3DefaultTrainTransform - from gluoncv.data.transforms.presets.yolo import YOLO3DefaultValTransform + from gluoncv.data.batchify import Pad, Stack, Tuple from gluoncv.data.dataloader import RandomTransformDataLoader - from gluoncv.utils.metrics.voc_detection import VOC07MApMetric - from gluoncv.utils.metrics.coco_detection import COCODetectionMetric + from gluoncv.data.transforms.presets.yolo import ( + YOLO3DefaultTrainTransform, + YOLO3DefaultValTransform, + ) + from gluoncv.model_zoo import get_model from gluoncv.utils import LRScheduler, LRSequential + from gluoncv.utils.metrics.coco_detection import COCODetectionMetric + from gluoncv.utils.metrics.voc_detection import VOC07MApMetric """Get dataloader.""" width, height = data_shape, data_shape @@ -318,14 +320,16 @@ def train(net, train_data, val_data, eval_metric, ctx, args): gcv.utils.check_version("0.6.0") from gluoncv import data as gdata from gluoncv import utils as gutils - from gluoncv.model_zoo import get_model - from gluoncv.data.batchify import Tuple, Stack, Pad - from gluoncv.data.transforms.presets.yolo import YOLO3DefaultTrainTransform - from gluoncv.data.transforms.presets.yolo import YOLO3DefaultValTransform + from gluoncv.data.batchify import Pad, Stack, Tuple from gluoncv.data.dataloader import RandomTransformDataLoader - from gluoncv.utils.metrics.voc_detection import VOC07MApMetric - from gluoncv.utils.metrics.coco_detection import COCODetectionMetric + from gluoncv.data.transforms.presets.yolo import ( + YOLO3DefaultTrainTransform, + YOLO3DefaultValTransform, + ) + from gluoncv.model_zoo import get_model from gluoncv.utils import LRScheduler, LRSequential + from gluoncv.utils.metrics.coco_detection import COCODetectionMetric + from gluoncv.utils.metrics.voc_detection import VOC07MApMetric """Training pipeline""" net.collect_params().reset_ctx(ctx) @@ -521,14 +525,16 @@ def train(net, train_data, val_data, eval_metric, ctx, args): gcv.utils.check_version("0.6.0") from gluoncv import data as gdata from gluoncv import utils as gutils - from gluoncv.model_zoo import get_model - from gluoncv.data.batchify import Tuple, Stack, Pad - from gluoncv.data.transforms.presets.yolo import YOLO3DefaultTrainTransform - from gluoncv.data.transforms.presets.yolo import YOLO3DefaultValTransform + from gluoncv.data.batchify import Pad, Stack, Tuple from gluoncv.data.dataloader import RandomTransformDataLoader - from gluoncv.utils.metrics.voc_detection import VOC07MApMetric - from gluoncv.utils.metrics.coco_detection import COCODetectionMetric + from gluoncv.data.transforms.presets.yolo import ( + YOLO3DefaultTrainTransform, + YOLO3DefaultValTransform, + ) + from gluoncv.model_zoo import get_model from gluoncv.utils import LRScheduler, LRSequential + from gluoncv.utils.metrics.coco_detection import COCODetectionMetric + from gluoncv.utils.metrics.voc_detection import VOC07MApMetric args = parse_args() diff --git a/sagemaker_neo_compilation_jobs/imageclassification_caltech/inference.py b/sagemaker_neo_compilation_jobs/imageclassification_caltech/inference.py index ae88931164..41f3cbcdda 100644 --- a/sagemaker_neo_compilation_jobs/imageclassification_caltech/inference.py +++ b/sagemaker_neo_compilation_jobs/imageclassification_caltech/inference.py @@ -1,12 +1,13 @@ -import numpy as np +import io import json +import logging +import os + import mxnet as mx # Please make sure to import neomx import neomx # noqa: F401 -import io -import os -import logging +import numpy as np # Change the context to mx.gpu() if deploying to a GPU endpoint ctx = mx.cpu() diff --git a/sagemaker_neo_compilation_jobs/mxnet_mnist/get_input.py b/sagemaker_neo_compilation_jobs/mxnet_mnist/get_input.py index d2a221b1e9..68b4d55cca 100644 --- a/sagemaker_neo_compilation_jobs/mxnet_mnist/get_input.py +++ b/sagemaker_neo_compilation_jobs/mxnet_mnist/get_input.py @@ -1,8 +1,9 @@ import math -import numpy as np from tkinter import * from tkinter import messagebox +import numpy as np + # track welcome status so we only display the welcome message once welcome = False diff --git a/sagemaker_neo_compilation_jobs/mxnet_mnist/mnist.py b/sagemaker_neo_compilation_jobs/mxnet_mnist/mnist.py index d4e49d1586..b77f0f0f9e 100644 --- a/sagemaker_neo_compilation_jobs/mxnet_mnist/mnist.py +++ b/sagemaker_neo_compilation_jobs/mxnet_mnist/mnist.py @@ -1,13 +1,14 @@ import argparse import gzip +import io import json import logging import os -import io import struct +from collections import namedtuple + import mxnet as mx import numpy as np -from collections import namedtuple def load_data(path): diff --git a/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py b/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py index c59619a91c..8948ed6ee7 100644 --- a/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py +++ b/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py @@ -4,9 +4,9 @@ import os import pickle +import neopytorch import numpy as np import torch -import neopytorch import torchvision.transforms as transforms from PIL import Image # Training container doesn't have this package diff --git a/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py index 06e9f288cd..040a0bd6f1 100644 --- a/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py +++ b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py @@ -4,9 +4,9 @@ import os import pickle +import neopytorch import numpy as np import torch -import neopytorch import torchvision.transforms as transforms from PIL import Image # Training container doesn't have this package diff --git a/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py index 50eecb0233..de6a0819da 100644 --- a/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py +++ b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py @@ -4,9 +4,9 @@ import os import pickle +import neopytorch import numpy as np import torch -import neopytorch import torchvision.transforms as transforms from PIL import Image # Training container doesn't have this package diff --git a/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/mnist.py b/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/mnist.py index f731a22a71..38063e0092 100644 --- a/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/mnist.py +++ b/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/mnist.py @@ -1,9 +1,10 @@ +import argparse +import json import os + +import numpy as np import tensorflow as tf -import argparse from tensorflow.python.estimator.model_fn import ModeKeys as Modes -import numpy as np -import json INPUT_TENSOR_NAME = "inputs" SIGNATURE_NAME = "predictions" diff --git a/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/utils.py b/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/utils.py index cf8af1b2b1..c883d56685 100644 --- a/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/utils.py +++ b/sagemaker_neo_compilation_jobs/tensorflow_distributed_mnist/utils.py @@ -1,5 +1,6 @@ """Converts MNIST data to TFRecords file format with Example protos.""" import os + import tensorflow as tf diff --git a/sagemaker_processing/feature_transformation_with_sagemaker_processing/container/program/bootstrap.py b/sagemaker_processing/feature_transformation_with_sagemaker_processing/container/program/bootstrap.py index cd9333c4bf..54bfe7cc7b 100644 --- a/sagemaker_processing/feature_transformation_with_sagemaker_processing/container/program/bootstrap.py +++ b/sagemaker_processing/feature_transformation_with_sagemaker_processing/container/program/bootstrap.py @@ -1,12 +1,13 @@ +import json import os import socket -import json -import psutil import subprocess import sys import time from shutil import copyfile +import psutil + HADOOP_CONFIG_PATH = "/opt/hadoop-config/" HADOOP_PATH = "/usr/hadoop-3.0.0" SPARK_PATH = "/usr/spark-2.4.4" diff --git a/sagemaker_processing/feature_transformation_with_sagemaker_processing/container/program/submit b/sagemaker_processing/feature_transformation_with_sagemaker_processing/container/program/submit index dad70fe40e..acd8fda04f 100644 --- a/sagemaker_processing/feature_transformation_with_sagemaker_processing/container/program/submit +++ b/sagemaker_processing/feature_transformation_with_sagemaker_processing/container/program/submit @@ -3,16 +3,15 @@ from __future__ import print_function import os -import sys -import subprocess -import traceback -import bootstrap import signal +import subprocess import sys import time - +import traceback from shlex import quote +import bootstrap + prefix = '/opt/ml/processing/' jars_path = os.path.join(prefix, 'input/jars/') diff --git a/scientific_details_of_algorithms/lda_topic_modeling/generate_example_data.py b/scientific_details_of_algorithms/lda_topic_modeling/generate_example_data.py index 84a4281c40..5bbf054714 100644 --- a/scientific_details_of_algorithms/lda_topic_modeling/generate_example_data.py +++ b/scientific_details_of_algorithms/lda_topic_modeling/generate_example_data.py @@ -7,12 +7,11 @@ # or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. import matplotlib -import matplotlib.pyplot as plt import matplotlib.cm as cm +import matplotlib.pyplot as plt import numpy as np import scipy as sp import scipy.stats - from matplotlib.gridspec import GridSpec, GridSpecFromSubplotSpec diff --git a/step-functions-data-science-sdk/automate_model_retraining_workflow/code/glue_etl.py b/step-functions-data-science-sdk/automate_model_retraining_workflow/code/glue_etl.py index 0142f99f98..412e5ed760 100644 --- a/step-functions-data-science-sdk/automate_model_retraining_workflow/code/glue_etl.py +++ b/step-functions-data-science-sdk/automate_model_retraining_workflow/code/glue_etl.py @@ -1,10 +1,11 @@ import sys + +from awsglue.context import GlueContext +from awsglue.dynamicframe import DynamicFrame +from awsglue.job import Job from awsglue.transforms import * from awsglue.utils import getResolvedOptions from pyspark.context import SparkContext -from awsglue.context import GlueContext -from awsglue.job import Job -from awsglue.dynamicframe import DynamicFrame # Retrieve parameters for the Glue job. args = getResolvedOptions(sys.argv, ["JOB_NAME", "S3_SOURCE", "S3_DEST", "TRAIN_KEY", "VAL_KEY"]) diff --git a/step-functions-data-science-sdk/automate_model_retraining_workflow/code/query_training_status.py b/step-functions-data-science-sdk/automate_model_retraining_workflow/code/query_training_status.py index a7c9b4282e..0802355345 100644 --- a/step-functions-data-science-sdk/automate_model_retraining_workflow/code/query_training_status.py +++ b/step-functions-data-science-sdk/automate_model_retraining_workflow/code/query_training_status.py @@ -1,6 +1,7 @@ -import boto3 -import logging import json +import logging + +import boto3 logger = logging.getLogger() logger.setLevel(logging.INFO) diff --git a/step-functions-data-science-sdk/training_pipeline_pytorch_mnist/mnist.py b/step-functions-data-science-sdk/training_pipeline_pytorch_mnist/mnist.py index 461162f054..d1c4f3b369 100644 --- a/step-functions-data-science-sdk/training_pipeline_pytorch_mnist/mnist.py +++ b/step-functions-data-science-sdk/training_pipeline_pytorch_mnist/mnist.py @@ -2,8 +2,9 @@ import json import logging import os -import sagemaker_containers import sys + +import sagemaker_containers import torch import torch.distributed as dist import torch.nn as nn diff --git a/training/distributed_training/pytorch/data_parallel/maskrcnn/train_pytorch_smdataparallel_maskrcnn.py b/training/distributed_training/pytorch/data_parallel/maskrcnn/train_pytorch_smdataparallel_maskrcnn.py index 959e3b025f..3a97cc4ef0 100644 --- a/training/distributed_training/pytorch/data_parallel/maskrcnn/train_pytorch_smdataparallel_maskrcnn.py +++ b/training/distributed_training/pytorch/data_parallel/maskrcnn/train_pytorch_smdataparallel_maskrcnn.py @@ -20,31 +20,30 @@ from maskrcnn_benchmark.utils.env import setup_environment # noqa F401 isort:skip import argparse -import os -import logging import functools +import logging +import os import random -import numpy as np +import numpy as np +import smdistributed.dataparallel.torch.distributed as dist import torch from maskrcnn_benchmark.config import cfg from maskrcnn_benchmark.data import make_data_loader -from maskrcnn_benchmark.solver import make_lr_scheduler -from maskrcnn_benchmark.solver import make_optimizer from maskrcnn_benchmark.engine.inference import inference +from maskrcnn_benchmark.engine.tester import test from maskrcnn_benchmark.engine.trainer import do_train from maskrcnn_benchmark.modeling.detector import build_detection_model +from maskrcnn_benchmark.solver import make_lr_scheduler, make_optimizer from maskrcnn_benchmark.utils.checkpoint import DetectronCheckpointer from maskrcnn_benchmark.utils.collect_env import collect_env_info -from maskrcnn_benchmark.utils.comm import synchronize, get_rank, is_main_process +from maskrcnn_benchmark.utils.comm import get_rank, is_main_process, synchronize from maskrcnn_benchmark.utils.imports import import_file from maskrcnn_benchmark.utils.logger import setup_logger from maskrcnn_benchmark.utils.miscellaneous import mkdir -from maskrcnn_benchmark.engine.tester import test # Import SMDataParallel modules for PyTorch. from smdistributed.dataparallel.torch.parallel.distributed import DistributedDataParallel as DDP -import smdistributed.dataparallel.torch.distributed as dist dist.init_process_group() diff --git a/training/distributed_training/pytorch/data_parallel/mnist/code/inference.py b/training/distributed_training/pytorch/data_parallel/mnist/code/inference.py index 4457326a64..289acb5301 100644 --- a/training/distributed_training/pytorch/data_parallel/mnist/code/inference.py +++ b/training/distributed_training/pytorch/data_parallel/mnist/code/inference.py @@ -18,6 +18,7 @@ from __future__ import print_function import os + import torch # Network definition diff --git a/training/distributed_training/pytorch/data_parallel/mnist/code/model_def.py b/training/distributed_training/pytorch/data_parallel/mnist/code/model_def.py index b64f2664d7..beb2f58192 100644 --- a/training/distributed_training/pytorch/data_parallel/mnist/code/model_def.py +++ b/training/distributed_training/pytorch/data_parallel/mnist/code/model_def.py @@ -16,8 +16,8 @@ # under the License. import torch -import torch.nn.functional as F import torch.nn as nn +import torch.nn.functional as F class Net(nn.Module): diff --git a/training/distributed_training/pytorch/data_parallel/mnist/code/train_pytorch_smdataparallel_mnist.py b/training/distributed_training/pytorch/data_parallel/mnist/code/train_pytorch_smdataparallel_mnist.py index 1539ead181..0e07464cef 100644 --- a/training/distributed_training/pytorch/data_parallel/mnist/code/train_pytorch_smdataparallel_mnist.py +++ b/training/distributed_training/pytorch/data_parallel/mnist/code/train_pytorch_smdataparallel_mnist.py @@ -13,22 +13,23 @@ from __future__ import print_function -import os import argparse +import os import time + +import smdistributed.dataparallel.torch.distributed as dist import torch +import torch.nn as nn import torch.nn.functional as F import torch.optim as optim -import torch.nn as nn -from torchvision import datasets, transforms -from torch.optim.lr_scheduler import StepLR # Network definition from model_def import Net # Import SMDataParallel PyTorch Modules from smdistributed.dataparallel.torch.parallel.distributed import DistributedDataParallel as DDP -import smdistributed.dataparallel.torch.distributed as dist +from torch.optim.lr_scheduler import StepLR +from torchvision import datasets, transforms class CUDANotFoundException(Exception): diff --git a/training/distributed_training/pytorch/model_parallel/bert/bert_example/file_utils.py b/training/distributed_training/pytorch/model_parallel/bert/bert_example/file_utils.py index 2719169764..0905380800 100755 --- a/training/distributed_training/pytorch/model_parallel/bert/bert_example/file_utils.py +++ b/training/distributed_training/pytorch/model_parallel/bert/bert_example/file_utils.py @@ -23,10 +23,10 @@ import logging import os import shutil +import sys import tempfile from functools import wraps from hashlib import sha256 -import sys from io import open import boto3 diff --git a/training/distributed_training/pytorch/model_parallel/bert/bert_example/modeling.py b/training/distributed_training/pytorch/model_parallel/bert/bert_example/modeling.py index 44d32255bb..ea26980a34 100755 --- a/training/distributed_training/pytorch/model_parallel/bert/bert_example/modeling.py +++ b/training/distributed_training/pytorch/model_parallel/bert/bert_example/modeling.py @@ -23,9 +23,9 @@ import math import os import shutil +import sys import tarfile import tempfile -import sys from io import open import torch @@ -34,14 +34,13 @@ from torch.utils import checkpoint sys.path.append("/workspace/bert/") -from file_utils import cached_path - -from torch.nn import Module -from torch.nn.parameter import Parameter +import smdistributed.modelparallel.torch as smp import torch.nn.functional as F import torch.nn.init as init -import smdistributed.modelparallel.torch as smp +from file_utils import cached_path from smdistributed.modelparallel.torch.utils import rmsg +from torch.nn import Module +from torch.nn.parameter import Parameter logger = logging.getLogger(__name__) @@ -63,6 +62,7 @@ def load_tf_weights_in_bert(model, tf_checkpoint_path): """Load tf checkpoints in a pytorch model""" try: import re + import numpy as np import tensorflow as tf except ImportError: diff --git a/training/distributed_training/pytorch/model_parallel/bert/bert_example/sagemaker_smp_pretrain.py b/training/distributed_training/pytorch/model_parallel/bert/bert_example/sagemaker_smp_pretrain.py index 7323523e2d..3672598f64 100755 --- a/training/distributed_training/pytorch/model_parallel/bert/bert_example/sagemaker_smp_pretrain.py +++ b/training/distributed_training/pytorch/model_parallel/bert/bert_example/sagemaker_smp_pretrain.py @@ -17,49 +17,44 @@ """BERT finetuning runner.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function + +import argparse +import configparser # ================== import csv -import os -import sys -import time -import argparse -import random -import h5py -from tqdm import tqdm, trange -import os -import numpy as np -import torch -from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, Dataset -from torch.utils.data.distributed import DistributedSampler +import json import math -from apex import amp import multiprocessing +import os import pprint -import json +import random import subprocess +import sys +import time +from concurrent.futures import ProcessPoolExecutor -import modeling -from schedulers import PolyWarmUpScheduler - -from utils import format_step, get_world_size, get_rank -from utils import is_main_process -from apex.parallel import DistributedDataParallel as DDP -from smdistributed.modelparallel.torch.optimizers import FusedLAMB -from schedulers import LinearWarmUpScheduler -from apex.parallel.distributed import flat_dist_call import amp_C import apex_C -from apex.amp import _amp_state +import h5py +import modeling +import numpy as np # SMP: Import smp library import smdistributed.modelparallel.torch as smp -import configparser +import torch +from apex import amp +from apex.amp import _amp_state +from apex.parallel import DistributedDataParallel as DDP +from apex.parallel.distributed import flat_dist_call +from schedulers import LinearWarmUpScheduler, PolyWarmUpScheduler +from smdistributed.modelparallel.torch.optimizers import FusedLAMB +from torch.utils.data import DataLoader, Dataset, RandomSampler, SequentialSampler +from torch.utils.data.distributed import DistributedSampler +from tqdm import tqdm, trange -from concurrent.futures import ProcessPoolExecutor +from utils import format_step, get_rank, get_world_size, is_main_process torch._C._jit_set_profiling_mode(False) torch._C._jit_set_profiling_executor(False) @@ -71,6 +66,7 @@ import signal + # handle SIGTERM sent from the scheduler and mark so we # can gracefully save & exit def signal_handler(sig, frame): @@ -174,7 +170,8 @@ def forward( def aws_s3_sync(source, destination): """aws s3 sync in quiet mode and time profile""" - import time, subprocess + import subprocess + import time cmd = ["aws", "s3", "sync", "--quiet", source, destination] print(f"Syncing files from {source} to {destination}") @@ -187,9 +184,10 @@ def aws_s3_sync(source, destination): def sync_local_checkpoints_to_s3(local_path, s3_path): - """ sample function to sync checkpoints from local path to s3 """ + """sample function to sync checkpoints from local path to s3""" - import boto3, botocore + import boto3 + import botocore # check if local path exists if not os.path.exists(local_path): @@ -213,7 +211,7 @@ def sync_local_checkpoints_to_s3(local_path, s3_path): def sync_s3_checkpoints_to_local(local_path, s3_uri): - """ sample function to sync checkpoints from s3 to local path """ + """sample function to sync checkpoints from s3 to local path""" import boto3 @@ -922,7 +920,7 @@ def main(): batch = [t.to(device) for t in batch] input_ids, segment_ids, input_mask, masked_lm_labels, next_sentence_labels = batch if args.do_train: - from smdistributed.modelparallel.test.torch.utils import verify, dump_model + from smdistributed.modelparallel.test.torch.utils import dump_model, verify model.train() if args.smp > 0: diff --git a/training/distributed_training/pytorch/model_parallel/bert/bert_example/schedulers.py b/training/distributed_training/pytorch/model_parallel/bert/bert_example/schedulers.py index 4fb7b077ac..1ec0429364 100755 --- a/training/distributed_training/pytorch/model_parallel/bert/bert_example/schedulers.py +++ b/training/distributed_training/pytorch/model_parallel/bert/bert_example/schedulers.py @@ -13,9 +13,10 @@ # limitations under the License. import math + import torch -from torch.optim.optimizer import Optimizer from torch.optim.lr_scheduler import _LRScheduler +from torch.optim.optimizer import Optimizer class LRScheduler(_LRScheduler): diff --git a/training/distributed_training/pytorch/model_parallel/bert/bert_example/utils.py b/training/distributed_training/pytorch/model_parallel/bert/bert_example/utils.py index 66555eeec9..526317d474 100755 --- a/training/distributed_training/pytorch/model_parallel/bert/bert_example/utils.py +++ b/training/distributed_training/pytorch/model_parallel/bert/bert_example/utils.py @@ -11,12 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import torch -import torch.distributed as dist - from pathlib import Path import smdistributed.modelparallel.torch as smp +import torch +import torch.distributed as dist def get_rank(): diff --git a/training/distributed_training/pytorch/model_parallel/mnist/utils/pt_mnist.py b/training/distributed_training/pytorch/model_parallel/mnist/utils/pt_mnist.py index 8331b0a5de..6208707ed9 100644 --- a/training/distributed_training/pytorch/model_parallel/mnist/utils/pt_mnist.py +++ b/training/distributed_training/pytorch/model_parallel/mnist/utils/pt_mnist.py @@ -1,14 +1,19 @@ # Future from __future__ import print_function -# Standard Library -import os, time import argparse import math + +# Standard Library +import os import random +import time # Third Party import numpy as np + +# First Party +import smdistributed.modelparallel.torch as smp import torch import torch.nn as nn import torch.nn.functional as F @@ -18,9 +23,6 @@ from torchnet.dataset import SplitDataset from torchvision import datasets, transforms -# First Party -import smdistributed.modelparallel.torch as smp - # SM Distributed: import scaler from smdistributed.modelparallel.torch.amp, instead of torch.cuda.amp # Make cudnn deterministic in order to get the same losses across runs. @@ -34,7 +36,8 @@ def aws_s3_sync(source, destination): """aws s3 sync in quiet mode and time profile""" - import time, subprocess + import subprocess + import time cmd = ["aws", "s3", "sync", "--quiet", source, destination] print(f"Syncing files from {source} to {destination}") @@ -51,9 +54,10 @@ def sync_local_checkpoints_to_s3( s3_path=os.path.dirname(os.path.dirname(os.getenv("SM_MODULE_DIR", ""))) + "/checkpoints", ): - """ sample function to sync checkpoints from local path to s3 """ + """sample function to sync checkpoints from local path to s3""" - import boto3, botocore + import boto3 + import botocore # check if local path exists if not os.path.exists(local_path): @@ -81,9 +85,10 @@ def sync_s3_checkpoints_to_local( s3_path=os.path.dirname(os.path.dirname(os.getenv("SM_MODULE_DIR", ""))) + "/checkpoints", ): - """ sample function to sync checkpoints from s3 to local path """ + """sample function to sync checkpoints from s3 to local path""" - import boto3, botocore + import boto3 + import botocore # creat if local path does not exists if not os.path.exists(local_path): diff --git a/training/distributed_training/tensorflow/data_parallel/mnist/code/train_tensorflow_smdataparallel_mnist.py b/training/distributed_training/tensorflow/data_parallel/mnist/code/train_tensorflow_smdataparallel_mnist.py index 40aac73425..03a5e3286e 100644 --- a/training/distributed_training/tensorflow/data_parallel/mnist/code/train_tensorflow_smdataparallel_mnist.py +++ b/training/distributed_training/tensorflow/data_parallel/mnist/code/train_tensorflow_smdataparallel_mnist.py @@ -15,12 +15,11 @@ # specific language governing permissions and limitations # under the License. -import tensorflow as tf +import os # Import SMDataParallel TensorFlow2 Modules import smdistributed.dataparallel.tensorflow as dist - -import os +import tensorflow as tf tf.random.set_seed(42) diff --git a/training/distributed_training/tensorflow/model_parallel/mnist/utils/tf2.py b/training/distributed_training/tensorflow/model_parallel/mnist/utils/tf2.py index d7e8ff00de..374bec138b 100644 --- a/training/distributed_training/tensorflow/model_parallel/mnist/utils/tf2.py +++ b/training/distributed_training/tensorflow/model_parallel/mnist/utils/tf2.py @@ -1,10 +1,11 @@ -import os, time import errno -import tensorflow as tf -from tensorflow.keras.layers import Conv2D, Dense, Flatten +import os +import time # SMP: Import TF2.x API import smdistributed.modelparallel.tensorflow as smp +import tensorflow as tf +from tensorflow.keras.layers import Conv2D, Dense, Flatten tf.random.set_seed(1234) @@ -79,7 +80,8 @@ def call(self, x, training=None): def aws_s3_sync(source, destination): """aws s3 sync in quiet mode and time profile""" - import time, subprocess + import subprocess + import time cmd = ["aws", "s3", "sync", "--quiet", source, destination] print(f"Syncing files from {source} to {destination}") @@ -95,9 +97,10 @@ def sync_local_checkpoints_to_s3( local_path="/opt/ml/checkpoints", s3_path=os.path.dirname(os.path.dirname(os.getenv("SM_MODULE_DIR", ""))) + "/checkpoints", ): - """ sample function to sync checkpoints from local path to s3 """ + """sample function to sync checkpoints from local path to s3""" - import boto3, botocore + import boto3 + import botocore # check if local path exists if not os.path.exists(local_path): @@ -124,9 +127,10 @@ def sync_s3_checkpoints_to_local( local_path="/opt/ml/checkpoints", s3_path=os.path.dirname(os.path.dirname(os.getenv("SM_MODULE_DIR", ""))) + "/checkpoints", ): - """ sample function to sync checkpoints from s3 to local path """ + """sample function to sync checkpoints from s3 to local path""" - import boto3, botocore + import boto3 + import botocore # creat if local path does not exists if not os.path.exists(local_path): diff --git a/training/distributed_training/tensorflow/model_parallel/mnist/utils/tf2_hvd.py b/training/distributed_training/tensorflow/model_parallel/mnist/utils/tf2_hvd.py index 3e33d9b562..30326a6e75 100644 --- a/training/distributed_training/tensorflow/model_parallel/mnist/utils/tf2_hvd.py +++ b/training/distributed_training/tensorflow/model_parallel/mnist/utils/tf2_hvd.py @@ -1,11 +1,12 @@ -import os import errno +import os + import horovod.tensorflow as hvd -import tensorflow as tf -from tensorflow.keras.layers import Conv2D, Dense, Flatten # SMP: Import TF2.x API import smdistributed.modelparallel.tensorflow as smp +import tensorflow as tf +from tensorflow.keras.layers import Conv2D, Dense, Flatten tf.random.set_seed(1234) diff --git a/use-cases/computer_vision/deploy_model.py b/use-cases/computer_vision/deploy_model.py index cad4ff5e56..37287dd384 100644 --- a/use-cases/computer_vision/deploy_model.py +++ b/use-cases/computer_vision/deploy_model.py @@ -1,7 +1,7 @@ -import time -import boto3 import argparse +import time +import boto3 # Parse argument variables passed via the DeployModel processing step parser = argparse.ArgumentParser() diff --git a/use-cases/computer_vision/split_data.py b/use-cases/computer_vision/split_data.py index 67d7fa1775..bd2069eaf8 100644 --- a/use-cases/computer_vision/split_data.py +++ b/use-cases/computer_vision/split_data.py @@ -5,13 +5,14 @@ subprocess.check_call([sys.executable, "-m", "pip", "install", "mxnet"]) subprocess.check_call([sys.executable, "-m", "pip", "install", "opencv-python-headless"]) -import h5py +import os import pathlib + +import cv2 +import h5py import mxnet as mx import numpy as np from sklearn.model_selection import train_test_split -import cv2 -import os def write_to_recordio(X: np.ndarray, y: np.ndarray, prefix: str): diff --git a/use-cases/customer_churn/preprocessing.py b/use-cases/customer_churn/preprocessing.py index 6b93ebaab1..88d3be5863 100644 --- a/use-cases/customer_churn/preprocessing.py +++ b/use-cases/customer_churn/preprocessing.py @@ -1,10 +1,11 @@ -import os -import warnings -import time -import pandas as pd import argparse +import os import subprocess import sys +import time +import warnings + +import pandas as pd subprocess.check_call([sys.executable, "-m", "pip", "install", "awswrangler"]) import awswrangler as wr diff --git a/use-cases/customer_churn/preprocessing_predw.py b/use-cases/customer_churn/preprocessing_predw.py index 2556032293..213037d874 100644 --- a/use-cases/customer_churn/preprocessing_predw.py +++ b/use-cases/customer_churn/preprocessing_predw.py @@ -1,12 +1,11 @@ import argparse -import os -import warnings import glob -import time -import pandas as pd import json -import argparse +import os +import time +import warnings +import pandas as pd from sklearn.exceptions import DataConversionWarning warnings.filterwarnings(action="ignore", category=DataConversionWarning) diff --git a/use-cases/predmaint/demo_helpers.py b/use-cases/predmaint/demo_helpers.py index 0782417aff..022482ba31 100644 --- a/use-cases/predmaint/demo_helpers.py +++ b/use-cases/predmaint/demo_helpers.py @@ -1,4 +1,5 @@ import json + import boto3 import sagemaker diff --git a/use-cases/retail_recommend/data_wrangler_assets/preprocessing_3.py b/use-cases/retail_recommend/data_wrangler_assets/preprocessing_3.py index 95beaa611f..6fcb2f90b3 100644 --- a/use-cases/retail_recommend/data_wrangler_assets/preprocessing_3.py +++ b/use-cases/retail_recommend/data_wrangler_assets/preprocessing_3.py @@ -1,17 +1,17 @@ -import io -import numpy as np -from scipy.sparse import lil_matrix -from sklearn.model_selection import train_test_split import argparse +import io import json import subprocess import sys +import numpy as np +from scipy.sparse import lil_matrix +from sklearn.model_selection import train_test_split + subprocess.check_call([sys.executable, "-m", "pip", "install", "sagemaker"]) subprocess.check_call([sys.executable, "-m", "pip", "install", "awswrangler"]) -import sagemaker.amazon.common as smac import awswrangler as wr - +import sagemaker.amazon.common as smac # Parse argument variables passed via the DeployModel processing step parser = argparse.ArgumentParser() diff --git a/use-cases/retail_recommend/deploy.py b/use-cases/retail_recommend/deploy.py index cad4ff5e56..37287dd384 100644 --- a/use-cases/retail_recommend/deploy.py +++ b/use-cases/retail_recommend/deploy.py @@ -1,7 +1,7 @@ -import time -import boto3 import argparse +import time +import boto3 # Parse argument variables passed via the DeployModel processing step parser = argparse.ArgumentParser() diff --git a/use-cases/retail_recommend/preprocessing.py b/use-cases/retail_recommend/preprocessing.py index ede84aafbd..788d23e951 100644 --- a/use-cases/retail_recommend/preprocessing.py +++ b/use-cases/retail_recommend/preprocessing.py @@ -1,14 +1,14 @@ +import io import os +import subprocess +import sys + import numpy as np import pandas as pd -import io from scipy.sparse import csr_matrix, hstack -from sklearn.preprocessing import OneHotEncoder -from sklearn.model_selection import train_test_split from sklearn.feature_extraction.text import TfidfVectorizer - -import subprocess -import sys +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import OneHotEncoder subprocess.check_call([sys.executable, "-m", "pip", "install", "sagemaker"]) import sagemaker.amazon.common as smac diff --git a/utils/datasets.py b/utils/datasets.py index a8a05b5e42..6a9b02f977 100644 --- a/utils/datasets.py +++ b/utils/datasets.py @@ -1,6 +1,6 @@ -import boto3 import os +import boto3 PUBLIC_BUCKET = "sagemaker-sample-files"