From 552e08c26000cf9979f3ea18c3a02f9a6609d76c Mon Sep 17 00:00:00 2001 From: Brad Macdonald Date: Thu, 26 Oct 2023 09:08:43 -0600 Subject: [PATCH] removes demo file and pandas --- Pipfile.lock | 86 +++----------------------------------------- demo_nf_logs.py | 96 ------------------------------------------------- setup.cfg | 1 - 3 files changed, 5 insertions(+), 178 deletions(-) delete mode 100644 demo_nf_logs.py diff --git a/Pipfile.lock b/Pipfile.lock index 35fa605..46d138a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1459,7 +1459,7 @@ "extras": [ "all" ], - "version": "==1.3.2.post1.dev12+gae50aee.d20231025" + "version": "==1.3.2.post1.dev19+gcdfdad2.d20231026" }, "pycparser": { "hashes": [ @@ -2524,10 +2524,10 @@ }, "asttokens": { "hashes": [ - "sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e", - "sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69" + "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24", + "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0" ], - "version": "==2.4.0" + "version": "==2.4.1" }, "async-timeout": { "hashes": [ @@ -4147,44 +4147,6 @@ "markers": "python_version >= '3.7'", "version": "==0.2.3" }, - "numpy": { - "hashes": [ - "sha256:06934e1a22c54636a059215d6da99e23286424f316fddd979f5071093b648668", - "sha256:1c59c046c31a43310ad0199d6299e59f57a289e22f0f36951ced1c9eac3665b9", - "sha256:1d1bd82d539607951cac963388534da3b7ea0e18b149a53cf883d8f699178c0f", - "sha256:1e11668d6f756ca5ef534b5be8653d16c5352cbb210a5c2a79ff288e937010d5", - "sha256:3649d566e2fc067597125428db15d60eb42a4e0897fc48d28cb75dc2e0454e53", - "sha256:59227c981d43425ca5e5c01094d59eb14e8772ce6975d4b2fc1e106a833d5ae2", - "sha256:6081aed64714a18c72b168a9276095ef9155dd7888b9e74b5987808f0dd0a974", - "sha256:6965888d65d2848e8768824ca8288db0a81263c1efccec881cb35a0d805fcd2f", - "sha256:76ff661a867d9272cd2a99eed002470f46dbe0943a5ffd140f49be84f68ffc42", - "sha256:78ca54b2f9daffa5f323f34cdf21e1d9779a54073f0018a3094ab907938331a2", - "sha256:82e871307a6331b5f09efda3c22e03c095d957f04bf6bc1804f30048d0e5e7af", - "sha256:8ab9163ca8aeb7fd32fe93866490654d2f7dda4e61bc6297bf72ce07fdc02f67", - "sha256:9696aa2e35cc41e398a6d42d147cf326f8f9d81befcb399bc1ed7ffea339b64e", - "sha256:97e5d6a9f0702c2863aaabf19f0d1b6c2628fbe476438ce0b5ce06e83085064c", - "sha256:9f42284ebf91bdf32fafac29d29d4c07e5e9d1af862ea73686581773ef9e73a7", - "sha256:a03fb25610ef560a6201ff06df4f8105292ba56e7cdd196ea350d123fc32e24e", - "sha256:a5b411040beead47a228bde3b2241100454a6abde9df139ed087bd73fc0a4908", - "sha256:af22f3d8e228d84d1c0c44c1fbdeb80f97a15a0abe4f080960393a00db733b66", - "sha256:afd5ced4e5a96dac6725daeb5242a35494243f2239244fad10a90ce58b071d24", - "sha256:b9d45d1dbb9de84894cc50efece5b09939752a2d75aab3a8b0cef6f3a35ecd6b", - "sha256:bb894accfd16b867d8643fc2ba6c8617c78ba2828051e9a69511644ce86ce83e", - "sha256:c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe", - "sha256:cd7837b2b734ca72959a1caf3309457a318c934abef7a43a14bb984e574bbb9a", - "sha256:cdd9ec98f0063d93baeb01aad472a1a0840dee302842a2746a7a8e92968f9575", - "sha256:d1cfc92db6af1fd37a7bb58e55c8383b4aa1ba23d012bdbba26b4bcca45ac297", - "sha256:d1d2c6b7dd618c41e202c59c1413ef9b2c8e8a15f5039e344af64195459e3104", - "sha256:d2984cb6caaf05294b8466966627e80bf6c7afd273279077679cb010acb0e5ab", - "sha256:d58e8c51a7cf43090d124d5073bc29ab2755822181fcad978b12e144e5e5a4b3", - "sha256:d78f269e0c4fd365fc2992c00353e4530d274ba68f15e968d8bc3c69ce5f5244", - "sha256:dcfaf015b79d1f9f9c9fd0731a907407dc3e45769262d657d754c3a028586124", - "sha256:e44ccb93f30c75dfc0c3aa3ce38f33486a75ec9abadabd4e59f114994a9c4617", - "sha256:e509cbc488c735b43b5ffea175235cec24bbc57b227ef1acc691725beb230d1c" - ], - "markers": "python_version == '3.11'", - "version": "==1.26.1" - }, "opentelemetry-api": { "hashes": [ "sha256:06abe351db7572f8afdd0fb889ce53f3c992dbf6f6262507b385cc1963e06983", @@ -4273,36 +4235,6 @@ "markers": "python_version >= '3.7'", "version": "==23.2" }, - "pandas": { - "hashes": [ - "sha256:02304e11582c5d090e5a52aec726f31fe3f42895d6bfc1f28738f9b64b6f0614", - "sha256:0489b0e6aa3d907e909aef92975edae89b1ee1654db5eafb9be633b0124abe97", - "sha256:05674536bd477af36aa2effd4ec8f71b92234ce0cc174de34fd21e2ee99adbc2", - "sha256:25e8474a8eb258e391e30c288eecec565bfed3e026f312b0cbd709a63906b6f8", - "sha256:29deb61de5a8a93bdd033df328441a79fcf8dd3c12d5ed0b41a395eef9cd76f0", - "sha256:366da7b0e540d1b908886d4feb3d951f2f1e572e655c1160f5fde28ad4abb750", - "sha256:3bcad1e6fb34b727b016775bea407311f7721db87e5b409e6542f4546a4951ea", - "sha256:4c3f32fd7c4dccd035f71734df39231ac1a6ff95e8bdab8d891167197b7018d2", - "sha256:4cdb0fab0400c2cb46dafcf1a0fe084c8bb2480a1fa8d81e19d15e12e6d4ded2", - "sha256:4f99bebf19b7e03cf80a4e770a3e65eee9dd4e2679039f542d7c1ace7b7b1daa", - "sha256:58d997dbee0d4b64f3cb881a24f918b5f25dd64ddf31f467bb9b67ae4c63a1e4", - "sha256:75ce97667d06d69396d72be074f0556698c7f662029322027c226fd7a26965cb", - "sha256:84e7e910096416adec68075dc87b986ff202920fb8704e6d9c8c9897fe7332d6", - "sha256:9e2959720b70e106bb1d8b6eadd8ecd7c8e99ccdbe03ee03260877184bb2877d", - "sha256:9e50e72b667415a816ac27dfcfe686dc5a0b02202e06196b943d54c4f9c7693e", - "sha256:a0dbfea0dd3901ad4ce2306575c54348d98499c95be01b8d885a2737fe4d7a98", - "sha256:b407381258a667df49d58a1b637be33e514b07f9285feb27769cedb3ab3d0b3a", - "sha256:b8bd1685556f3374520466998929bade3076aeae77c3e67ada5ed2b90b4de7f0", - "sha256:c1f84c144dee086fe4f04a472b5cd51e680f061adf75c1ae4fc3a9275560f8f4", - "sha256:c747793c4e9dcece7bb20156179529898abf505fe32cb40c4052107a3c620b49", - "sha256:cc1ab6a25da197f03ebe6d8fa17273126120874386b4ac11c1d687df288542dd", - "sha256:dc3657869c7902810f32bd072f0740487f9e030c1a3ab03e0af093db35a9d14e", - "sha256:f5ec7740f9ccb90aec64edd71434711f58ee0ea7f5ed4ac48be11cfa9abf7317", - "sha256:fecb198dc389429be557cde50a2d46da8434a17fe37d7d41ff102e3987fd947b", - "sha256:ffa8f0966de2c22de408d0e322db2faed6f6e74265aa0856f3824813cf124363" - ], - "version": "==2.1.1" - }, "pandocfilters": { "hashes": [ "sha256:0b679503337d233b4339a817bfc8c50064e2eff681314376a47cb582305a7a38", @@ -4483,7 +4415,7 @@ "extras": [ "all" ], - "version": "==1.3.2.post1.dev12+gae50aee.d20231025" + "version": "==1.3.2.post1.dev19+gcdfdad2.d20231026" }, "pycodestyle": { "hashes": [ @@ -5396,14 +5328,6 @@ "markers": "python_version >= '3.8'", "version": "==4.8.0" }, - "tzdata": { - "hashes": [ - "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a", - "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda" - ], - "markers": "python_version >= '2'", - "version": "==2023.3" - }, "uc-micro-py": { "hashes": [ "sha256:30ae2ac9c49f39ac6dce743bd187fcd2b574b16ca095fa74cd9396795c954c54", diff --git a/demo_nf_logs.py b/demo_nf_logs.py deleted file mode 100644 index a3a2902..0000000 --- a/demo_nf_logs.py +++ /dev/null @@ -1,96 +0,0 @@ -import asyncio -from typing import List - -import pandas as pd -from metaflow import FlowSpec, Parameter, step - -from orca.services.nextflowtower import NextflowTowerOps -from orca.services.nextflowtower.models import WorkflowTask - - -class LogsFlow(FlowSpec): - """Retrieve execution logs and failure reasons for all - failed tasks in a workflow.""" - - tower = NextflowTowerOps() - - workflow_id = Parameter( - "workflow_id", - type=str, - help="Nextflow workflow ID", - ) - - # Example error reasons for nf-histoqc - log_reasons = [ - "NO tissue remains detectable!", - "NO tissue remains detectable in mask!", - "Inappropriate argument value (of correct type)", - ] - - async def get_logs(self, task: WorkflowTask): - """Retrieve execution logs for an individual task.""" - print(f"Getting logs for task: {task.task_id}") - return { - "task_id": task.task_id, - "tag": task.tag, - "execution_log": self.tower.get_task_logs( - workflow_id=self.workflow_id, task_id=task.task_id - ), - } - - async def retrieve_workflow_task_logs(self): - """Asynchronously retrieve execution logs for all failed tasks in a workflow.""" - print("Getting workflow tasks") - tasks = self.tower.get_workflow_tasks(workflow_id=self.workflow_id) - failed_tasks = [task for task in tasks if task.status == "FAILED"] - print("Getting logs for each failed task") - task_log_list = await asyncio.gather( - *[self.get_logs(task) for task in failed_tasks] - ) - return task_log_list - - def determine_failure_reasons(self, task_log_list: List[dict]): - """Check for failure reasons in execution logs. - Attribute failure reasons to each task.""" - for task in task_log_list: - failure_reasons = [] - for reason in self.log_reasons: - if reason in task["execution_log"]: - failure_reasons.append(reason) - task["failure_reasons"] = ", ".join(failure_reasons) - if not task["failure_reasons"]: - task["failure_reasons"] = "Failure Reason Undetermined" - task.pop("execution_log") - - @step - def start(self): - """Entry point.""" - self.next(self.get_task_logs) - - @step - def get_task_logs(self): - """Collect task logs for a workflow.""" - self.task_log_list = asyncio.run(self.retrieve_workflow_task_logs()) - self.next(self.compile_failure_reasons) - - @step - def compile_failure_reasons(self): - """Compile failure reasons for each task.""" - self.determine_failure_reasons(self.task_log_list) - self.next(self.export_results) - - @step - def export_results(self): - """Export results to CSV.""" - logs_df = pd.DataFrame(self.task_log_list) - logs_df.to_csv("task_logs.csv", index=False) - self.next(self.end) - - @step - def end(self): - """End point.""" - print("Done") - - -if __name__ == "__main__": - LogsFlow() diff --git a/setup.cfg b/setup.cfg index 8790d81..42bcda2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -101,7 +101,6 @@ dev = typing-extensions~=4.5 metaflow~=2.9 s3fs~=2023.5 - pandas~=2.1.1 [options.entry_points] # Add here console scripts like: