From bbc52c8261abceb658e34c97f83310a4144c3b14 Mon Sep 17 00:00:00 2001 From: Stephen Nielson Date: Mon, 6 Nov 2023 12:40:11 -0500 Subject: [PATCH] Asynchronous Export Progress Updates. (#6977) * Asynchronous Export Progress Updates. This change adds progress indicators to the export process. It will loop through each batch individually and process the batch while reporting progress in 5 second intervals on the currently processed batch. If a batch fails it can be retried. Added a new column to the task to hold the current result progress. Changed up the index page to function as both the main index and an ajax controller. Probably could break this apart but for expediency did it this way. * Fix escaping, deprecation notices. Had an escaping issue on the twig file. There were a number of dynamic properties created in php 8.2. Thanks to @stephen.waite for pointing them out. --- .../oe-module-ehi-exporter/CHANGELOG.md | 2 + .../oe-module-ehi-exporter/info.txt | 2 +- .../public/assets/js/ehi-exporter.js | 254 ++++++++++++++++++ .../oe-module-ehi-exporter/public/index.php | 75 ++++-- .../oe-module-ehi-exporter/src/Bootstrap.php | 5 + .../src/Models/EhiExportJobTask.php | 25 ++ .../src/Models/ExportResult.php | 13 + .../src/Models/ExportState.php | 6 + .../src/Models/ExportTableResult.php | 6 + .../src/Services/EhiExportJobService.php | 32 +++ .../src/Services/EhiExportJobTaskService.php | 47 ++++ .../src/Services/EhiExporter.php | 87 ++++++ .../ExportTableDefinition.php | 2 + .../oe-module-ehi-exporter/table.sql | 5 + .../ehi-exporter-tasks.html.twig | 103 +++++++ .../ehi-exporter.html.twig | 33 +++ 16 files changed, 680 insertions(+), 17 deletions(-) create mode 100644 interface/modules/custom_modules/oe-module-ehi-exporter/public/assets/js/ehi-exporter.js create mode 100644 interface/modules/custom_modules/oe-module-ehi-exporter/templates/oe-module-ehi-exporter/ehi-exporter-tasks.html.twig diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/CHANGELOG.md b/interface/modules/custom_modules/oe-module-ehi-exporter/CHANGELOG.md index 84abd25f18a..6a98cee685e 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/CHANGELOG.md +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/CHANGELOG.md @@ -1 +1,3 @@ +# 1.0.1 +Implemented asynchronous progress status and progress updates for batch exports # 1.0.0 Initial release of the EHI exporter diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/info.txt b/interface/modules/custom_modules/oe-module-ehi-exporter/info.txt index 47da6ac505c..fa20e8b4ea1 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/info.txt +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/info.txt @@ -1 +1 @@ -Electronic Health Information Exporter v1.0.0 +Electronic Health Information Exporter v1.0.1 diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/public/assets/js/ehi-exporter.js b/interface/modules/custom_modules/oe-module-ehi-exporter/public/assets/js/ehi-exporter.js new file mode 100644 index 00000000000..7f979a6973d --- /dev/null +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/public/assets/js/ehi-exporter.js @@ -0,0 +1,254 @@ +(function(window, oeExporter) { + + class ExporterState { + taskIds = []; + currentTaskIndex = 0; + ajaxUrl = ""; + csrfToken = ""; + + currentTaskPollingInterval = 0; + currentTaskPollingTimeout = 5000; + + startExport() { + this.currentTaskIndex = -1; + this.runNextExport(); + + } + runNextExport() { + this.currentTaskIndex++; + if (this.currentTaskIndex < this.taskIds.length) { + let callBack = function() { + this.startExportRequestForTask(this.taskIds[this.currentTaskIndex]); + }; + // just give a way to break promise callback chain + setTimeout(callBack.bind(this), 100); + } else { + // if we've finished everything... then we should clear the polling interval + this.clearPollingForExportStatus(); + } + } + + showErrorCardForTaskId(taskId, errorMessage='') { + // hide the processing div template node + let processingTask = document.querySelector(".template-task-processing[data-task-id='" + taskId + "']"); + if (!processingTask) { + console.error("Could not find processing task for task id: " + taskId); + return; + } + processingTask.classList.add("d-none"); + + // grab the error div template node + let errorTaskTemplate = document.querySelector(".template-task-failed"); + let errorTask = errorTaskTemplate.cloneNode(true); + + // populate the error div template node with the task id, the patient pids + errorTask.querySelector(".taskId").innerText = taskId; + errorTask.dataset['taskId'] = taskId; + // show the error div template node + if (errorMessage) { + errorTask.querySelector(".errorMessage").innerText = errorMessage; + } + // TODO: @adunsulag need to handle what happens when they retry the export and we need to move on to the next + // possible export. Should we disable all of the buttons until the export has processed everything... + errorTask.querySelector(".btn-retry-export-task").addEventListener("click", () => { + errorTask.remove(); + this.startExportRequestForTask(taskId); + }); + errorTask.classList.remove("d-none"); + processingTask.insertAdjacentElement("afterend", errorTask); + processingTask.remove(); // remove the processing node at the end since we don't need it. + } + + startExportRequestForTask(taskId) { + // hide the queued div template node + this.showProcessingCardForTaskId(taskId, {taskId: taskId}); + // send off the ajax request to start the export + let formParams = new FormData(); + formParams.set("taskId", taskId); + formParams.set("submit", "Start Export"); + formParams.set("action", "startExport"); + formParams.set("_token", this.csrfToken); + window.top.restoreSession(); // make sure the session is populated before we send off an ajax request + let resultPromise = window.fetch(this.ajaxUrl, { + method: 'POST', + body: new URLSearchParams(formParams) + }); + let exporterState = this; + resultPromise.then(response => { + if (response.ok) { + return response.json(); + } else { + throw new Error('Failed to receive response from server'); + } + }) + .then(data => { + if (data.status == 'failed') { + this.showErrorCardForTaskId(taskId, data.error_message); + // move onto the next task in the queue + return exporterState.runNextExport(); + } else { + this.showSuccessCardForTaskId(taskId, data); + return exporterState.runNextExport(); + } + }) + .catch(error => { + console.log(error); + this.showErrorCardForTaskId(taskId, error.message); + return exporterState.runNextExport(); + }); + // TODO: @adunsulag start the polling for the export status + this.startPollingForExportStatus(taskId); + } + + startPollingForExportStatus(taskId) { + if (this.currentTaskPollingInterval > 0) { + this.clearPollingForExportStatus(); + } + this.currentTaskPollingInterval = setInterval(this.pollForExportStatus.bind(this), this.currentTaskPollingTimeout, taskId); + } + clearPollingForExportStatus() { + clearInterval(this.currentTaskPollingInterval); + } + pollForExportStatus(taskId) { + let formParams = new FormData(); + formParams.set("taskId", taskId); + formParams.set("submit", "Get Status"); + formParams.set("action", "statusUpdate"); + formParams.set("_token", this.csrfToken); + window.top.restoreSession(); // make sure the session is populated before we send off an ajax request + let resultPromise = window.fetch(this.ajaxUrl, { + method: 'POST', + body: new URLSearchParams(formParams) + }); + let exporterState = this; + resultPromise.then(response => { + if (response.ok) { + return response.json(); + } else { + throw new Error('Failed to receive response from server'); + } + }) + .then(data => { + if (data.status == 'failed') { + this.showErrorCardForTaskId(taskId, data.error_message); + } else if (data.status == 'completed') { + this.showSuccessCardForTaskId(taskId, data); + } else { + this.showProcessingCardForTaskId(taskId, data); + } + }) + .catch(error => { + this.showErrorCardForTaskId(taskId, error.message); + console.log(error); + }); + } + + showProcessingCardForTaskId(taskId, data) { + let queuedTask = document.querySelector(".template-task-queued[data-task-id='" + taskId + "']"); + queuedTask.classList.add("d-none"); + + // if there are any existing processing tasks we need to remove them + let processingTasks = document.querySelectorAll(".template-task-processing[data-task-id='" + taskId + "']"); + processingTasks.forEach(function (task) { + task.remove(); + }); + + // grab the processing div template node + let processingTaskTemplate = document.querySelector(".template-task-processing"); + let processingTask = processingTaskTemplate.cloneNode(true); + + // populate the processing div template node with the task id, the patient pids + processingTask.querySelector(".taskId").innerText = taskId; + processingTask.dataset['taskId'] = taskId; + processingTask.querySelector(".patientPids").innerText = queuedTask.querySelector(".patientPids").innerText; + // show the processing div template node + if (data.exportedResult) { + this.populateCardWithResultData(processingTask, taskId, data); + } + processingTask.classList.remove("d-none"); + queuedTask.insertAdjacentElement("afterend", processingTask); + } + + populateCardWithResultData(cardNode, taskId, data) { + // .exportedTablesList needs to be looped on the data.exportedResult table + let totalTablesExported = 0; + let totalRecordsExported = 0; + if (data.exportedResult) { + if (data.exportedResult.exportedTables) { + let tableNames = Object.keys(data.exportedResult.exportedTables); + totalTablesExported = tableNames.length; + let itemTemplate = cardNode.querySelector(".exportedTableListItem"); + let templateParent = itemTemplate.parentNode; + for (let i = 0; i < totalTablesExported; i++) { + let tableItem = data.exportedResult.exportedTables[tableNames[i]]; + let exportedTableListItem = itemTemplate.cloneNode(true); + exportedTableListItem.classList.remove("d-none"); + exportedTableListItem.querySelector(".exportedTableName").innerText = tableItem.tableName + ".csv"; + exportedTableListItem.querySelector(".exportedTableCount").innerText = tableItem.count; + totalRecordsExported += tableItem.count; + templateParent.appendChild(exportedTableListItem); + } + } + if (data.exportedResult.exportedDocumentCount >= 0) { + cardNode.querySelector(".documentsExportedCount").innerText = data.exportedResult.exportedDocumentCount; + } + } + cardNode.querySelector(".total-tables-exported").innerText = totalTablesExported; + cardNode.querySelector(".total-records-exported").innerText = totalRecordsExported; + + if (data.includePatientDocuments) { + cardNode.querySelector(".documentsExportedSection").classList.remove("d-none"); + } + } + + showSuccessCardForTaskId(taskId, data) { + let processingTask = document.querySelector(".template-task-processing[data-task-id='" + taskId + "']"); + processingTask.classList.add("d-none"); + + // grab the error div template node + let successTemplate = document.querySelector(".template-result-success"); + let successTask = successTemplate.cloneNode(true); + + successTask.querySelector(".taskId").innerText = taskId; + successTask.dataset['taskId'] = taskId; + + // .download-link .download-link-name need to be populated + successTask.querySelector(".download-link-name").innerText = data.downloadName; + successTask.querySelector(".download-link").href = data.downloadLink; + successTask.querySelector(".download-link").addEventListener('click', function() { + window.top.restoreSession(); // make sure the session is populated before the download starts + }); + // .hash-algo-title, .hash-text need to be populated + successTask.querySelector(".hash-algo-title").innerText = data.hashAlgoTitle; + successTask.querySelector(".hash-text").innerText = data.hash; + this.populateCardWithResultData(successTask, taskId, data); + successTask.classList.remove("d-none"); + processingTask.insertAdjacentElement("afterend", successTask); + processingTask.remove(); // remove the processing node at the end since we don't need it. + } + } + let exporterState; + + function displayExportStartDialog(dialogId) { + let container = document.getElementById(dialogId); + let modal = new bootstrap.Modal(container, {keyboard: false, focus: true, backdrop: 'static'}); + modal.show(); + } + + oeExporter.displayExportStartDialog = displayExportStartDialog; + oeExporter.startTaskExports = function (ajaxUrl, csrfToken) { + let queuedTasks = document.querySelectorAll(".template-task-queued[data-task-id]"); + let queuedTaskIds = []; + queuedTasks.forEach(function (task) { + queuedTaskIds.push(+task.dataset.taskId); + }); + if (queuedTaskIds.length > 0) { + exporterState = new ExporterState(); + exporterState.ajaxUrl = ajaxUrl; + exporterState.csrfToken = csrfToken; + exporterState.taskIds = queuedTaskIds; + exporterState.startExport(); + } + }; + window.oeExporter = oeExporter; +})(window, window.oeExporter || window.top.oeExporter || {}); diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/public/index.php b/interface/modules/custom_modules/oe-module-ehi-exporter/public/index.php index cf3d7f82b35..0d2e5b64b38 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/public/index.php +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/public/index.php @@ -27,6 +27,7 @@ $defaultZipSize = 500; // size in MB $memoryLimitUpdated = false; $errorMessage = ""; +$twig = $bootstrap->getTwig(); if (isset($_POST['submit'])) { try { if (!CsrfUtils::verifyCsrfToken($_POST['_token'] ?? '')) { @@ -36,26 +37,68 @@ $pid = intval($_POST['pid'] ?? 0); $includeDocuments = intval($_POST['include_documents'] ?? 0) === 1; $fileSizeLimit = intval($_POST['file_size_limit'] ?? 500); - if ($pid > 0) { - $result = $exporter->exportPatient($pid, $includeDocuments, $fileSizeLimit); - } else { - $result = $exporter->exportAll($includeDocuments, $fileSizeLimit); + if ($_POST['action'] == 'createExport') { + if ($pid > 0) { + $job = $exporter->createExportPatientJob($pid, $includeDocuments, $fileSizeLimit); +// $result = $exporter->exportPatient($pid, $includeDocuments, $fileSizeLimit); + } else { + $job = $exporter->createExportPatientPopulationJob($includeDocuments, $fileSizeLimit); +// $result = $exporter->exportAll($includeDocuments, $fileSizeLimit); + } + echo $twig->render( + Bootstrap::MODULE_NAME . DIRECTORY_SEPARATOR . 'ehi-exporter-tasks.html.twig', + [ + 'result' => $result + ,'job' => $job + , 'assetPath' => $bootstrap->getAssetPath() + ,'postUrl' => $GLOBALS['webroot'] . Bootstrap::MODULE_INSTALLATION_PATH . '/' + . Bootstrap::MODULE_NAME . '/public/index.php' + ] + ); + // TODO: @adunsulag we really should move all of this into a controller to be cleaner, but we are time crunched here. + } else if ($_POST['action'] == 'startExport') { + try { + $taskId = intval($_POST['taskId'] ?? 0); + $task = $exporter->runExportTask($taskId); + echo json_encode($task->getJSON()); + } catch (\Exception $exception) { + $errorMessage = $exception->getMessage(); + $bootstrap->getLogger()->errorLogCaller($errorMessage, ['trace' => $exception->getTraceAsString()]); + echo json_encode(['status' => 'failed', 'error_message' => $errorMessage, 'taskId' => $taskId]); + } + exit; + } else if ($_POST['action'] == 'statusUpdate') { + try { + $taskId = intval($_POST['taskId'] ?? 0); + $task = $exporter->getExportTaskForStatusUpdate($taskId); + // will already have the encoded progress results in the task + echo json_encode($task->getJSON()); + } catch (\Exception $exception) { + $errorMessage = $exception->getMessage(); + $bootstrap->getLogger()->errorLogCaller($errorMessage, ['trace' => $exception->getTraceAsString()]); + echo json_encode(['status' => 'failed', 'error_message' => $errorMessage, 'taskId' => $taskId]); + } + exit; } } catch (\Exception $exception) { $errorMessage = $exception->getMessage(); $bootstrap->getLogger()->errorLogCaller($errorMessage, ['trace' => $exception->getTraceAsString()]); } +} else { + $exportSizeSettings = $exporter->getExportSizeSettings($defaultZipSize); + + echo $twig->render( + Bootstrap::MODULE_NAME . DIRECTORY_SEPARATOR . 'ehi-exporter.html.twig', + [ + 'result' => $result + , 'exportSizeSettings' => $exportSizeSettings + , 'memoryLimitUpdated' => $memoryLimitUpdated + // TODO: @adunsulag add most recent exports here. + , 'errorMessage' => $errorMessage + , 'postAction' => $_SERVER['PHP_SELF'] + , 'site_addr_oath' => trim($GLOBALS['site_addr_oath'] ?? '') + , 'assetPath' => $bootstrap->getAssetPath() + ] + ); } -$exportSizeSettings = $exporter->getExportSizeSettings($defaultZipSize); -$twig = $bootstrap->getTwig(); -echo $twig->render( - Bootstrap::MODULE_NAME . DIRECTORY_SEPARATOR . 'ehi-exporter.html.twig', - [ - 'result' => $result, 'exportSizeSettings' => $exportSizeSettings, 'memoryLimitUpdated' => $memoryLimitUpdated - // TODO: @adunsulag add most recent exports here. - ,'errorMessage' => $errorMessage - ,'postAction' => $_SERVER['PHP_SELF'] - ,'site_addr_oath' => trim($GLOBALS['site_addr_oath'] ?? '') - ] -); exit; diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Bootstrap.php b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Bootstrap.php index 5cf9f7c4536..5b5a7877e0d 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Bootstrap.php +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Bootstrap.php @@ -99,6 +99,11 @@ public static function instantiate(EventDispatcher $eventDispatcher, Kernel $ker return self::$instance; } + public function getAssetPath() + { + return $GLOBALS['webroot'] . self::MODULE_INSTALLATION_PATH . $this->moduleDirectoryName . "/public/assets/"; + } + public function getLogger() { return new SystemLogger(); diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/EhiExportJobTask.php b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/EhiExportJobTask.php index 12869fb9c06..dc7c11e6e77 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/EhiExportJobTask.php +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/EhiExportJobTask.php @@ -96,4 +96,29 @@ public function hasPatientIds() { return !empty($this->pids); } + + public function getJSON() + { + $data = [ + 'status' => $this->status + , 'taskId' => $this->ehi_task_id + , 'includePatientDocuments' => false + ]; + if (isset($this->ehiExportJob)) { + $data['includePatientDocuments'] = $this->ehiExportJob->include_patient_documents; + } + if (isset($this->exportedResult)) { + // so we can update progress on the client side + $data['exportedResult'] = $this->exportedResult; + } + if ($this->status == 'completed') { + $data['hashAlgoTitle'] = $this->document->get_hash_algo_title(); + $data['hash'] = $this->document->get_hash(); + $data['downloadLink'] = $this->exportedResult->downloadLink; + $data['downloadName'] = $this->document->get_name(); + } else if ($this->status == 'failed') { + $data['errorMessage'] = $this->error_message; + } + return $data; + } } diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportResult.php b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportResult.php index 3bca5cec07e..a64b7988403 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportResult.php +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportResult.php @@ -24,4 +24,17 @@ class ExportResult public $exportedTables; public $exportedDocumentCount = 0; + + public function fromJSON(array $exportedResult) + { + $this->downloadLink = $exportedResult['downloadLink'] ?? ''; + $this->exportedDocumentCount = $exportedResult['exportedDocumentCount'] ?? 0; + if (is_array($exportedResult['exportedTables'] ?? [])) { + foreach ($exportedResult['exportedTables'] as $exportedTable) { + $exportTableResult = new ExportTableResult(); + $exportTableResult->fromJSON($exportedTable); + $this->exportedTables[] = $exportTableResult; + } + } + } } diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportState.php b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportState.php index 1122f14015c..39c4db5b680 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportState.php +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportState.php @@ -51,6 +51,12 @@ class ExportState */ private string $tempDir; + private \SimpleXMLElement $metaNode; + + private ExportKeyDefinitionFilterer $keyFilterer; + + private EhiExportJobTask $jobTask; + public function __construct(SystemLogger $logger, \SimpleXMLElement $tableNode, \SimpleXMLElement $metaNode, EhiExportJobTask $jobTask) { $this->rootNode = $tableNode; diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportTableResult.php b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportTableResult.php index 5c41fe12110..ab3272223b3 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportTableResult.php +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Models/ExportTableResult.php @@ -17,4 +17,10 @@ class ExportTableResult { public $count; public $tableName; + + public function fromJSON(array $exportedTable) + { + $this->tableName = $exportedTable['tableName'] ?? ''; + $this->count = $exportedTable['count'] ?? 0; + } } diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExportJobService.php b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExportJobService.php index 7153f8462ab..590a6415a3d 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExportJobService.php +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExportJobService.php @@ -17,6 +17,7 @@ use OpenEMR\Modules\EhiExporter\Models\EhiExportJob; use OpenEMR\Modules\EhiExporter\Models\EhiExportJobTask; use OpenEMR\Services\BaseService; +use OpenEMR\Validators\ProcessingResult; class EhiExportJobService extends BaseService { @@ -64,8 +65,10 @@ public function insert(EhiExportJob $job) public function update(EhiExportJob $job) { $sql = "UPDATE " . self::TABLE_NAME . " SET `status`= ?" . ($job->isCompleted() ? ",`completion_date`= NOW() " : ""); + $sql .= " WHERE `ehi_export_job_id` = ? "; $bind = [ $job->getStatus() + ,$job->getId() ]; QueryUtils::startTransaction(); try { @@ -78,4 +81,33 @@ public function update(EhiExportJob $job) } return $job; } + + public function getJobById(?int $ehi_export_job_id, $loadPatients = false): ?EhiExportJob + { + $ehiExportJob = null; + $processingResult = $this->search(['ehi_export_job_id' => $ehi_export_job_id]); + if ($processingResult->hasData()) { + $record = ProcessingResult::extractDataArray($processingResult)[0]; + $ehiExportJob = new EhiExportJob(); + $ehiExportJob->setId($record['ehi_export_job_id']); + $ehiExportJob->setStatus($record['status']); + $ehiExportJob->setDocumentLimitSize($record['document_limit_size']); + $ehiExportJob->include_patient_documents = $record['include_patient_documents'] == 1; + $ehiExportJob->user_id = $record['user_id']; + $ehiExportJob->uuid = $record['uuid']; + + // now we need to grab all of the patient ids here + if ($loadPatients) { + $patientPids = $this->getPatientPidsForJobId($ehiExportJob->getId()); + $ehiExportJob->addPatientIdList($patientPids); + } + } + return $ehiExportJob; + } + + private function getPatientPidsForJobId(?int $jobId) + { + return QueryUtils::fetchTableColumn("SELECT pid FROM " . self::TABLE_NAME_PATIENT_JOIN_TABLE + . " WHERE ehi_export_job_id = ?", 'pid', [$jobId]); + } } diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExportJobTaskService.php b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExportJobTaskService.php index 0d529b04053..1f82432f446 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExportJobTaskService.php +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExportJobTaskService.php @@ -15,7 +15,9 @@ use OpenEMR\Common\Database\QueryUtils; use OpenEMR\Modules\EhiExporter\Models\EhiExportJobTask; +use OpenEMR\Modules\EhiExporter\Models\ExportResult; use OpenEMR\Services\BaseService; +use OpenEMR\Validators\ProcessingResult; class EhiExportJobTaskService extends BaseService { @@ -27,6 +29,36 @@ public function __construct() parent::__construct(self::TABLE_NAME); } + public function getTaskFromId(int $taskId, bool $loadPatients = true): ?EhiExportJobTask + { + $ehiExportJobTask = null; + $processingResult = $this->search(['ehi_task_id' => $taskId]); + if ($processingResult->hasData()) { + $taskRecord = ProcessingResult::extractDataArray($processingResult)[0]; + $ehiExportJobTask = new EhiExportJobTask(); + $ehiExportJobTask->ehi_task_id = $taskRecord['ehi_task_id']; + $ehiExportJobTask->ehi_export_job_id = $taskRecord['ehi_export_job_id']; + $ehiExportJobTask->export_document_id = $taskRecord['export_document_id']; + $ehiExportJobTask->error_message = $taskRecord['error_message']; + $ehiExportJobTask->setStatus($taskRecord['status']); + if (isset($ehiExportJobTask->export_document_id)) { + $ehiExportJobTask->document = new \Document($ehiExportJobTask->export_document_id); + } + + if (isset($taskRecord['exported_result'])) { + $exportedResult = json_decode($taskRecord['exported_result'], true); + $exportResult = new ExportResult(); + $exportResult->fromJSON($exportedResult); + $ehiExportJobTask->exportedResult = $exportResult; + } + if ($loadPatients) { + $patientPids = $this->getPatientPidsForJobTaskId($ehiExportJobTask->getId()); + $ehiExportJobTask->addPatientIdList($patientPids); + } + } + return $ehiExportJobTask; + } + public function insert(EhiExportJobTask $task) { $sql = "INSERT INTO " . self::TABLE_NAME . " (`ehi_export_job_id`, `status`) " @@ -78,6 +110,15 @@ public function update(EhiExportJobTask $task) } else { $sql .= ",error_message= NULL "; } + if (isset($task->exportedResult)) { + $sql .= ",exported_result= ? "; + $bind[] = json_encode($task->exportedResult); + } else { + $sql .= ",exported_result= NULL "; + } + + $sql .= " WHERE ehi_task_id = ? "; + $bind[] = $task->getId(); QueryUtils::startTransaction(); try { @@ -90,4 +131,10 @@ public function update(EhiExportJobTask $task) } return $task; } + + private function getPatientPidsForJobTaskId(?int $taskId) + { + return QueryUtils::fetchTableColumn("SELECT pid FROM " . self::TABLE_NAME_PATIENT_JOIN_TABLE + . " WHERE ehi_task_id = ?", 'pid', [$taskId]); + } } diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExporter.php b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExporter.php index 972c286527e..a2a1a199c6f 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExporter.php +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/src/Services/EhiExporter.php @@ -64,6 +64,8 @@ class EhiExporter private SystemLogger $logger; private EhiExportJobTaskService $taskService; private CryptoGen $cryptoGen; + private EhiExportJobService $jobService; + public function __construct(private $modulePublicDir, private $modulePublicUrl, private $xmlConfigPath, private Environment $twig) { $this->logger = new SystemLogger(); @@ -73,6 +75,50 @@ public function __construct(private $modulePublicDir, private $modulePublicUrl, $this->cryptoGen = new CryptoGen(); } + + public function createExportPatientJob(int $pid, bool $includePatientDocuments, int $defaultZipSize) + { + $patientPids = [$pid]; + $job = null; + try { + $job = $this->createJobForRequest($patientPids, $includePatientDocuments, $defaultZipSize); + } catch (\Exception $exception) { + if ($job !== null) { + $job->setStatus("failed"); + try { + $this->jobService->update($job); + } catch (\Exception $exception) { + $this->logger->errorLogCaller("Failed to mark job as failed ", [$exception->getMessage()]); + return $job; + } + } + throw $exception; + } + return $job; + } + + public function createExportPatientPopulationJob(bool $includePatientDocuments, int $defaultZipSize): EhiExportJob + { + $job = null; + try { + $sql = "SELECT pid FROM patient_data"; // We do everything here + $patientPids = QueryUtils::fetchTableColumn($sql, 'pid', []); + $job = $this->createJobForRequest($patientPids, $includePatientDocuments, $defaultZipSize); + } catch (\Exception $exception) { + if ($job !== null) { + $job->setStatus("failed"); + try { + $this->jobService->update($job); + } catch (\Exception $exception) { + $this->logger->errorLogCaller("Failed to mark job as failed ", [$exception->getMessage()]); + return $job; + } + } + throw $exception; + } + return $job; + } + public function exportPatient(int $pid, bool $includePatientDocuments, $defaultZipSize) { $patientPids = [$pid]; @@ -138,6 +184,16 @@ private function createJobForRequest(array &$patientPids, bool $includePatientDo $job->addPatientIdList($patientPids); $job->setDocumentLimitSize($defaultZipSize * 1024 * 1024); // set our max size in bytes $updatedJob = $this->jobService->insert($job); + + // now create the job tasks + $jobTasks = $this->createExportTasksFromJob($job); + if (empty($jobTasks)) { + $job->setStatus("failed"); // no tasks to process, we mark as failed. + } else { + foreach ($jobTasks as $jobTask) { + $job->addJobTask($jobTask); + } + } return $updatedJob; } @@ -356,6 +412,8 @@ private function exportBreadthAlgorithm(EhiExportJobTask $jobTask): EhiExportJob // write out the csv file $this->writeCsvFile($jobTask, $records, $tableDefinition->table, $exportState->getTempSysDir(), $tableDefinition->getColumnNames()); $exportState->addExportResultTable($tableDefinition->table, count($records)); + $jobTask->exportedResult = $exportState->getExportResult(); + $this->taskService->update($jobTask); // for progress updates $tableDefinition->setHasNewData(false); if (!empty($keyDefinitions)) { foreach ($keyDefinitions['keys'] as $keyDefinition) { @@ -723,4 +781,33 @@ private function createExportTasksFromJobWithoutDocuments(EhiExportJob $job, arr } return $tasks; } + + public function runExportTask(int $taskId): EhiExportJobTask + { + $task = $this->taskService->getTaskFromId($taskId); + if (empty($task)) { + throw new \InvalidArgumentException("Invalid task id"); + } + $job = $this->jobService->getJobById($task->ehi_export_job_id); + if (empty($job)) { + throw new \InvalidArgumentException("Invalid job id. This should never happen and indicates there is a system error"); + } + $task->ehiExportJob = $job; + if ($task->getStatus() == 'completed') { + // if the task is already complete we are just going to return it. + return $task; + } + $task->setStatus('processing'); + $updatedTask = $this->taskService->update($task); + return $this->processJobTask($updatedTask); + } + + public function getExportTaskForStatusUpdate(int $taskId) + { + $task = $this->taskService->getTaskFromId($taskId); + if (empty($task)) { + throw new \InvalidArgumentException("Invalid task id"); + } + return $task; + } } diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/src/TableDefinitions/ExportTableDefinition.php b/interface/modules/custom_modules/oe-module-ehi-exporter/src/TableDefinitions/ExportTableDefinition.php index 2b34559e7ca..9553f35ff50 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/src/TableDefinitions/ExportTableDefinition.php +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/src/TableDefinitions/ExportTableDefinition.php @@ -38,6 +38,8 @@ class ExportTableDefinition */ private array $tableColumnNames; + private array $primaryKeys; + public function __construct(?string $table = null, array $pks = []) { $this->table = $table; diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/table.sql b/interface/modules/custom_modules/oe-module-ehi-exporter/table.sql index 2037d1191f0..3e15f8f402a 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/table.sql +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/table.sql @@ -40,6 +40,7 @@ CREATE TABLE `ehi_export_job_tasks`( `completion_date` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, `status` VARCHAR(20) NOT NULL DEFAULT 'pending' COMMENT 'pending=task export has not started, processing=task export in progress, failed=error occurred in the task, completed=export task completed without errors', `error_message` TEXT COMMENT 'The error that occurred in the export process, only populated if status=failed', + `exported_result` TEXT COMMENT 'The JSON encoded result of the export process, populated during processing for status updates', PRIMARY KEY(`ehi_task_id`), CONSTRAINT `FK_task_ehi_export_job_id` FOREIGN KEY (`ehi_export_job_id`) REFERENCES `ehi_export_job`(`ehi_export_job_id`) ON DELETE RESTRICT ON UPDATE RESTRICT, CONSTRAINT `FK_task_ehi_export_document_id` FOREIGN KEY (`export_document_id`) REFERENCES `documents`(`id`) ON DELETE RESTRICT ON UPDATE RESTRICT @@ -62,3 +63,7 @@ INSERT INTO categories(`id`,`name`, `value`, `parent`, `lft`, `rght`, `aco_spec` UPDATE categories SET rght = rght + 2 WHERE name = 'Categories'; UPDATE categories_seq SET id = (select MAX(id) from categories); #EndIf + +#IfMissingColumn ehi_export_job_tasks exported_result +ALTER TABLE ehi_export_job_tasks ADD COLUMN `exported_result` TEXT COMMENT 'The JSON encoded result of the export process, populated during processing for status updates'; +#Endif diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/templates/oe-module-ehi-exporter/ehi-exporter-tasks.html.twig b/interface/modules/custom_modules/oe-module-ehi-exporter/templates/oe-module-ehi-exporter/ehi-exporter-tasks.html.twig new file mode 100644 index 00000000000..8370d844d9c --- /dev/null +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/templates/oe-module-ehi-exporter/ehi-exporter-tasks.html.twig @@ -0,0 +1,103 @@ +{# +# Handles the asynchronous task processing of the EHI Export +# +# @package OpenEMR +# @link http://www.open-emr.org +# +# @author Stephen Nielson + + {{ "Electronic Health Information Export"|xlt }} + {{ setupHeader() }} +{# #} + + + +

{{ "Electronic Health Information Export"|xlt }} {{ "Go Back"|xlt }}

+
+
+
+

{{ "Export Task:"|xlt }} - {{ "Failed"|xlt }}

+
+
+ +
+ {{ "Check system logs as export failed to process"|xlt }} +
+
+
+
+
+
+

{{ "Export Task:"|xlt }} - {{ "Processing"|xlt }}

+
+
+

{{ "Export Task is being processed"|xlt }}

+
{{ "Total Tables Exported:"|xlt }} 0
+
{{ "Total Records Exported:"|xlt }} 0
+
{{ "Total Patient Documents Exported:"|xlt }} 0
+
+ + {{ "Expand for Export Zip Details"|xlt }} + +

{{ "Patient Records Being Exported"|xlt }}

+
+

{{ "Records Exported"|xlt }}

+
    +
  1. :
    2. +
+
+
+
+
+
+

{{ "Export Task:"|xlt }} - {{ "Completed"|xlt }}

+
+
+

- {{ "Download"|xlt }}

+
:
+
{{ "Total Tables Exported:"|xlt }}
+
{{ "Total Records Exported:"|xlt }}
+
{{ "Total Patient Documents Exported:"|xlt }}
+
+ + {{ "Expand for Export Zip Details"|xlt }} + +

{{ "Records Exported"|xlt }}

+
    +
  1. :
    2. +
+
+
+
+ {% for task in job.getJobTasks() %} +
+
+

{{ "Export Task:"|xlt }} {{ task.getId()|text }} - {{ "Pending"|xlt }}

+
+
+

{{ "Patients to be exported:"|xlt }} {{ task.getPatientIds()|length|text }}

+

{{ "Export Task has been queued and will start shortly"|xlt }}

+
+ + {{ "Expand for Export Zip Details"|xlt }} + +

{{ "Patient Pids To Be Exported"|xlt }}

+
{{ task.getPatientIds()|json_encode|text }}
+
+
+
+ {% endfor %} + + + diff --git a/interface/modules/custom_modules/oe-module-ehi-exporter/templates/oe-module-ehi-exporter/ehi-exporter.html.twig b/interface/modules/custom_modules/oe-module-ehi-exporter/templates/oe-module-ehi-exporter/ehi-exporter.html.twig index 4656f302d1a..24f9b64cd72 100644 --- a/interface/modules/custom_modules/oe-module-ehi-exporter/templates/oe-module-ehi-exporter/ehi-exporter.html.twig +++ b/interface/modules/custom_modules/oe-module-ehi-exporter/templates/oe-module-ehi-exporter/ehi-exporter.html.twig @@ -13,6 +13,7 @@ {{ "Electronic Health Information Export"|xlt }} {{ setupHeader() }} + {% if result %} @@ -120,8 +121,40 @@

{{ "The export process will attempt to set the ram memory usage to unlimited in order to process the export."|xlt}} {{ "If this setting is disabled in your environment you will need to manually set your php.ini memory_limit setting."|xlt}}

{{ csrfToken() }} + + {% endif %} +