From 80adcd830ad3becb64c3e2d0e95c80bf1f0af4b0 Mon Sep 17 00:00:00 2001
From: Penny How <pho@geus.dk>
Date: Tue, 24 Sep 2024 12:16:45 -0100
Subject: [PATCH] Bug fix for tx fetching (#305) (#306)

* Bug fix for tx fetching (#305)

* Indentation change

* Moved from pypromice to separate processing pipeline

* version bump
---
 setup.py                         |   2 +-
 src/pypromice/tx/get_watsontx.py | 147 -------------------------------
 src/pypromice/tx/tx.py           |   8 +-
 3 files changed, 5 insertions(+), 152 deletions(-)
 delete mode 100644 src/pypromice/tx/get_watsontx.py

diff --git a/setup.py b/setup.py
index 8c6db22d..fd1be5d2 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="pypromice",
-    version="1.4.2",
+    version="1.4.3",
     author="GEUS Glaciology and Climate",
     description="PROMICE/GC-Net data processing toolbox",
     long_description=long_description,
diff --git a/src/pypromice/tx/get_watsontx.py b/src/pypromice/tx/get_watsontx.py
deleted file mode 100644
index e5b74a27..00000000
--- a/src/pypromice/tx/get_watsontx.py
+++ /dev/null
@@ -1,147 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-"""
-Created on Fri Jul 22 16:20:09 2022
-
-Script to get L0tx transmission messages from the Watson River station using 
-the tx module
-
-@author: Penelope How, pho@geus.dk
-"""
-from argparse import ArgumentParser
-
-from configparser import ConfigParser
-import os, imaplib, email, re
-from glob import glob
-from datetime import datetime
-
-from pypromice.tx import getMail, L0tx, sortLines
-
-
-def parse_arguments_watson():
-    parser = ArgumentParser(description="AWS L0 transmission fetcher for Watson River measurements")       
-    parser.add_argument('-a', '--account', default=None, type=str, required=True, help='Email account .ini file')
-    parser.add_argument('-p', '--password', default=None, type=str, required=True, help='Email credentials .ini file')                      
-    parser.add_argument('-o', '--outpath', default=None, type=str, required=False, help='Path where to write output (if given)')           
-    parser.add_argument('-f', '--formats', default=None, type=str, required=False, help='Path to Payload format .csv file')
-    parser.add_argument('-t', '--types', default=None, type=str, required=False, help='Path to Payload type .csv file')  	
-    parser.add_argument('-u', '--uid', default=None, type=str, required=True, help='Last AWS uid .ini file')	        
-    args = parser.parse_args()
-    return args
-
-#------------------------------------------------------------------------------
-def get_watsontx():
-    """Executed from the command line"""
-    args = parse_arguments_watson()
-    
-     # Set payload formatter paths
-    formatter_file = args.formats
-    type_file = args.types
-
- 	# Set credential paths
-    accounts_file = args.account
-    credentials_file = args.password 
-
- 	# Set last aws uid path
- 	# last_uid = 1000000
-    uid_file = args.uid
-    
-    # Set last aws uid path
-    with open(uid_file, 'r') as last_uid_f:
-        last_uid = int(last_uid_f.readline())
-    
-    # Set output file directory
-    out_dir = args.outpath
-    if not os.path.exists(out_dir):
-        os.mkdir(out_dir)
-    
-#------------------------------------------------------------------------------
-    
-    # Define accounts and credentials ini file paths
-    accounts_ini = ConfigParser()
-    accounts_ini.read_file(open(accounts_file))
-    accounts_ini.read(credentials_file)
-                 
-    # Get credentials
-    account = accounts_ini.get('aws', 'account')
-    server = accounts_ini.get('aws', 'server')
-    port = accounts_ini.getint('aws', 'port')    
-    password = accounts_ini.get('aws', 'password')
-    if not password:
-        password = input('password for AWS email account: ')
-    print('AWS data from server %s, account %s' %(server, account))
-    
-#------------------------------------------------------------------------------
-    
-    # Log in to email server
-    mail_server = imaplib.IMAP4_SSL(server, port)
-    typ, accountDetails = mail_server.login(account, password)
-    if typ != 'OK':
-        print('Not able to sign in!')
-        raise
-        
-    # Grab new emails
-    result, data = mail_server.select(mailbox='"[Gmail]/All Mail"', 
-                                      readonly=True)
-    print('mailbox contains %s messages' %data[0])
-    
-#------------------------------------------------------------------------------
-    
-    # Get L0tx datalines from email transmissions
-    for uid, mail in getMail(mail_server, last_uid=last_uid):
-        message = email.message_from_string(mail)
-        try:
-            name = str(message.get_all('subject')[0])
-            d = datetime.strptime(message.get_all('date')[0], 
-                                  '%a, %d %b %Y %H:%M:%S %z')
-        except:
-            name=None
-            d=None
-        
-        if name and ('Watson' in name or 'GIOS' in name):
-            print(f'Watson/GIOS station message, {d.strftime("%Y-%m-%d %H:%M:%S")}')
-
-            l0 = L0tx(message, formatter_file, type_file, 
-                      sender_name=['emailrelay@konectgds.com','sbdservice'])
-            
-            if l0.msg: 
-                content, attachment = l0.getEmailBody()
-                attachment_name = str(attachment.get_filename())               
-                out_fn = re.sub(r'\d*\.dat$', '', attachment_name) + '.txt'
-                out_path = os.sep.join((out_dir, out_fn))
-        
-                print(f'Writing to {out_fn}')
-                print(l0.msg)
-            
-                with open(out_path, mode='a') as out_f:
-                    out_f.write(l0.msg + '\n')    
-    
-#------------------------------------------------------------------------------
-
-    # Sort L0tx files and add tails    
-    for f in glob(out_dir+'/*.txt'):
-        
-        # Sort lines in L0tx file and remove duplicates
-        in_dirn, in_fn = os.path.split(f)    
-        out_fn = 'sorted_' + in_fn
-        out_pn = os.sep.join((in_dirn, out_fn))
-        sortLines(f, out_pn)
-        
-    # Close mail server if open
-    if 'mail_server' in locals():
-        print(f'\nClosing {account}')
-        mail_server.close()
-        resp = mail_server.logout()
-        assert resp[0].upper() == 'BYE'
-    
-    # Write last aws uid to ini file
-    try:
-        with open(uid_file, 'w') as last_uid_f:
-            last_uid_f.write(uid)
-    except:
-        print(f'Could not write last uid {uid} to {uid_file}')
-         
-    print('Finished')
-        
-if __name__ == "__main__":  
-    get_watsontx()
diff --git a/src/pypromice/tx/tx.py b/src/pypromice/tx/tx.py
index 2167b874..601932e6 100644
--- a/src/pypromice/tx/tx.py
+++ b/src/pypromice/tx/tx.py
@@ -894,10 +894,10 @@ def sortLines(in_file, out_file, replace_unsorted=True):                       #
             # out_f.write(headers)
             out_f.writelines(unique_lines)
     
-    # Replace input file with new sorted file
-    if replace_unsorted:
-        os.remove(in_file)
-        os.rename(out_file, in_file)
+        # Replace input file with new sorted file
+        if replace_unsorted:
+            os.remove(in_file)
+            os.rename(out_file, in_file)
 
 def addTail(in_file, out_dir, aws_name, header_names='', lines_limit=100):    
     '''Generate tails file from L0tx file