From f25ed4b1257f49042d33f6f0c698cf33be663271 Mon Sep 17 00:00:00 2001 From: mfcss Date: Fri, 25 Nov 2022 08:39:51 +0100 Subject: [PATCH] Misc updates --- aws_lambda_example/README.md | 23 ++++++++++++----------- utils.py | 2 +- utils_db.py | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/aws_lambda_example/README.md b/aws_lambda_example/README.md index af835d9..a4706fd 100644 --- a/aws_lambda_example/README.md +++ b/aws_lambda_example/README.md @@ -26,22 +26,23 @@ For 'production setups', we recommend self-hosting InfluxDB or using a paid Infl ## Deploy your AWS Lambda function -1. Create an [IAM execution](https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html) role with permissions: `AWSLambdaBasicExecutionRole` + `AmazonS3FullAccess` +1. Create an [IAM execution](https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html) role (not user) with permissions: `AWSLambdaBasicExecutionRole` + `AmazonS3FullAccess` 2. Go to 'Services/Lambda', then select your S3 bucket region (upper right corner) -3. Add a new Lambda function with a name, a Python 3.7 environment and your execution role +3. Add a new Lambda function with a name, a Python 3.7 environment and your new execution role 4. Add a 'Trigger': Select S3, your test bucket, `All object create events` and suffix `.MF4` 5. Create a zip with `lambda_function.py`, `utils.py`, `utils_db.py`, `inputs.py` and your `*.dbc` (ensure your inputs are updated) 6. Upload the zip via the Lambda 'Actions' button and confirm that your code shows in the online code editor 7. Find the pre-built layer ARN for your AWS S3 region in the `lambda_layer_arns.csv` file -8. In the 'Designer' tap, click Layers/Add a layer/Specify an ARN and parse your ARN -9. Scroll to 'Basic settings' and set the 'Timeout' to `3 min` and memory to `1024 MB` (you can test/tweak these later) -10. Save the script and click 'Deploy', then 'Test' (using the below test data) and verify that it succeeds -11. Click 'Test' in the upper right corner and add the test JSON content below -12. When you're ready, click 'Actions/Publish' to save a new version -13. In AWS Services, go to Cloudwatch/Logs/Log groups and click your Lambda function to monitor events -14. Download a logfile via CANcloud from your main bucket and upload to your test bucket via CANcloud (from the Home tab) -15. Verify that the Lambda function is triggered within a minute and check from the log output that it processes the data -16. Verify that data is written correctly to InfluxDB +8. In the 'Layers' section, click 'Add a layer/Specify an ARN' and parse the ARN matching your region +9. Go to 'Configuration/General configuration' and set the 'Timeout' to `3 min` and memory to `1024 MB` (you can tweak these later) +10. Save the script and click 'Deploy' (important), then 'Test' (using the below test data) and verify that it succeeds +11. Click 'Test' in the upper right corner +12. Add the test event JSON content below (update to match details of an actual MF4 test file on S3) +13. When you're ready, click 'Actions/Publish' to save a new version +14. In AWS Services, go to Cloudwatch/Logs/Log groups and click your Lambda function to monitor events +15. Download a logfile via CANcloud from your main bucket and upload to your test bucket via CANcloud (from the Home tab) +16. Verify that the Lambda function is triggered within a minute and check from the log output that it processes the data +17. Verify that data is written correctly to InfluxDB Once tested, change the 'Trigger' S3 bucket to your main bucket and verify that it works as intended over a period. diff --git a/utils.py b/utils.py index 2bf84ff..d1887f4 100644 --- a/utils.py +++ b/utils.py @@ -211,7 +211,7 @@ def filter_signals(self, df_phys): return df_phys - def get_raw_data(self, log_file, lin=False, passwords={}): + def get_raw_data(self, log_file, passwords={},lin=False): """Extract a df of raw data and device ID from log file. Optionally include LIN bus data by setting lin=True """ diff --git a/utils_db.py b/utils_db.py index 72108df..6f35a4b 100644 --- a/utils_db.py +++ b/utils_db.py @@ -77,7 +77,7 @@ def write_signals(self, device_id, df_phys): df_signal = group.to_frame().rename(columns={"Physical Value": signal}) if self.res != "": - df_signal = df_signal.resample(self.res).pad().dropna() + df_signal = df_signal.resample(self.res).ffill().dropna() if self.verbose: print(f"Signal: {signal} (mean: {round(df_signal[signal].mean(),2)} | records: {len(df_signal)} | resampling: {self.res})")