feat: data lake (#504)

* feat: data lake Signed-off-by: hxtree <[email protected]> * fix: readme syntax Signed-off-by: hxtree <[email protected]> --------- Signed-off-by: hxtree <[email protected]>
hxtree · Nov 6, 2023 · 14fac1e · 14fac1e
1 parent 98c45e6
commit 14fac1e
Show file tree

Hide file tree

Showing 5 changed files with 343 additions and 40 deletions.
diff --git a/README.md b/README.md
@@ -37,13 +37,12 @@ Alternatively, build, test, and deploy apps locally using the
    git checkout -b feature/improve-readme
    ```
 
-2. Work on changes (e.g. fix a bug or add a new feature). Build, lint, and unit
-   test projects.
+2. Work on changes (e.g. fix a bug or add a new feature). Test-driven
+   development encouraged.
 
-   ```bashProd Environment
-   rush build
-   rush linthttps://github.com/hxtree/cats-cradle
-   rush test
+   ```bash
+   rushx dev
+   rushx test
    ```
 
 3. Stage and commit changes using

diff --git a/platform/constructs/src/queue/README.md b/platform/constructs/src/queue/README.md
@@ -0,0 +1,9 @@
+# TODO make a construct for turn key pub sub, e.g
+
+```
+import { Queue } from 'aws-cdk-lib/aws-sqs';
+new Queue(this, 'EmailSendCommand', { queueName:
+EmailSendCommand.topicName });
+```
+
+Queues should be deployed with the service they run on Make turn-key DLE
diff --git a/platform/message-bus/README.md b/platform/message-bus/README.md
@@ -3,9 +3,23 @@
 ![TypeScript](https://shields.io/badge/TypeScript-3178C6?logo=TypeScript&logoColor=FFF&style=flat-square)
 ![Lifecycle](https://img.shields.io/badge/lifecycle-stable-brightgreen)
 
-This service deploys SNS topics from all defined message schemas. It ensures
-that message sent to a topic gets stored in a S3 data lake.
+This service deploys a SNS topic for every registered
+`@cats-cradle/message-schema`. Other services can then subscribe to these
+topics.
+
+## Data Lake
+
+Message sent to any topic also get stored in a S3 data lake. This works using a
+Kinesis Firehose.
+
+Messages are saved in the `123456789-default-data-lake-bucket` in a structured
+manner for analysis. For example:
+
+> messages/2023/11/06/04/test-firehose-delivery-stream-2-2023-11-06-04-28-20-be72f66c-f0e7-4812-9b1d-064cee498d23
 
 ## References
 
-- [structure data lake for analysis](https://www.youtube.com/watch?v=4xjckHvapFk)
+- [Event-Driven Architecture.](https://aws.amazon.com/event-driven-architecture/)
+- [Pub/sub](https://cloud.google.com/pubsub/docs/overview)
+- [Structure Data Lake for Analysis](https://www.youtube.com/watch?v=4xjckHvapFk)
+- [SNS to Kinesis Data Firehouse to S3](https://docs.aws.amazon.com/sns/latest/dg/sns-firehose-as-subscriber.html)
diff --git a/platform/message-bus/stacks/__snapshots__/message-bus.stack.test.ts.snap b/platform/message-bus/stacks/__snapshots__/message-bus.stack.test.ts.snap
@@ -18,6 +18,103 @@ exports[`MessageBusStack should match snapshot test 1`] = `
       },
       Type: AWS::SNS::Topic,
     },
+    DataLakeFirehoseDeliveryStream: {
+      Properties: {
+        DeliveryStreamName: test-firehose-delivery-stream,
+        DeliveryStreamType: DirectPut,
+        S3DestinationConfiguration: {
+          BucketARN: {
+            Fn::GetAtt: [
+              defaultdatalake9656A812,
+              Arn,
+            ],
+          },
+          BufferingHints: {
+            IntervalInSeconds: 60,
+            SizeInMBs: 1,
+          },
+          CompressionFormat: UNCOMPRESSED,
+          EncryptionConfiguration: {
+            NoEncryptionConfig: NoEncryption,
+          },
+          ErrorOutputPrefix: messages-logs/,
+          Prefix: messages/,
+          RoleARN: {
+            Fn::GetAtt: [
+              KinesisFirehoseRole0891766E,
+              Arn,
+            ],
+          },
+        },
+      },
+      Type: AWS::KinesisFirehose::DeliveryStream,
+    },
+    DataLakeFirehoseSubscriptionForCharacterLevelUpEvent36356735: {
+      Properties: {
+        Endpoint: {
+          Fn::GetAtt: [
+            DataLakeFirehoseDeliveryStream,
+            Arn,
+          ],
+        },
+        Protocol: firehose,
+        RawMessageDelivery: true,
+        SubscriptionRoleArn: {
+          Fn::GetAtt: [
+            SnsRoleToPutMessagesInFirehose73474B02,
+            Arn,
+          ],
+        },
+        TopicArn: {
+          Ref: CharacterLevelUpEventEE3941E0,
+        },
+      },
+      Type: AWS::SNS::Subscription,
+    },
+    DataLakeFirehoseSubscriptionForEmailSendCommand6B92740C: {
+      Properties: {
+        Endpoint: {
+          Fn::GetAtt: [
+            DataLakeFirehoseDeliveryStream,
+            Arn,
+          ],
+        },
+        Protocol: firehose,
+        RawMessageDelivery: true,
+        SubscriptionRoleArn: {
+          Fn::GetAtt: [
+            SnsRoleToPutMessagesInFirehose73474B02,
+            Arn,
+          ],
+        },
+        TopicArn: {
+          Ref: EmailSendCommand52EDF753,
+        },
+      },
+      Type: AWS::SNS::Subscription,
+    },
+    DataLakeFirehoseSubscriptionForItemDestroyCommand1F71CBDC: {
+      Properties: {
+        Endpoint: {
+          Fn::GetAtt: [
+            DataLakeFirehoseDeliveryStream,
+            Arn,
+          ],
+        },
+        Protocol: firehose,
+        RawMessageDelivery: true,
+        SubscriptionRoleArn: {
+          Fn::GetAtt: [
+            SnsRoleToPutMessagesInFirehose73474B02,
+            Arn,
+          ],
+        },
+        TopicArn: {
+          Ref: ItemDestroyCommand739F092B,
+        },
+      },
+      Type: AWS::SNS::Subscription,
+    },
     EmailSendCommand52EDF753: {
       Properties: {
         DisplayName: DefaultEmailSendCommandTopic,
@@ -34,6 +131,149 @@ exports[`MessageBusStack should match snapshot test 1`] = `
       },
       Type: AWS::SNS::Topic,
     },
+    KinesisFirehoseRole0891766E: {
+      Properties: {
+        AssumeRolePolicyDocument: {
+          Statement: [
+            {
+              Action: sts:AssumeRole,
+              Effect: Allow,
+              Principal: {
+                Service: firehose.amazonaws.com,
+              },
+            },
+          ],
+          Version: 2012-10-17,
+        },
+      },
+      Type: AWS::IAM::Role,
+    },
+    KinesisFirehoseRoleDefaultPolicy9DF4ED0B: {
+      Properties: {
+        PolicyDocument: {
+          Statement: [
+            {
+              Action: [
+                s3:GetObject*,
+                s3:GetBucket*,
+                s3:List*,
+                s3:DeleteObject*,
+                s3:PutObject,
+                s3:PutObjectLegalHold,
+                s3:PutObjectRetention,
+                s3:PutObjectTagging,
+                s3:PutObjectVersionTagging,
+                s3:Abort*,
+              ],
+              Effect: Allow,
+              Resource: [
+                {
+                  Fn::GetAtt: [
+                    defaultdatalake9656A812,
+                    Arn,
+                  ],
+                },
+                {
+                  Fn::Join: [
+                    ,
+                    [
+                      {
+                        Fn::GetAtt: [
+                          defaultdatalake9656A812,
+                          Arn,
+                        ],
+                      },
+                      /*,
+                    ],
+                  ],
+                },
+              ],
+            },
+          ],
+          Version: 2012-10-17,
+        },
+        PolicyName: KinesisFirehoseRoleDefaultPolicy9DF4ED0B,
+        Roles: [
+          {
+            Ref: KinesisFirehoseRole0891766E,
+          },
+        ],
+      },
+      Type: AWS::IAM::Policy,
+    },
+    SnsRoleToPutMessagesInFirehose73474B02: {
+      Properties: {
+        AssumeRolePolicyDocument: {
+          Statement: [
+            {
+              Action: sts:AssumeRole,
+              Effect: Allow,
+              Principal: {
+                Service: sns.amazonaws.com,
+              },
+            },
+          ],
+          Version: 2012-10-17,
+        },
+        ManagedPolicyArns: [
+          {
+            Fn::Join: [
+              ,
+              [
+                arn:,
+                {
+                  Ref: AWS::Partition,
+                },
+                :iam::aws:policy/service-role/AmazonSNSRole,
+              ],
+            ],
+          },
+        ],
+      },
+      Type: AWS::IAM::Role,
+    },
+    SnsRoleToPutMessagesInFirehoseDefaultPolicyECC99DF6: {
+      Properties: {
+        PolicyDocument: {
+          Statement: [
+            {
+              Action: [
+                firehose:DescribeDeliveryStream,
+                firehose:ListDeliveryStreams,
+                firehose:ListTagsForDeliveryStream,
+                firehose:PutRecord,
+                firehose:PutRecordBatch,
+              ],
+              Effect: Allow,
+              Resource: {
+                Fn::Join: [
+                  ,
+                  [
+                    arn:aws:firehose:,
+                    {
+                      Ref: AWS::Region,
+                    },
+                    :,
+                    {
+                      Ref: AWS::AccountId,
+                    },
+                    :deliverystream/*,
+                  ],
+                ],
+              },
+            },
+          ],
+          Version: 2012-10-17,
+        },
+        PolicyName: SnsRoleToPutMessagesInFirehoseDefaultPolicyECC99DF6,
+        Roles: [
+          {
+            Ref: SnsRoleToPutMessagesInFirehose73474B02,
+          },
+        ],
+      },
+      Type: AWS::IAM::Policy,
+    },
     defaultdatalake9656A812: {
       DeletionPolicy: Retain,
       Properties: {