From 565bb482024016a985957f74a0dc3eadb4f328b1 Mon Sep 17 00:00:00 2001 From: Kylie Pace Date: Mon, 18 Jan 2021 16:11:30 +0000 Subject: [PATCH] rename consumer folder --- README.md | 4 +- package.json | 2 +- src/constants.json | 2 +- src/{consumers => consumer}/KafkaConsumer.ts | 0 src/{consumers => consumer}/app.ts | 0 .../services/SaveDataService.ts | 3 +- .../services/TransformService.ts | 0 test/websocket.test.ts | 38 ++++++++++++++++--- 8 files changed, 39 insertions(+), 10 deletions(-) rename src/{consumers => consumer}/KafkaConsumer.ts (100%) rename src/{consumers => consumer}/app.ts (100%) rename src/{consumers => consumer}/services/SaveDataService.ts (87%) rename src/{consumers => consumer}/services/TransformService.ts (100%) diff --git a/README.md b/README.md index 1322010..3a7b55b 100644 --- a/README.md +++ b/README.md @@ -49,11 +49,13 @@ Therefore, in order to save the data in its transformed state, I replaced Redis ### What I'd add - a proper logging client so that logs are searchable by session_id and request_id header - security & rate limiting +- database access: the server application only needs read permission - CI/CD -- architecture: no need for this to be a monolithic application +- architecture: no need for this to be a monolithic application. deploying separate resources would improve resilience, scalability, and security. - replace services from docker-compose with actual deployed instances - validate event model - create index on mongodb on session_id +- currently, there is a potential problem if the websocket client tries to send the same data again. There is no uniqueness constraint on each item in the `children` array so events would be duplicated. Maybe any existing database record should be deleted or archived when a SESSION_START event arrives, or the SaveDataService should use mongo's `$unset` operator to clear the `children` array. diff --git a/package.json b/package.json index 4ca6193..6a1e068 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ }, "scripts": { "compile": "rm -Rf ./lib && tsc", - "consumer:dev": "ts-node-dev src/consumers/app.ts", + "consumer:dev": "ts-node-dev src/consumer/app.ts", "lint": "eslint --ext .js,.ts src/", "mocha": "ts-mocha \"test/**/*.ts\" --recursive --exit", "server:dev": "ts-node-dev src/server/index.ts", diff --git a/src/constants.json b/src/constants.json index 30d4bfc..b63b29f 100644 --- a/src/constants.json +++ b/src/constants.json @@ -1,7 +1,7 @@ { "PORT": 8080, "topics": { - "EVENTS": "events" + "EVENTS": "websocket.events" }, "urlPaths": { "websocket": "websocket" diff --git a/src/consumers/KafkaConsumer.ts b/src/consumer/KafkaConsumer.ts similarity index 100% rename from src/consumers/KafkaConsumer.ts rename to src/consumer/KafkaConsumer.ts diff --git a/src/consumers/app.ts b/src/consumer/app.ts similarity index 100% rename from src/consumers/app.ts rename to src/consumer/app.ts diff --git a/src/consumers/services/SaveDataService.ts b/src/consumer/services/SaveDataService.ts similarity index 87% rename from src/consumers/services/SaveDataService.ts rename to src/consumer/services/SaveDataService.ts index d0ff5a5..badb264 100644 --- a/src/consumers/services/SaveDataService.ts +++ b/src/consumer/services/SaveDataService.ts @@ -29,7 +29,8 @@ export default class SaveDataService { $push: { children: { $each: data.children, - $sort: { 'data.timestamp': 1 } + // sort asc on timestamp so oldest events will be at beginning of array + $sort: { 'timestamp': 1 } } } }; diff --git a/src/consumers/services/TransformService.ts b/src/consumer/services/TransformService.ts similarity index 100% rename from src/consumers/services/TransformService.ts rename to src/consumer/services/TransformService.ts diff --git a/test/websocket.test.ts b/test/websocket.test.ts index aa57a23..eba4d81 100644 --- a/test/websocket.test.ts +++ b/test/websocket.test.ts @@ -24,9 +24,12 @@ describe('websocket http request', () => { }); - describe('sending multiple messages', () => { - beforeEach(done => setTimeout(done, 500)); + describe('sending multiple messages with child events not in order', () => { + // give consumer app some time to receive kafka message + beforeEach(done => setTimeout(done, 1800)); before(() => { + + // send session_start and the last event clients[0].send( JSON.stringify([ { @@ -37,17 +40,23 @@ describe('websocket http request', () => { { timestamp: 1569972085, type: "EVENT", - name: "basket_removed" + name: "last_event" } ]) ); + // send the middle events and then session_end clients[0].send( JSON.stringify([ { timestamp: 1569972083, type: "EVENT", - name: "purchase_completed" + name: "middle_event" + }, + { + timestamp: 1569972083, + type: "EVENT", + name: "first_event" }, { timestamp: 1569972086, @@ -58,14 +67,31 @@ describe('websocket http request', () => { ); }); - it('saves data', async () => { + it('saves data with child events in order by timestamp and with start and end timestamps', async () => { const data = await database.findOne({session_id}); console.log(data) chai.expect(data).to.have.property('session_id').which.equals(session_id); chai.expect(data).to.have.property('end').which.equals(1569972086); chai.expect(data).to.have.property('start').which.equals(1569972082); chai.expect(data).to.have.property('children').which.is.an('array'); - chai.expect(data.children.length).to.equal(2); + chai.expect(data.children.length).to.equal(3); + chai.expect(data.children).to.deep.equal([ + { + type: 'EVENT', + timestamp: 1569972083, + name: 'middle_event' + }, + { + type: 'EVENT', + timestamp: 1569972083, + name: 'first_event' + }, + { + type: 'EVENT', + timestamp: 1569972085, + name: 'last_event' + } + ]) }); });