Saas-549 - logs masking (#43)

* added log masking support
codefresh-io · May 4, 2020 · c9dfe09 · c9dfe09
1 parent 0722d00
commit c9dfe09
Show file tree

Hide file tree

Showing 9 changed files with 640 additions and 31 deletions.
diff --git a/lib/ContainerLogger.js b/lib/ContainerLogger.js
@@ -111,6 +111,7 @@ class ContainerLogger extends EventEmitter {
         this.handledStreams++;
         stdout
             .pipe(this._logSizeLimitStream())
+            .pipe(this.stepLogger.createMaskingStream())
             .pipe(this.stepLogger.stepNameTransformStream().once('end', this._handleFinished.bind(this)))
             .pipe(stepLoggerWritableStream, {end: false});
 
@@ -123,6 +124,7 @@ class ContainerLogger extends EventEmitter {
         stderr
             .pipe(this._logSizeLimitStream())
             .pipe(this._errorTransformerStream())
+            .pipe(this.stepLogger.createMaskingStream())
             .pipe(this.stepLogger.stepNameTransformStream().once('end', this._handleFinished.bind(this)))
             .pipe(stepLoggerWritableStream, {end: false});
 

diff --git a/lib/addNewMask.js b/lib/addNewMask.js
@@ -0,0 +1,42 @@
+const rp = require('request-promise');
+
+
+function updateMasks(secret) {
+    const port = process.env.PORT || 8080;
+    const host = process.env.HOST || 'localhost';
+
+    const opt = {
+        uri: `http://${host}:${port}/secrets`,
+        method: 'POST',
+        json: true,
+        body: secret,
+        resolveWithFullResponse: true,
+    };
+
+    rp(opt)
+        .then((res) => {
+            if (res.statusCode >= 400) {
+                console.log(`could not create mask for secret: ${secret.key}, because server responded with: ${res.statusCode}\n\n${res.body}`);
+                process.exit(1);
+            }
+            console.log(`successfully updated masks with secret: ${secret.key}`);
+            process.exit(0);
+        })
+        .catch((err) => {
+            console.log(`could not create mask for secret: ${secret.key}, due to error: ${err}`);
+            process.exit(1);
+        });
+}
+
+if (require.main === module) {
+    // first argument is the secret key second argument is the secret value
+    if (process.argv.length < 4) {
+        console.log('not enough arguments, need secret key and secret value');
+        process.exit(2);
+    }
+    const key = process.argv[2];
+    const value = process.argv[3];
+    updateMasks({ key, value });
+} else {
+    module.exports = updateMasks;
+}
diff --git a/lib/logger.js b/lib/logger.js
@@ -13,6 +13,7 @@ const LoggerStrategy          = require('./enums').LoggerStrategy;
 const ContainerHandlingStatus = require('./enums').ContainerHandlingStatus;
 const ContainerLogger         = require('./ContainerLogger');
 const { TaskLogger }          = require('@codefresh-io/task-logger');
+const express                 = require('express');
 
 const initialState = { pid: process.pid, status: 'init', lastLogsDate: new Date() , failedHealthChecks: [] , restartCounter: 0, containers: {} };
 class Logger {
@@ -117,6 +118,8 @@ class Logger {
                 }));
                 return;
             });
+
+        this._listenForEngineUpdates();
     }
 
     _readState() {
@@ -337,6 +340,33 @@ class Logger {
         });
     }
 
+    _listenForEngineUpdates() {
+        const app = express();
+        this._app = app;
+        const port = process.env.PORT || 8080;
+        const host = process.env.HOST || 'localhost';
+
+        app.use(require('body-parser').json());
+
+        app.post('/secrets', (req, res) => {
+            try {
+                const secret = req.body;
+                logger.info(`got request to add new mask: ${JSON.stringify(secret)}`);
+
+                // secret must have { key, value } structure
+                this.taskLogger.addNewMask(secret);
+                res.status(201).end('secret added');
+            } catch (err) {
+                logger.info(`could not create new mask due to error: ${err}`);
+                res.status(400).end(err);
+            }
+        });
+
+        app.listen(port, host, () => {
+            logger.info(`listening for engine updates on ${host}:${port}`);
+        });
+    }
+
     _handleContainerStreamEnd() {
         this.finishedContainers++;
         this.finishedContainersEmitter.emit('end');

diff --git a/package.json b/package.json
@@ -6,14 +6,18 @@
     "cf-container-logger"
   ],
   "dependencies": {
-    "@codefresh-io/task-logger": "1.6.0",
+    "@codefresh-io/task-logger": "1.7.0",
+    "body-parser": "^1.19.0",
     "cf-errors": "^0.1.11",
     "cf-logs": "^1.1.0",
     "docker-events": "0.0.2",
     "dockerode": "^2.3.0",
+    "express": "^4.17.1",
     "forever": "^0.15.3",
     "lodash": "^4.15.0",
-    "q": "^1.4.1"
+    "q": "^1.4.1",
+    "request": "^2.88.2",
+    "request-promise": "^4.2.5"
   },
   "devDependencies": {
     "chai": "^4.2.0",

diff --git a/service.yaml b/service.yaml
@@ -1 +1 @@
-version: 1.3.0
+version: 1.4.0
diff --git a/test/ContainerLogger.unit.spec.js b/test/ContainerLogger.unit.spec.js
@@ -814,6 +814,7 @@ describe('Container Logger tests', () => {
                     transformSpies.push(transform);
                     return transform;
                 },
+                createMaskingStream: () => new PassThrough(),
                 opts: {
                     logsRateLimitConfig: {} // use stream
                 }

diff --git a/test/addNewMask.unit.spec.js b/test/addNewMask.unit.spec.js
@@ -0,0 +1,78 @@
+/* jshint ignore:start */
+
+'use strict';
+const Q = require('q');
+
+const chai       = require('chai');
+const expect     = chai.expect;
+const sinon      = require('sinon');
+const sinonChai  = require('sinon-chai');
+const proxyquire = require('proxyquire').noCallThru();
+chai.use(sinonChai);
+
+const originalProcessExit = process.exit;
+
+describe('addNewMask', () => {
+    before(() => {
+        process.exit = sinon.spy();
+    });
+
+    beforeEach(() => {
+        process.exit.resetHistory();
+    });
+
+    after(() => {
+        process.exit = originalProcessExit;
+    });
+
+    describe('positive', () => {
+        it('should send a request to add a secret', async () => {
+            const rpSpy = sinon.spy(() => Q.resolve({ statusCode: 201 }));
+            const addNewMask = proxyquire('../lib/addNewMask', {
+                'request-promise': rpSpy
+            });
+
+            process.env.PORT = 1337;
+            process.env.HOST = '127.0.0.1';
+
+            const secret = {
+                key: '123',
+                value: 'ABC',
+            };
+
+            addNewMask(secret);
+
+            expect(rpSpy).to.have.been.calledOnceWith({
+                uri: `http://127.0.0.1:1337/secrets`,
+                method: 'POST',
+                json: true,
+                body: secret,
+                resolveWithFullResponse: true,
+            });
+            await Q.delay(10);
+            expect(process.exit).to.have.been.calledOnceWith(0);
+        });
+    });
+
+    describe('negative', () => {
+        it('should send a request to add a secret', async () => {
+            const rpSpy = sinon.spy(() => Q.reject('could not send request'));
+            const addNewMask = proxyquire('../lib/addNewMask', {
+                'request-promise': rpSpy
+            });
+
+            process.env.PORT = 1337;
+            process.env.HOST = '127.0.0.1';
+
+            const secret = {
+                key: '123',
+                value: 'ABC',
+            };
+
+            addNewMask(secret);
+            await Q.delay(10);
+            expect(process.exit).to.have.been.calledOnceWith(1);
+        });
+    });
+});
+
diff --git a/test/logger.unit.spec.js b/test/logger.unit.spec.js
@@ -11,6 +11,15 @@ const ContainerStatus = require('../lib/enums').ContainerStatus;
 const LoggerStrategy  = require('../lib/enums').LoggerStrategy;
 const { EventEmitter } = require('events');
 
+const expressMock = function() {
+    return {
+        use: sinon.spy(),
+        listen: sinon.spy(),
+        post: sinon.spy(),
+        get: sinon.spy(),
+    }
+}
+
 describe('Logger tests', () => {
 
     let processExit;
@@ -83,7 +92,8 @@ describe('Logger tests', () => {
                 });
 
                 const Logger = proxyquire('../lib/logger', {
-                    '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory }
+                    '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
+                    'express': expressMock,
                 });
 
                 const loggerId               = 'loggerId';
@@ -126,7 +136,8 @@ describe('Logger tests', () => {
                 });
 
                 const Logger = proxyquire('../lib/logger', {
-                    '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory }
+                    '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
+                    'express': expressMock,
                 });
 
                 const loggerId               = 'loggerId';
@@ -167,7 +178,8 @@ describe('Logger tests', () => {
                 });
 
                 const Logger = proxyquire('../lib/logger', {
-                    '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory }
+                    '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
+                    'express': expressMock,
                 });
 
                 const loggerId               = 'loggerId';
@@ -207,7 +219,8 @@ describe('Logger tests', () => {
                 });
 
                 const Logger = proxyquire('../lib/logger', {
-                    '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory }
+                    '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
+                    'express': expressMock,
                 });
 
                 const loggerId               = 'loggerId';
@@ -252,7 +265,8 @@ describe('Logger tests', () => {
                 });
 
                 const Logger = proxyquire('../lib/logger', {
-                    '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory }
+                    '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
+                    'express': expressMock,
                 });
 
                 const loggerId               = 'loggerId';
@@ -1365,6 +1379,7 @@ describe('Logger tests', () => {
                 '@codefresh-io/task-logger': { TaskLogger: () => Q.resolve(taskLogger) },
                 'docker-events': function () { return dockerEvents; },
                 './ContainerLogger': function () { return containerLogger; },
+                'express': expressMock,
             });
 
             const loggerId = 'loggerId';
@@ -1454,6 +1469,7 @@ describe('Logger tests', () => {
                 '@codefresh-io/task-logger': { TaskLogger: () => Q.resolve(taskLogger) },
                 'docker-events': function () { return dockerEvents; },
                 './ContainerLogger': function () { return containerLogger; },
+                'express': expressMock,
             });
 
             const logger = new Logger({
@@ -1475,4 +1491,46 @@ describe('Logger tests', () => {
             expect(logger._updateLastLoggingDate).to.have.been.calledOnce;
         });
     });
+
+    describe('engine updates', () => {
+        it('should listen for engine updates', async () => {
+            const taskLogger = {
+                on: sinon.spy(),
+                restore: sinon.spy(() => Q.resolve()),
+                startHealthCheck: sinon.spy(),
+                onHealthCheckReported: sinon.spy(),
+                getStatus: sinon.spy(),
+            };
+            const TaskLoggerFactory = sinon.spy(() => {
+                return Q.resolve(taskLogger);
+            });
+
+            const Logger = proxyquire('../lib/logger', {
+                '@codefresh-io/task-logger': { TaskLogger: TaskLoggerFactory },
+                'express': expressMock,
+            });
+
+            const loggerId               = 'loggerId';
+            const taskLoggerConfig = {task: {}, opts: {}};
+            const findExistingContainers = false;
+
+            const logger                        = new Logger({
+                loggerId,
+                taskLoggerConfig,
+                findExistingContainers,
+            });
+            logger._listenForNewContainers      = sinon.spy();
+            logger._writeNewState               = sinon.spy();
+            logger._listenForExistingContainers = sinon.spy();
+            process.env.PORT = 1337;
+            process.env.HOST = '127.0.0.1';
+            logger.start();
+
+            await Q.delay(10);
+
+            expect(logger._app).to.not.be.undefined;
+            expect(logger._app.listen).to.have.been.calledOnce;
+            expect(logger._app.listen).to.have.been.calledWithMatch(1337, '127.0.0.1');
+        });
+    });
 });