From acecf021893419ebec28aad1b833da27b910b0c2 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Mon, 10 Jun 2019 07:37:47 -0700 Subject: [PATCH 1/4] Adding logging of user-agent header --- dist/app.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dist/app.js b/dist/app.js index bd8ac436..329fbd61 100644 --- a/dist/app.js +++ b/dist/app.js @@ -67,8 +67,8 @@ app.enable("trust proxy"); app.set("views", path.join(__dirname, "views")); app.set("view engine", "jade"); app.use("/public", express.static(__dirname + "/public")); -//app.use(logger("dev")) -app.use(logger(":remote-addr :remote-user :method :url :status :response-time ms - :res[content-length]")); +// Log each request to the console with IP addresses. +app.use(logger(":remote-addr :remote-user :method :url :status :response-time ms - :res[content-length] :user-agent")); app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: false })); app.use(cookieParser()); From e1616b6cfc6f54bbf6fa59ee82fba97aa827eea9 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Mon, 10 Jun 2019 09:27:13 -0700 Subject: [PATCH 2/4] Improved local logging --- dist/app.js | 2 ++ dist/middleware/req-logging.js | 36 +++++++++++++++++++++ dist/util/winston-logging.js | 2 +- src/app.ts | 9 +++++- src/middleware/req-logging.ts | 57 ++++++++++++++++++++++++++++++++++ src/util/winston-logging.js | 4 ++- 6 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 dist/middleware/req-logging.js create mode 100644 src/middleware/req-logging.ts diff --git a/dist/app.js b/dist/app.js index 329fbd61..2c068f02 100644 --- a/dist/app.js +++ b/dist/app.js @@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); var express = require("express"); // Middleware var route_ratelimit_1 = require("./middleware/route-ratelimit"); +var req_logging_1 = require("./middleware/req-logging"); var path = require("path"); var logger = require("morgan"); var wlogger = require("./util/winston-logging"); @@ -73,6 +74,7 @@ app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: false })); app.use(cookieParser()); app.use(express.static(path.join(__dirname, "public"))); +app.use("/", req_logging_1.logReqInfo); // // let username = process.env.USERNAME; // let password = process.env.PASSWORD; diff --git a/dist/middleware/req-logging.js b/dist/middleware/req-logging.js new file mode 100644 index 00000000..6d2ec1ec --- /dev/null +++ b/dist/middleware/req-logging.js @@ -0,0 +1,36 @@ +"use strict"; +/* + +*/ +Object.defineProperty(exports, "__esModule", { value: true }); +var wlogger = require("../util/winston-logging"); +// Used for debugging and iterrogating JS objects. +var util = require("util"); +util.inspect.defaultOptions = { depth: 1 }; +var logReqInfo = function (req, res, next) { + /* + //console.log(`req: ${util.inspect(req)}`) + console.log(`req.headers: ${util.inspect(req.headers)}`) + console.log(`req.url: ${req.url}`) + console.log(`req.method: ${req.method}`) + console.log(`req.sws.ip: ${req.sws.ip}`) + console.log(`req.sws.real_ip: ${req.sws.real_ip}`) + console.log(`req.body: ${util.inspect(req.body)}`) + console.log(` `) + console.log(` `) + */ + var ip = req.sws.real_ip; + var method = req.method; + var url = req.url; + var dataToLog = { + headers: req.headers, + url: url, + method: method, + ip: req.sws.ip, + real_ip: ip, + body: req.body + }; + wlogger.info("Request: " + ip + " " + method + " " + url, dataToLog); + next(); +}; +exports.logReqInfo = logReqInfo; diff --git a/dist/util/winston-logging.js b/dist/util/winston-logging.js index 5fce8b08..98fffba2 100644 --- a/dist/util/winston-logging.js +++ b/dist/util/winston-logging.js @@ -13,7 +13,7 @@ var transport = new winston.transports.DailyRotateFile({ datePattern: "YYYY-MM-DD", zippedArchive: false, maxSize: "1m", - maxFiles: "5d", + maxFiles: "3", format: winston.format.combine(winston.format.timestamp(), winston.format.json()) }); transport.on("rotate", function (oldFilename, newFilename) { diff --git a/src/app.ts b/src/app.ts index a1197458..dd5cf24c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -2,6 +2,7 @@ import * as express from "express" // Middleware import { routeRateLimit } from "./middleware/route-ratelimit" +import { logReqInfo } from "./middleware/req-logging" const path = require("path") const logger = require("morgan") @@ -87,13 +88,19 @@ app.set("view engine", "jade") app.use("/public", express.static(`${__dirname}/public`)) // Log each request to the console with IP addresses. -app.use(logger(`:remote-addr :remote-user :method :url :status :response-time ms - :res[content-length] :user-agent`)) +app.use( + logger( + `:remote-addr :remote-user :method :url :status :response-time ms - :res[content-length] :user-agent` + ) +) app.use(bodyParser.json()) app.use(bodyParser.urlencoded({ extended: false })) app.use(cookieParser()) app.use(express.static(path.join(__dirname, "public"))) +app.use(`/`, logReqInfo) + // // let username = process.env.USERNAME; // let password = process.env.PASSWORD; diff --git a/src/middleware/req-logging.ts b/src/middleware/req-logging.ts new file mode 100644 index 00000000..099b9ee0 --- /dev/null +++ b/src/middleware/req-logging.ts @@ -0,0 +1,57 @@ +/* + +*/ + +import * as express from "express" +const wlogger = require("../util/winston-logging") + +// Used for debugging and iterrogating JS objects. +const util = require("util") +util.inspect.defaultOptions = { depth: 1 } + +// Add the 'locals' property to the express.Request interface. +declare global { + namespace Express { + interface Request { + locals: any, + sws: any + } + } +} + +const logReqInfo = function( + req: express.Request, + res: express.Response, + next: express.NextFunction +) { + +/* + //console.log(`req: ${util.inspect(req)}`) + console.log(`req.headers: ${util.inspect(req.headers)}`) + console.log(`req.url: ${req.url}`) + console.log(`req.method: ${req.method}`) + console.log(`req.sws.ip: ${req.sws.ip}`) + console.log(`req.sws.real_ip: ${req.sws.real_ip}`) + console.log(`req.body: ${util.inspect(req.body)}`) + console.log(` `) + console.log(` `) +*/ + const ip = req.sws.real_ip + const method = req.method + const url = req.url + + const dataToLog = { + headers: req.headers, + url: url, + method: method, + ip: req.sws.ip, + real_ip: ip, + body: req.body + } + + wlogger.info(`Request: ${ip} ${method} ${url}`, dataToLog) + + next() +} + +export { logReqInfo } diff --git a/src/util/winston-logging.js b/src/util/winston-logging.js index 14cd908a..647c1c97 100644 --- a/src/util/winston-logging.js +++ b/src/util/winston-logging.js @@ -12,12 +12,14 @@ require("winston-daily-rotate-file") var NETWORK = process.env.NETWORK // Configure daily-rotation transport. +// Configured to only save 20 megs worth of files. Specifically 20 files of +// 1 megabyte each. Old log files will be deleted to make room for new log files. var transport = new winston.transports.DailyRotateFile({ filename: `${__dirname}/../../logs/rest-${NETWORK}-%DATE%.log`, datePattern: "YYYY-MM-DD", zippedArchive: false, maxSize: "1m", - maxFiles: "5d", + maxFiles: "20", format: winston.format.combine( winston.format.timestamp(), winston.format.json() From 36db0c65dbcf8ef50b7e818c84a102d3af89bbab Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Tue, 11 Jun 2019 09:24:25 -0700 Subject: [PATCH 3/4] Refining local logging code after tests --- src/app.ts | 1 + src/middleware/req-logging.ts | 6 ++++-- src/util/winston-logging.js | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/app.ts b/src/app.ts index dd5cf24c..855a66e7 100644 --- a/src/app.ts +++ b/src/app.ts @@ -99,6 +99,7 @@ app.use(bodyParser.urlencoded({ extended: false })) app.use(cookieParser()) app.use(express.static(path.join(__dirname, "public"))) +// Local logging middleware for tracking incoming connection information. app.use(`/`, logReqInfo) // diff --git a/src/middleware/req-logging.ts b/src/middleware/req-logging.ts index 099b9ee0..3db3ee3d 100644 --- a/src/middleware/req-logging.ts +++ b/src/middleware/req-logging.ts @@ -1,5 +1,7 @@ /* - + This middleware logs connection information to local logs. It gives the ability + to detect when the server is being DDOS attacked, and also to collect metrics, + like the most popular endpoints. */ import * as express from "express" @@ -49,7 +51,7 @@ const logReqInfo = function( body: req.body } - wlogger.info(`Request: ${ip} ${method} ${url}`, dataToLog) + wlogger.verbose(`Request: ${ip} ${method} ${url}`, dataToLog) next() } diff --git a/src/util/winston-logging.js b/src/util/winston-logging.js index 647c1c97..6d418ead 100644 --- a/src/util/winston-logging.js +++ b/src/util/winston-logging.js @@ -18,8 +18,8 @@ var transport = new winston.transports.DailyRotateFile({ filename: `${__dirname}/../../logs/rest-${NETWORK}-%DATE%.log`, datePattern: "YYYY-MM-DD", zippedArchive: false, - maxSize: "1m", - maxFiles: "20", + maxSize: "1m", // 1 megabyte per file. + maxFiles: "100", // Will overwrite old log files after 100 megs used. format: winston.format.combine( winston.format.timestamp(), winston.format.json() From 6d31142bd9eb0206607839760b724caf0360a204 Mon Sep 17 00:00:00 2001 From: Gabriel Cardona Date: Thu, 13 Jun 2019 10:51:25 -0700 Subject: [PATCH 4/4] Bump v. --- dist/app.js | 1 + dist/middleware/req-logging.js | 6 ++++-- dist/public/bitcoin-com-mainnet-rest-v2.json | 2 +- dist/public/bitcoin-com-testnet-rest-v2.json | 2 +- dist/util/winston-logging.js | 4 +++- package.json | 2 +- swaggerJSONFiles/info.json | 2 +- swaggerJSONFilesBuilt/mainnet/info.json | 2 +- swaggerJSONFilesBuilt/testnet/info.json | 2 +- 9 files changed, 14 insertions(+), 9 deletions(-) diff --git a/dist/app.js b/dist/app.js index 2c068f02..f3c9109e 100644 --- a/dist/app.js +++ b/dist/app.js @@ -74,6 +74,7 @@ app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ extended: false })); app.use(cookieParser()); app.use(express.static(path.join(__dirname, "public"))); +// Local logging middleware for tracking incoming connection information. app.use("/", req_logging_1.logReqInfo); // // let username = process.env.USERNAME; diff --git a/dist/middleware/req-logging.js b/dist/middleware/req-logging.js index 6d2ec1ec..bcc6599a 100644 --- a/dist/middleware/req-logging.js +++ b/dist/middleware/req-logging.js @@ -1,6 +1,8 @@ "use strict"; /* - + This middleware logs connection information to local logs. It gives the ability + to detect when the server is being DDOS attacked, and also to collect metrics, + like the most popular endpoints. */ Object.defineProperty(exports, "__esModule", { value: true }); var wlogger = require("../util/winston-logging"); @@ -30,7 +32,7 @@ var logReqInfo = function (req, res, next) { real_ip: ip, body: req.body }; - wlogger.info("Request: " + ip + " " + method + " " + url, dataToLog); + wlogger.verbose("Request: " + ip + " " + method + " " + url, dataToLog); next(); }; exports.logReqInfo = logReqInfo; diff --git a/dist/public/bitcoin-com-mainnet-rest-v2.json b/dist/public/bitcoin-com-mainnet-rest-v2.json index d26d6da3..a27835b1 100644 --- a/dist/public/bitcoin-com-mainnet-rest-v2.json +++ b/dist/public/bitcoin-com-mainnet-rest-v2.json @@ -1129,7 +1129,7 @@ "openapi": "3.0.0", "info": { "description": "rest.bitcoin.com is the REST layer for Bitcoin.com's Cloud. More info: [developer.bitcoin.com/rest](https://developer.bitcoin.com/rest). Chatroom [geni.us/CashDev](http://geni.us/CashDev)", - "version": "3.11.1", + "version": "3.11.2", "title": "REST", "license": { "name": "MIT", diff --git a/dist/public/bitcoin-com-testnet-rest-v2.json b/dist/public/bitcoin-com-testnet-rest-v2.json index ea5dc603..9a47956e 100644 --- a/dist/public/bitcoin-com-testnet-rest-v2.json +++ b/dist/public/bitcoin-com-testnet-rest-v2.json @@ -1129,7 +1129,7 @@ "openapi": "3.0.0", "info": { "description": "trest.bitcoin.com is the REST layer for Bitcoin.com's Cloud. More info: [developer.bitcoin.com/rest](https://developer.bitcoin.com/rest). Chatroom [geni.us/CashDev](http://geni.us/CashDev)", - "version": "3.11.1", + "version": "3.11.2", "title": "REST", "license": { "name": "MIT", diff --git a/dist/util/winston-logging.js b/dist/util/winston-logging.js index 98fffba2..3ccdeabf 100644 --- a/dist/util/winston-logging.js +++ b/dist/util/winston-logging.js @@ -8,12 +8,14 @@ var winston = require("winston"); require("winston-daily-rotate-file"); var NETWORK = process.env.NETWORK; // Configure daily-rotation transport. +// Configured to only save 20 megs worth of files. Specifically 20 files of +// 1 megabyte each. Old log files will be deleted to make room for new log files. var transport = new winston.transports.DailyRotateFile({ filename: __dirname + "/../../logs/rest-" + NETWORK + "-%DATE%.log", datePattern: "YYYY-MM-DD", zippedArchive: false, maxSize: "1m", - maxFiles: "3", + maxFiles: "100", format: winston.format.combine(winston.format.timestamp(), winston.format.json()) }); transport.on("rotate", function (oldFilename, newFilename) { diff --git a/package.json b/package.json index 00534d70..0db67d8c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rest.bitcoin.com", - "version": "3.11.1", + "version": "3.11.2", "description": "REST API for Bitcoin.com's Cloud", "author": "Gabriel Cardona ", "contributors": [ diff --git a/swaggerJSONFiles/info.json b/swaggerJSONFiles/info.json index 886bef6a..4a82c998 100644 --- a/swaggerJSONFiles/info.json +++ b/swaggerJSONFiles/info.json @@ -2,7 +2,7 @@ "openapi": "3.0.0", "info": { "description": "The Bitcoin Cash JSON PRC over HTTP", - "version": "3.11.1", + "version": "3.11.2", "title": "REST", "license": { "name": "MIT", diff --git a/swaggerJSONFilesBuilt/mainnet/info.json b/swaggerJSONFilesBuilt/mainnet/info.json index d60052b0..94c0949c 100644 --- a/swaggerJSONFilesBuilt/mainnet/info.json +++ b/swaggerJSONFilesBuilt/mainnet/info.json @@ -2,7 +2,7 @@ "openapi": "3.0.0", "info": { "description": "rest.bitcoin.com is the REST layer for Bitcoin.com's Cloud. More info: [developer.bitcoin.com/rest](https://developer.bitcoin.com/rest). Chatroom [geni.us/CashDev](http://geni.us/CashDev)", - "version": "3.11.1", + "version": "3.11.2", "title": "REST", "license": { "name": "MIT", diff --git a/swaggerJSONFilesBuilt/testnet/info.json b/swaggerJSONFilesBuilt/testnet/info.json index 08ffc62e..49be883d 100644 --- a/swaggerJSONFilesBuilt/testnet/info.json +++ b/swaggerJSONFilesBuilt/testnet/info.json @@ -2,7 +2,7 @@ "openapi": "3.0.0", "info": { "description": "trest.bitcoin.com is the REST layer for Bitcoin.com's Cloud. More info: [developer.bitcoin.com/rest](https://developer.bitcoin.com/rest). Chatroom [geni.us/CashDev](http://geni.us/CashDev)", - "version": "3.11.1", + "version": "3.11.2", "title": "REST", "license": { "name": "MIT",