From 86d6fbe80cc76cc9899ceb46c52f44494408bf1a Mon Sep 17 00:00:00 2001 From: Mokhtar
Date: Mon, 25 Jul 2022 17:27:38 +0200 Subject: [PATCH 01/28] Bulk user upload (#7678) This PR adds a button to the users admin page to create users in bulk from a CSV file. Co-authored-by: m5r
Co-authored-by: Jennifer Q <66472237+latin-panda@users.noreply.github.com> Co-authored-by: Ashley <8253488+mrjones-plip@users.noreply.github.com> Co-authored-by: mrjones-plip
Co-authored-by: Njuguna Ndung'u
--- Gruntfile.js | 1 + admin/src/css/theme.less | 30 ++ admin/src/js/controllers/edit-user.js | 2 +- admin/src/js/controllers/multiple-user.js | 150 +++++++ admin/src/js/controllers/upgrade.js | 19 +- admin/src/js/controllers/users.js | 8 + admin/src/js/directives/modal.js | 2 +- admin/src/js/main.js | 1 + admin/src/js/services/create-user.js | 101 +++-- admin/src/js/services/db.js | 14 +- admin/src/js/services/location.js | 3 +- admin/src/js/services/version.js | 14 +- admin/src/templates/edit_user.html | 1 + admin/src/templates/modal.html | 2 +- admin/src/templates/multiple_user_modal.html | 89 ++++ admin/src/templates/upgrade.html | 18 + admin/src/templates/users.html | 9 +- admin/tests/unit/controllers/edit-user.js | 8 +- .../unit/controllers/multiple-user.spec.js | 175 ++++++++ admin/tests/unit/services/update-user.js | 4 +- api/src/controllers/users.js | 32 +- api/src/promise-utils.js | 8 - api/src/routing.js | 30 +- api/src/services/bulk-upload-log.js | 42 ++ api/src/services/token-login.js | 2 +- api/src/services/users.js | 257 ++++++++--- api/tests/mocha/services/token-login.spec.js | 2 +- api/tests/mocha/services/users.spec.js | 420 +++++++++++++++--- .../translations/messages-en.properties | 28 ++ tests/e2e/api/controllers/users.js | 33 +- tests/e2e/users/bulk-upload-test.csv | 2 + tests/e2e/users/bulk-user-upload.wdio-spec.js | 52 +++ tests/page-objects/admin/user.wdio.page.js | 56 ++- 33 files changed, 1404 insertions(+), 211 deletions(-) create mode 100644 admin/src/js/controllers/multiple-user.js create mode 100644 admin/src/templates/multiple_user_modal.html create mode 100644 admin/tests/unit/controllers/multiple-user.spec.js delete mode 100644 api/src/promise-utils.js create mode 100644 api/src/services/bulk-upload-log.js create mode 100644 tests/e2e/users/bulk-upload-test.csv create mode 100644 tests/e2e/users/bulk-user-upload.wdio-spec.js diff --git a/Gruntfile.js b/Gruntfile.js index 4e97d2c746e..22565b84d10 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -129,6 +129,7 @@ module.exports = function(grunt) { test: { files: { ['http://admin:pass@localhost:4984/medic-test']: 'build/ddocs/medic.json', + ['http://admin:pass@localhost:4984/medic-test-logs']: 'build/ddocs/medic/_attachments/ddocs/logs.json', }, }, staging: { diff --git a/admin/src/css/theme.less b/admin/src/css/theme.less index feb99af3300..929e3a20734 100644 --- a/admin/src/css/theme.less +++ b/admin/src/css/theme.less @@ -185,3 +185,33 @@ ul { .people-section { margin-bottom: 30px; } + +.modal { + height: fit-content; + max-height: 70%; + top: 20%; + left: 25%; + width: 50%; + margin: 0px; + padding: 0px; +} + +.modal-dialog { + width: auto; + margin: 0px; + padding: 0px; +} + +.modal-content { + width: auto; +} + +.summary-box { + border-bottom-width:0px !important; + padding-bottom:20px !important; +} + +.summary-number { + font-size: 7rem; + margin-bottom: -15px; +} \ No newline at end of file diff --git a/admin/src/js/controllers/edit-user.js b/admin/src/js/controllers/edit-user.js index 70af95be825..db88f3bd916 100644 --- a/admin/src/js/controllers/edit-user.js +++ b/admin/src/js/controllers/edit-user.js @@ -386,7 +386,7 @@ angular return UpdateUser($scope.editUserModel.username, updates); } - return CreateUser(updates); + return CreateUser.createSingleUser(updates); }) .then(() => { $scope.setFinished(); diff --git a/admin/src/js/controllers/multiple-user.js b/admin/src/js/controllers/multiple-user.js new file mode 100644 index 00000000000..7416a2cdc89 --- /dev/null +++ b/admin/src/js/controllers/multiple-user.js @@ -0,0 +1,150 @@ +angular.module('controllers').controller('MultipleUserCtrl', function( + $scope, + $uibModalInstance, + $window, + CreateUser, + DB +) { + + 'use strict'; + 'ngInject'; + + $scope.status = { uploading: false }; + $scope.displayAddMultipleModal = true; + $scope.displayUnavailableModal = false; + $scope.displayUploadConfirm = false; + $scope.displayProcessingStatus = false; + $scope.displayFinishSummary = false; + $scope.outputFileUrl = ''; + $scope.processTotal = ''; + const USER_LOG_DOC_ID = 'bulk-user-upload-'; + + $scope.onCancel = function () { + $scope.clearScreen(); + $uibModalInstance.dismiss(); + }; + + const getLogsByType = (docPrefix) => { + return DB({ logsDB: true }) + .allDocs({ + startkey: docPrefix, + endkey: docPrefix + '\ufff0', + include_docs: true + }) + .then(result => { + if (!result || !result.rows || !result.rows.length) { + return; + } + return result.rows + .map(row => row.doc) + .sort((a, b) => new Date(b.bulk_uploaded_on) - new Date(a.bulk_uploaded_on)); + }); + }; + + const prepareStringForCSV = (str) => { + if (!str) { + return str; + } + return '"' + str.replace(/"/g, '""') + '"'; + }; + + const convertToCSV = doc => { + if (!doc || !doc.data) { + return; + } + const eol = '\r\n'; + const delimiter = ','; + const columns = ['import.status:excluded', 'import.message:excluded', 'import.username:excluded']; + let output = columns.join(delimiter) + eol; + + doc.data.forEach(record => { + if (record.import) { + output += [ + prepareStringForCSV(record.import.status), + prepareStringForCSV(record.import.message), + prepareStringForCSV(record.username), + ].join(delimiter) + eol; + } + }); + + const file = new Blob([output], { type: 'text/csv;charset=utf-8;' }); + return URL.createObjectURL(file); + }; + + $scope.processUpload = function () { + $scope.clearScreen(); + $scope.displayProcessingStatus = true; + return $scope.uploadedData + .text() + .then(data => CreateUser.createMultipleUsers(data)) + .then(() => getLogsByType(USER_LOG_DOC_ID)) + .then(docs => { + if (!docs || !docs.length) { + // eslint-disable-next-line no-console + console.error('CreateMultipleUser : Error getting logs by type'); + $scope.setError('CreateMultipleUser : Error getting logs by type'); + } else { + $scope.uploadProcessLog = docs[0]; + $scope.processTotal = $scope.uploadProcessLog.progress.parsing.total; + $scope.successUsersNumber = $scope.uploadProcessLog.progress.saving.successful; + $scope.ignoredUsersNumber = $scope.uploadProcessLog.progress.saving.ignored; + $scope.failedUsersNumber = $scope.uploadProcessLog.progress.saving.failed; + $scope.$apply(); + $scope.outputFileUrl = convertToCSV( $scope.uploadProcessLog); + $scope.showFinishSummary(); + } + }) + .catch(error => { + // eslint-disable-next-line no-console + console.error(error, 'CreateMultipleUser : Error processing data after upload'); + $scope.setError(error, 'CreateMultipleUser : Error processing data after upload'); + }); + }; + + $scope.showFinishSummary = function () { + $scope.clearScreen(); + $scope.displayFinishSummary = true; + $scope.$apply(); + }; + + $scope.showDisplayUploadConfirm = function () { + $scope.clearScreen(); + $scope.displayUploadConfirm = true; + $scope.$apply(); + }; + + $scope.backToAppManagement = function () { + $window.location.href = '/admin/#/users'; + $window.location.reload(); + $scope.clearScreen(); + $uibModalInstance.dismiss(); + $scope.$apply(); + }; + + $scope.clearScreen = function () { + $scope.displayAddMultipleModal = false; + $scope.displayUnavailableModal = false; + $scope.displayUploadConfirm = false; + $scope.displayProcessingStatus = false; + $scope.displayFinishSummary = false; + }; + + const upload = function() { + const files = $('#users-upload .uploader')[0].files; + if (!files || files.length === 0) { + return; + } + $scope.usersFilename = files[0].name; + $scope.uploadedData = files[0]; + $scope.showDisplayUploadConfirm(); + }; + + angular.element(function () { + $('#users-upload .uploader').on('change', upload); + $('#users-upload .choose').on('click', function(e) { + e.preventDefault(); + $('#users-upload .uploader').click(); + }); + }); +} +); diff --git a/admin/src/js/controllers/upgrade.js b/admin/src/js/controllers/upgrade.js index 1c43ecf09f4..32c584617ad 100644 --- a/admin/src/js/controllers/upgrade.js +++ b/admin/src/js/controllers/upgrade.js @@ -40,6 +40,9 @@ angular.module('controllers').controller('UpgradeCtrl', return DB().get('_design/medic') .then(function(ddoc) { $scope.currentDeploy = ddoc.deploy_info; + + const currentVersion = Version.currentVersion($scope.currentDeploy); + $scope.isUsingFeatureRelease = !!currentVersion && typeof currentVersion.featureRelease !== 'undefined'; }); }; @@ -94,7 +97,21 @@ angular.module('controllers').controller('UpgradeCtrl', endkey: [ 'release', 'medic', 'medic', minVersion.major, minVersion.minor, minVersion.patch], descending: true, limit: 50 - }) + }), + featureReleases: !$scope.isUsingFeatureRelease ? builds({ + startkey: [minVersion.featureRelease, 'medic', 'medic', {}], + endkey: [ + minVersion.featureRelease, + 'medic', + 'medic', + minVersion.major, + minVersion.minor, + minVersion.patch, + minVersion.beta, + ], + descending: true, + limit: 50, + }) : [], }).then(function(results) { $scope.versions = results; }); diff --git a/admin/src/js/controllers/users.js b/admin/src/js/controllers/users.js index b7af055d855..652018985ee 100644 --- a/admin/src/js/controllers/users.js +++ b/admin/src/js/controllers/users.js @@ -52,6 +52,14 @@ angular.module('controllers').controller('UsersCtrl', }); }; + $scope.showAddMultipleUsersModal = function() { + Modal({ + templateUrl: 'templates/multiple_user_modal.html', + controller: 'MultipleUserCtrl', + model: {}, + }); + }; + $scope.$on('UsersUpdated', function() { $scope.updateList(); }); diff --git a/admin/src/js/directives/modal.js b/admin/src/js/directives/modal.js index aea188ba0f5..a92eefa5213 100644 --- a/admin/src/js/directives/modal.js +++ b/admin/src/js/directives/modal.js @@ -52,7 +52,7 @@ angular.module('directives').directive('mmModal', function() { disableSubmit: '=', // string: (optional) the expression which, if true, will show the delete button - showDelete: '=' + showDelete: '=', } }; }); diff --git a/admin/src/js/main.js b/admin/src/js/main.js index b8f4589a3b1..28377f84c19 100644 --- a/admin/src/js/main.js +++ b/admin/src/js/main.js @@ -66,6 +66,7 @@ require('./controllers/targets-edit'); require('./controllers/upgrade'); require('./controllers/upgrade-confirm'); require('./controllers/users'); +require('./controllers/multiple-user'); angular.module('directives', ['ngSanitize']); require('./directives/file-model'); diff --git a/admin/src/js/services/create-user.js b/admin/src/js/services/create-user.js index 425e383a82e..6eb34581baf 100644 --- a/admin/src/js/services/create-user.js +++ b/admin/src/js/services/create-user.js @@ -1,38 +1,67 @@ -angular.module('services').factory('CreateUser', - function( - $http, - $log, - $q - ) { - 'ngInject'; - 'use strict'; - - /** - * Creates a user from a collection of updates - * - * Updates are in the style of the /api/v1/users/{username} service, see - * its documentation for more details. - * - * @param {Object} updates Updates you wish to make - */ - return function(updates) { - const url = '/api/v1/users'; - - if (!updates.username) { - return $q.reject('You must provide a username to create a user'); - } - - $log.debug('CreateUser', url, updates); - - return $http({ - method: 'POST', - url: url, - data: updates, - headers: { - 'Content-Type': 'application/json', - 'Accept': 'application/json' +(function () { + + 'use strict'; + const URL = '/api/v2/users'; + + angular.module('services').factory('CreateUser', + function ( + $http, + $log, + $q + ) { + 'ngInject'; + + /** + * Creates a user from a collection of updates + * + * Updates are in the style of the /api/v2/users/{username} service, see + * its documentation for more details. + * + * @param {Object} updates Updates you wish to make + */ + const createSingleUser = (updates) => { + if (!updates.username) { + return $q.reject('You must provide a username to create a user'); } - }); - }; - } + + $log.debug('CreateSingleUser', URL, updates); + + return $http({ + method: 'POST', + url: URL, + data: updates, + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + }); + }; + + /** + * Creates a user from a collection of updates + * + * @param {Object} data content of the csv file + */ + const createMultipleUsers = (data) => { + $log.debug('CreateMultipleUsers', URL, data); + + return $http({ + method: 'POST', + url: URL, + data: data, + headers: { + 'Content-Type': 'text/csv', + 'Accept': 'application/json' + } + }); + }; + + return { + createSingleUser, + createMultipleUsers + }; + } + ); + +}() ); diff --git a/admin/src/js/services/db.js b/admin/src/js/services/db.js index 3cb35b4546d..cdf730cf637 100644 --- a/admin/src/js/services/db.js +++ b/admin/src/js/services/db.js @@ -5,6 +5,7 @@ const DISALLOWED_CHARS = /[^a-z0-9_$()+/-]/g; const USER_DB_SUFFIX = 'user'; const META_DB_SUFFIX = 'meta'; const USERS_DB_SUFFIX = 'users'; +const MEDIC_LOGS_DB_SUFFIX = 'logs'; angular.module('inboxServices').factory('DB', function( @@ -31,14 +32,19 @@ angular.module('inboxServices').factory('DB', return username.replace(DISALLOWED_CHARS, match => `(${match.charCodeAt(0)})`); }; - const getDbName = (remote, meta, usersMeta) => { + const getDbName = (remote, meta, usersMeta, logsDB) => { const parts = []; if (remote) { parts.push(Location.url); } else { parts.push(Location.dbName); } - if ((!remote || meta) && !usersMeta) { + + if (logsDB) { + parts.push(MEDIC_LOGS_DB_SUFFIX); + } + + if ((!remote || meta) && !usersMeta && !logsDB) { parts.push(USER_DB_SUFFIX); parts.push(getUsername(remote)); } else if (usersMeta) { @@ -63,8 +69,8 @@ angular.module('inboxServices').factory('DB', return clone; }; - const get = ({ remote=isOnlineOnly, meta=false, usersMeta=false }={}) => { - const name = getDbName(remote, meta, usersMeta); + const get = ({ remote=isOnlineOnly, meta=false, usersMeta=false, logsDB=false }={}) => { + const name = getDbName(remote, meta, usersMeta, logsDB); if (!cache[name]) { cache[name] = pouchDB(name, getParams(remote, meta, usersMeta)); } diff --git a/admin/src/js/services/location.js b/admin/src/js/services/location.js index b3077e811ca..eb0f3c44f0a 100644 --- a/admin/src/js/services/location.js +++ b/admin/src/js/services/location.js @@ -5,7 +5,8 @@ angular.module('inboxServices').factory('Location', 'ngInject'; const location = $window.location; - const dbName = 'medic'; + const isTestEnv = $window.localStorage.getItem('isTestEnv'); + const dbName = isTestEnv ? 'medic-test' : 'medic'; const path = '/'; const adminPath = '/admin/'; const port = location.port ? ':' + location.port : ''; diff --git a/admin/src/js/services/version.js b/admin/src/js/services/version.js index 95f45adf8f4..9995d27cde4 100644 --- a/admin/src/js/services/version.js +++ b/admin/src/js/services/version.js @@ -16,8 +16,10 @@ angular.module('services').factory('Version', }; const versionInformation = function(versionString) { + // TODO: replace this regex with named capture groups once we deprecate node 8 + // /^(?
\d+)\.(?
\d+)\.(?
\d+)(?
-FR(?:-\w+)+)?(?:-beta\.(?
\d+))?$/ const versionMatch = versionString && - versionString.match(/^([0-9]+)\.([0-9]+)\.([0-9]+)(?:-beta\.([0-9]+))?$/); + versionString.match(/^(\d+)\.(\d+)\.(\d+)(-FR(?:-\w+)+)?(?:-beta\.(\d+))?$/); if (versionMatch) { const version = { @@ -26,8 +28,16 @@ angular.module('services').factory('Version', patch: parseInt(versionMatch[3]) }; + if (versionMatch[5] !== undefined) { + version.beta = parseInt(versionMatch[5]); + } + if (versionMatch[4] !== undefined) { - version.beta = parseInt(versionMatch[4]); + version.featureRelease = versionMatch[4].slice(1); // remove leading dash '-' + + if (version.beta) { + version.featureRelease += '-beta'; + } } return version; diff --git a/admin/src/templates/edit_user.html b/admin/src/templates/edit_user.html index 5a5f6e13f00..b92696d2a2d 100644 --- a/admin/src/templates/edit_user.html +++ b/admin/src/templates/edit_user.html @@ -7,6 +7,7 @@ on-cancel="cancel()" on-submit="editUser()" > +
diff --git a/admin/src/templates/modal.html b/admin/src/templates/modal.html index 28620a7b56a..f7b77679302 100644 --- a/admin/src/templates/modal.html +++ b/admin/src/templates/modal.html @@ -4,7 +4,7 @@
{{titleKey}}
-