Merge branch 'master' into dependabot/npm_and_yarn/backend/axios-1.6.2

backend/Dockerfile.pe

@@ -9,6 +9,8 @@ COPY src ./src
 
 RUN apt update && apt install git zlib1g-dev
 
+RUN apt-get update && apt-get install -y jq
+
 RUN wget -c https://www.python.org/ftp/python/3.10.11/Python-3.10.11.tar.xz && tar -Jxvf Python-3.10.11.tar.xz
 RUN cd Python-3.10.11 && ./configure && make -j4 && make altinstall
 RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3.10 1
@@ -17,8 +19,25 @@ RUN pip3.10 install --upgrade pip
 
 RUN apt remove dav1d && apt autoclean && apt autoremove
 
+# Install AWS CLI
+RUN curl --insecure "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip
+RUN ./aws/install
+
 # Install pe-source module
 # Sync the latest from cf-staging branch
-RUN git clone -b cf-source-staging https://github.com/cisagov/pe-reports.git && cd pe-reports && git checkout c9cbbd73b22ef38cabe1da6ba50aeb2dc0be4f99 && pip install .
+RUN git clone -b AL-staging-SQS https://github.com/cisagov/pe-reports.git && \
+    cd pe-reports && \
+    git checkout 6dcd017551ba14022e110c073e1bdbc804c795f8 && \
+    pip install .
+
+RUN python -m spacy download en_core_web_lg
+
+# Create database.ini 
+RUN echo "[database]" > database.ini \ 
+    && echo "user=$(cat db_user.txt)" >> database.ini \ 
+    && echo "password=$(cat db_password.txt)" >> database.ini
+
+COPY worker worker
 
-CMD ["./worker/pe-worker-entry.sh"]
+CMD ["./worker/generate_config.sh", "./worker/pe-worker-entry.sh"]

backend/package.json

@@ -23,6 +23,7 @@
     "express": "^4.18.1",
     "global-agent": "^2.2.0",
     "got": "^11.8.5",
+    "handlebars": "^4.7.8",
     "helmet": "^4.1.1",
     "http-proxy-middleware": "^2.0.6",
     "jsdom": "^22.1",

backend/serverless.yml

@@ -43,6 +43,9 @@ provider:
           Action:
             - ecs:RunTask
             - ecs:ListTasks
+            - ecs:DescribeTasks
+            - ecs:DescribeServices
+            - ecs:UpdateService
             - iam:PassRole
           Resource: '*'
         - Effect: Allow
@@ -65,7 +68,9 @@ provider:
         - Effect: Allow
           Action:
             - sqs:ReceiveMessage
+            - sqs:DeleteMessage
             - sqs:SendMessage
+            - sqs:GetQueueAttributes
           Resource: '*'
         - Effect: Allow
           Action:
@@ -108,7 +113,6 @@ resources:
         MaximumMessageSize: 262144 # 256 KB
         MessageRetentionPeriod: 604800 # 7 days
 
-
 functions:
   - ${file(./src/tasks/functions.yml)}
   - ${file(./src/api/functions.yml)}

backend/src/api/app.ts

@@ -63,6 +63,7 @@ app.use(cookieParser());
 app.get('/', handlerToExpress(healthcheck));
 app.post('/auth/login', handlerToExpress(auth.login));
 app.post('/auth/callback', handlerToExpress(auth.callback));
+app.post('/users/register', handlerToExpress(users.register));
 
 const checkUserLoggedIn = async (req, res, next) => {
   req.requestContext = {
@@ -227,6 +228,7 @@ app.use(
 const authenticatedNoTermsRoute = express.Router();
 authenticatedNoTermsRoute.use(checkUserLoggedIn);
 authenticatedNoTermsRoute.get('/users/me', handlerToExpress(users.me));
+// authenticatedNoTermsRoute.post('/users/register', handlerToExpress(users.register));
 authenticatedNoTermsRoute.post(
   '/users/me/acceptTerms',
   handlerToExpress(users.acceptTerms)
@@ -313,6 +315,14 @@ authenticatedRoute.get(
   '/organizations/:organizationId',
   handlerToExpress(organizations.get)
 );
+authenticatedRoute.get(
+  '/organizations/state/:state',
+  handlerToExpress(organizations.getByState)
+);
+authenticatedRoute.get(
+  '/organizations/regionId/:regionId',
+  handlerToExpress(organizations.getByRegionId)
+);
 authenticatedRoute.post(
   '/organizations',
   handlerToExpress(organizations.create)
@@ -325,6 +335,10 @@ authenticatedRoute.delete(
   '/organizations/:organizationId',
   handlerToExpress(organizations.del)
 );
+authenticatedRoute.post(
+  '/v2/organizations/:organizationId/users',
+  handlerToExpress(organizations.addUserV2)
+);
 authenticatedRoute.post(
   '/organizations/:organizationId/roles/:roleId/approve',
   handlerToExpress(organizations.approveRole)
@@ -349,6 +363,14 @@ authenticatedRoute.post('/stats', handlerToExpress(stats.get));
 authenticatedRoute.post('/users', handlerToExpress(users.invite));
 authenticatedRoute.get('/users', handlerToExpress(users.list));
 authenticatedRoute.delete('/users/:userId', handlerToExpress(users.del));
+authenticatedRoute.get(
+  '/users/state/:state',
+  handlerToExpress(users.getByState)
+);
+authenticatedRoute.get(
+  '/users/regionId/:regionId',
+  handlerToExpress(users.getByRegionId)
+);
 authenticatedRoute.post('/users/search', handlerToExpress(users.search));
 
 authenticatedRoute.post(
@@ -361,6 +383,35 @@ authenticatedRoute.post(
   handlerToExpress(reports.list_reports)
 );
 
+//Authenticated Registration Routes
+authenticatedRoute.put(
+  '/users/:userId/register/approve',
+  handlerToExpress(users.registrationApproval)
+);
+
+authenticatedRoute.put(
+  '/users/:userId/register/deny',
+  handlerToExpress(users.registrationDenial)
+);
+
+//************* */
+//  V2 Routes   //
+//************* */
+
+// Users
+authenticatedRoute.put('/v2/users/:userId', handlerToExpress(users.updateV2));
+authenticatedRoute.get('/v2/users', handlerToExpress(users.getAllV2));
+
+// Organizations
+authenticatedRoute.put(
+  '/v2/organizations/:organizationId',
+  handlerToExpress(organizations.updateV2)
+);
+authenticatedRoute.get(
+  '/v2/organizations',
+  handlerToExpress(organizations.getAllV2)
+);
+
 app.use(authenticatedRoute);
 
 export default app;

backend/src/api/auth.ts

@@ -257,6 +257,15 @@ export const isGlobalViewAdmin = (event: APIGatewayProxyEvent) => {
     : false;
 };
 
+/** Check if a user has regionalAdmin view permissions */
+export const isRegionalAdmin = (event: APIGatewayProxyEvent) => {
+  return event.requestContext.authorizer &&
+    (event.requestContext.authorizer.userType === UserType.REGIONAL_ADMIN ||
+      event.requestContext.authorizer.userType === UserType.GLOBAL_ADMIN)
+    ? true
+    : false;
+};
+
 /** Checks if the current user is allowed to access (modify) a user with id userId */
 export const canAccessUser = (event: APIGatewayProxyEvent, userId?: string) => {
   return userId && (userId === getUserId(event) || isGlobalWriteAdmin(event));

backend/src/api/helpers.ts

@@ -9,6 +9,9 @@ import { ClassType } from 'class-transformer/ClassTransformer';
 import { plainToClass } from 'class-transformer';
 import { SES } from 'aws-sdk';
 import * as nodemailer from 'nodemailer';
+import * as fs from 'fs';
+import * as handlebars from 'handlebars';
+import * as util from 'util';
 
 export const validateBody = async <T>(
   obj: ClassType<T>,
@@ -71,12 +74,12 @@ export const wrapHandler: WrapHandler =
 
 export const NotFound: APIGatewayProxyResult = {
   statusCode: 404,
-  body: ''
+  body: 'Item not found. View logs for details.'
 };
 
 export const Unauthorized: APIGatewayProxyResult = {
   statusCode: 403,
-  body: ''
+  body: 'Unauthorized access. View logs for details.'
 };
 
 export const sendEmail = async (
@@ -96,3 +99,142 @@ export const sendEmail = async (
     replyTo: process.env.CROSSFEED_SUPPORT_EMAIL_REPLYTO!
   });
 };
+
+export const sendUserNotificationEmail = async (
+  recepient: string,
+  p_subject: string,
+  p_firstName: string,
+  p_lastname: string,
+  template_file: string
+) => {
+  const transporter = nodemailer.createTransport({
+    SES: new SES({ region: 'us-east-1' })
+  });
+
+  const fs = require('fs').promises;
+  const html = await fs.readFile(template_file, 'utf8');
+  const template = handlebars.compile(html);
+  const data = {
+    first_name: p_firstName,
+    last_name: p_lastname
+  };
+
+  const htmlToSend = template(data);
+
+  const mailOptions = {
+    from: process.env.CROSSFEED_SUPPORT_EMAIL_SENDER,
+    to: recepient,
+    subject: p_subject,
+    html: htmlToSend,
+    attachments: [
+      {
+        filename: 'banner.png',
+        path: '/app/src/email_templates/banner.png',
+        cid: 'CISA Banner'
+      },
+      {
+        filename: 'web.png',
+        path: '/app/src/email_templates/banner.png',
+        cid: 'CISA Web'
+      },
+      {
+        filename: 'email.png',
+        path: '/app/src/email_templates/email.png',
+        cid: 'CISA Email'
+      },
+      {
+        filename: 'linkedin.png',
+        path: '/app/src/email_templates/linkedin.png',
+        cid: 'CISA LinkedIn'
+      },
+      {
+        filename: 'twitter.png',
+        path: '/app/src/email_templates/twitter.png',
+        cid: 'CISA Twitter'
+      },
+      {
+        filename: 'facebook.png',
+        path: '/app/src/email_templates/facebooK.png',
+        cid: 'CISA Facebook'
+      },
+      {
+        filename: 'instagram.png',
+        path: '/app/src/email_templates/instagram.png',
+        cid: 'CISA Instagram'
+      }
+    ]
+  };
+
+  await transporter.sendMail(mailOptions);
+};
+
+export const sendRegionalAdminNotificationEmail = async (
+  recepient: string,
+  p_subject: string,
+  p_firstName: string,
+  p_lastname: string,
+  p_username: string
+) => {
+  const transporter = nodemailer.createTransport({
+    SES: new SES({ region: 'us-east-1' })
+  });
+
+  const fs = require('fs').promises;
+  const html = await fs.readFile(
+    '/app/src/email_templates/crossfeed_regional_admin_notification.html',
+    'utf8'
+  );
+  const template = handlebars.compile(html);
+  const data = {
+    first_name: p_firstName,
+    last_name: p_lastname
+  };
+
+  const htmlToSend = template(data);
+
+  const mailOptions = {
+    from: process.env.CROSSFEED_SUPPORT_EMAIL_SENDER,
+    to: recepient,
+    subject: p_subject,
+    html: htmlToSend,
+    attachments: [
+      {
+        filename: 'banner.png',
+        path: '/app/src/email_templates/banner.png',
+        cid: 'CISA Banner'
+      },
+      {
+        filename: 'web.png',
+        path: '/app/src/email_templates/banner.png',
+        cid: 'CISA Web'
+      },
+      {
+        filename: 'email.png',
+        path: '/app/src/email_templates/email.png',
+        cid: 'CISA Email'
+      },
+      {
+        filename: 'linkedin.png',
+        path: '/app/src/email_templates/linkedin.png',
+        cid: 'CISA LinkedIn'
+      },
+      {
+        filename: 'twitter.png',
+        path: '/app/src/email_templates/twitter.png',
+        cid: 'CISA Twitter'
+      },
+      {
+        filename: 'facebook.png',
+        path: '/app/src/email_templates/facebooK.png',
+        cid: 'CISA Facebook'
+      },
+      {
+        filename: 'instagram.png',
+        path: '/app/src/email_templates/instagram.png',
+        cid: 'CISA Instagram'
+      }
+    ]
+  };
+
+  await transporter.sendMail(mailOptions);
+};