Feat: Email with any SMTP server + Testing Email

.env.sample

@@ -6,8 +6,13 @@ LOCAL_TIMEZONE=US/Pacific
 
 GOOGLE_CLIENT_ID=
 GOOGLE_CLIENT_SECRET=
+
 CANVAS_SERVER_URL=
 CANVAS_CLIENT_ID=
 CANVAS_CLIENT_SECRET=
 CANVAS_COURSE_ID=
-SENDGRID_API_KEY=
+
+EMAIL_SERVER=
+EMAIL_PORT=
+EMAIL_USE_TLS=
+EMAIL_USERNAME=

README.md

@@ -105,27 +105,7 @@ Student and email data could come from different sources (Google Sheets, csv upl
 
 #### Emailing students
 
-(Under Construction; content might be outdated)
-
-Students will receive an email that looks like
-
-```
-Hi -name-,
-
-Here's your assigned seat for -exam-:
-
-Room: -room-
-
-Seat: -seat-
-
-You can view this seat's position on the seating chart at:
-<domain>/seat/-seatid-/
-
--additional text-
-```
-
-The "additional text" is a good place to tell them what to do if they have an
-issue with their seat, and to sign the email (e.g. "- Cal CS 61A Staff").
+In the emailing page, you should specify minimally sender's address and sender's signature. The signature is appended to the end of the email body. You can also specify additional text to be appended to the end of the email body (this is a good place to tell students what to do if they have concerns about their seating assignments). The email body is generated from a template. The template is a html file that you can edit. The template is located at `server/services/email/templates/assignment_inform_email.html` and its variables are automatically filled in.
 
 #### Get Roster Photos
 
@@ -231,11 +211,14 @@ CANVAS_SERVER_URL=
 CANVAS_CLIENT_ID=
 CANVAS_CLIENT_SECRET=
 
-# sendgrid api key, get it from sendgrid dashboard
-SENDGRID_API_KEY=
+# email setup, use any smtp server
+EMAIL_SERVER=
+EMAIL_PORT=
+EMAIL_USE_TLS=
+EMAIL_USERNAME=
 
 # misc
-DOMAIN=localhost:5000
+SERVER_BASE_URL=localhost:5000
 LOCAL_TIMEZONE=US/Pacific
 ```
 

config.py

@@ -35,10 +35,13 @@ def getenv(key, default: Optional[str] = None):
     MOCK_CANVAS = getenv('MOCK_CANVAS', 'false').lower() == 'true'
     SEND_EMAIL = getenv('SEND_EMAIL', 'off').lower()
 
-    # Email setup. Domain environment is for link in email.
-    SENDGRID_API_KEY = getenv('SENDGRID_API_KEY', "placeholder")
+    # Email setup.
+    EMAIL_SERVER = getenv('EMAIL_SERVER', "unset")
+    EMAIL_PORT = getenv('EMAIL_PORT', "unset")
+    EMAIL_USERNAME = getenv('EMAIL_USERNAME', "unset")
+    EMAIL_PASSWORD = getenv('EMAIL_PASSWORD', "unset")
 
-    PHOTO_DIRECTORY = getenv('PHOTO_DIRECTORY', "placeholder")
+    PHOTO_DIRECTORY = getenv('PHOTO_DIRECTORY', "unset")
 
 
 class ProductionConfig(ConfigBase):

requirements-dev.txt

@@ -6,4 +6,5 @@ safety==1.10.3
 pip-audit==2.6.1
 selenium==4.14.0
 responses==0.23.3
+aiosmtpd==1.4.4.post2
 flask-fixtures==0.3.8

requirements.txt

@@ -8,7 +8,6 @@ Flask_WTF==1.1.1
 google_api_python_client==2.101.0
 natsort==8.4.0
 oauth2client==4.1.3
-sendgrid==6.10.0
 SQLAlchemy==1.4.42
 Werkzeug==0.16.1
 WTForms==3.0.1

server/forms.py

@@ -83,11 +83,11 @@ class AssignForm(FlaskForm):
 
 
 class EmailForm(FlaskForm):
-    from_email = StringField('from_email', [Email()])
-    from_name = StringField('from_name', [InputRequired()])
-    subject = StringField('subject', [InputRequired()])
-    test_email = StringField('test_email')
-    additional_text = TextAreaField('additional_text')
+    from_addr = StringField('from_addr', [Email(), InputRequired()])
+    signature = StringField('signature', [InputRequired()])
+    additional_info = TextAreaField('additional_info')
+    override_subject = StringField('override_subject')
+    override_to_addr = StringField('override_to_addr')
     submit = SubmitField('send')
 
 

server/models.py

@@ -84,6 +84,18 @@
     def unassigned_students(self):
         return [student for student in self.students if student.assignment == None]  # noqa
 
+    def get_assignments(self, emailed=None, limit=None, offset=None):
+        query = SeatAssignment.query.join(SeatAssignment.seat).join(Seat.room).filter(
+            Room.exam_id == self.id,
+        )
+        if emailed is not None:
+            query = query.filter(SeatAssignment.emailed == emailed)
+        if limit is not None:
+            query = query.limit(limit)
+        if offset is not None:
+            query = query.offset(offset)
+        return query.all()
+
     def __repr__(self):
         return '<Exam {}>'.format(self.name)
 

server/services/email/__init__.py

@@ -1,73 +1,73 @@
+from server import app
+from server.models import db, Exam, SeatAssignment, Offering
+from server.services.email.smtp import SMTPConfig, send_email
+import server.services.email.templates as templates
+from server.typings.enum import EmailTemplate
+from flask import url_for
+import os
 
-import sendgrid
+_email_config = SMTPConfig(
+    app.config.get('EMAIL_SERVER'),
+    app.config.get('EMAIL_PORT'),
+    app.config.get('EMAIL_USERNAME'),
+    app.config.get('EMAIL_PASSWORD')
+)
 
-from server import app
-from server.models import db, Room, Seat, SeatAssignment
 
+def email_students(exam: Exam, form):
+    ASSIGNMENT_PER_PAGE = 500
+    page_number = 1
+    emailed_count = 0
 
-def email_students(exam, form):
-    """Emails students in batches of 900"""
-    sg = sendgrid.SendGridAPIClient(api_key=app.config['SENDGRID_API_KEY'])
-    test = form.test_email.data
     while True:
-        limit = 1 if test else 900
-        assignments = SeatAssignment.query.join(SeatAssignment.seat).join(Seat.room).filter(
-            Room.exam_id == exam.id,
-            not SeatAssignment.emailed
-        ).limit(limit).all()
+        assignments = exam.get_assignments(
+            emailed=False,
+            limit=ASSIGNMENT_PER_PAGE,
+            offset=(page_number - 1) * ASSIGNMENT_PER_PAGE
+        )
         if not assignments:
             break
+        page_number += 1
 
-        data = {
-            'personalizations': [
-                {
-                    'to': [
-                        {
-                            'email': test if test else assignment.student.email,
-                        }
-                    ],
-                    'substitutions': {
-                        '-name-': assignment.student.first_name,
-                        '-room-': assignment.seat.room.display_name,
-                        '-seat-': assignment.seat.name,
-                        '-seatid-': str(assignment.seat.id),
-                    },
-                }
-                for assignment in assignments
-            ],
-            'from': {
-                'email': form.from_email.data,
-                'name': form.from_name.data,
-            },
-            'subject': form.subject.data,
-            'content': [
-                {
-                    'type': 'text/plain',
-                    'value': '''
-Hi -name-,
+        for assignment in assignments:
+            result = _email_single_assignment(exam.offering, exam, assignment, form)
+            if result[0]:
+                emailed_count += 1
+                assignment.emailed = True
+            else:
+                db.session.commit()
+                return result
+        else:
+            db.session.commit()
 
-Here's your assigned seat for {}:
+    if emailed_count == 0:
+        return (False, "No unemailed assignments found.")
 
-Room: -room-
+    return (True, )
 
-Seat: -seat-
 
-You can view this seat's position on the seating chart at:
-{}/seat/-seatid-/
+def _email_single_assignment(offering: Offering, exam: Exam, assignment: SeatAssignment, form) -> bool:
+    seat_path = url_for('student_single_seat', seat_id=assignment.seat.id)
+    seat_absolute_path = os.path.join(app.config.get('SERVER_BASE_URL'), seat_path)
 
-{}
-'''.format(exam.display_name, app.config['DOMAIN'], form.additional_text.data)
-                },
-            ],
-        }
+    student_email = \
+        templates.get_email(EmailTemplate.ASSIGNMENT_INFORM_EMAIL,
+                            {"EXAM": exam.display_name},
+                            {"NAME": assignment.student.first_name,
+                                "COURSE": offering.name,
+                                "EXAM": exam.display_name,
+                                "ROOM": assignment.seat.room.display_name,
+                                "SEAT": assignment.seat.name,
+                                "URL": seat_absolute_path,
+                                "ADDITIONAL_INFO": form.additional_info.data,
+                                "SIGNATURE": form.signature.data})
 
-        response = sg.client.mail.send.post(request_body=data)
-        if response.status_code < 200 or response.status_code >= 400:
-            raise Exception('Could not send mail. Status: {}. Body: {}'.format(
-                response.status_code, response.body
-            ))
-        if test:
-            return
-        for assignment in assignments:
-            assignment.emailed = True
-        db.session.commit()
+    effective_to_addr = form.override_to_addr.data or assignment.student.email
+    effective_subject = form.override_subject.data or student_email.subject
+
+    return send_email(smtp=_email_config,
+                      from_addr=form.from_addr.data,
+                      to_addr=effective_to_addr,
+                      subject=effective_subject,
+                      body=student_email.body,
+                      body_html=student_email.body if student_email.body_html else None)

server/services/email/smtp.py

@@ -0,0 +1,40 @@
+from smtplib import SMTP
+from email.message import EmailMessage
+
+
+class SMTPConfig:
+    def __init__(self, smtp_server, smtp_port, username, password, use_tls=True, use_auth=True):
+        self.smtp_server = smtp_server
+        self.smtp_port = smtp_port
+        self.username = username
+        self.password = password
+        self.use_tls = use_tls
+        self.use_auth = use_auth
+
+    def __repr__(self) -> str:
+        return f'SMTPConfig(smtp_server={self.smtp_server}, smtp_port={self.smtp_port}, username={self.username}, use_tls={self.use_tls}, use_auth={self.use_auth})'  # noqa
+
+
+def send_email(*, smtp: SMTPConfig, from_addr, to_addr, subject, body, body_html=None):
+    msg = EmailMessage()
+    msg['From'] = from_addr
+    msg['To'] = to_addr
+    msg['Subject'] = subject
+    msg.set_content(body)
+
+    if body_html:
+        msg.add_alternative(body_html, subtype='html')
+
+    try:
+        server = SMTP(smtp.smtp_server, smtp.smtp_port)
+        # if server.has_extn('STARTTLS'):
+        if smtp.use_tls:
+            server.starttls()
+        # if server.has_extn('AUTH'):
+        if smtp.use_auth:
+            server.login(smtp.username, smtp.password)
+        server.send_message(msg)
+        server.quit()
+        return (True, )
+    except Exception as e:
+        return (False, f"Error sending email: {e.message}\n Config: \n{smtp}")

server/services/email/templates/__init__.py

@@ -0,0 +1,40 @@
+from server.typings.enum import EmailTemplate
+import os
+import json
+import re
+
+
+class EmailContent:
+    def __init__(self, subject: str, body: str, body_html: bool):
+        self.subject = subject
+        self.body = body
+        self.body_html = body_html
+
+    def __repr__(self) -> str:
+        return f'EmailContent(subject={self.subject}, body={self.body}, body_type={self.body_html})'
+
+
+def get_email(
+        template: EmailTemplate,
+        subject_substitutions: dict[str, str],
+        body_substitutions: dict[str, str]
+) -> EmailContent:
+    # template.value is the file name for email metadata (json)
+    # let us read it that first. The file sits in the same path as this file.
+    email_metadata = None
+    with open(os.path.join(os.path.dirname(__file__), template.value + ".json")) as f:
+        email_metadata = json.load(f)
+    subject = _make_substitutions(email_metadata['subject'], subject_substitutions)
+    body_html = email_metadata['body_html']
+    body_path = os.path.join(os.path.dirname(__file__), email_metadata['body_path'])
+    body = None
+    with open(body_path) as f:
+        body = _make_substitutions(f.read(), body_substitutions)
+    content = EmailContent(subject, body, re.match('[Tt]rue', body_html) is not None)
+    return content
+
+
+def _make_substitutions(text: str, substitutions: dict[str, str]) -> str:
+    for key, value in substitutions.items():
+        text = re.sub(r'\{\{\s*' + key + r'\s*\}\}', value, text)
+    return text

server/services/email/templates/assignment_inform_email.html

@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Seating Assignment for {{COURSE}} {{EXAM}}</title>
+  </head>
+  <body>
+    <p>Dear <strong>{{NAME}}</strong>,</p>
+
+    <p>Here's your assigned seat for the upcoming exam:</p>
+
+    <ul>
+      <li>Course: <strong>{{COURSE}}</strong></li>
+      <li>Exam: <strong>{{EXAM}}</strong></li>
+      <li>Room: <strong>{{ROOM}}</strong></li>
+      <li>Seat: <strong>{{SEAT}}</strong></li>
+    </ul>
+
+    <p>You can view this seat's position on the seating chart at:</p>
+    <a href="{{URL}}">View Seating Chart</a>
+
+    <p>
+      If you have any questions or require further assistance, please do not
+      hesitate to contact the relevant course staff.
+    </p>
+
+    <p>Good luck on your exam!</p>
+
+    <p>{{ADDITIONAL_INFO}}</p>
+
+    <p>Best,</p>
+    <p>{{SIGNATURE}}</p>
+  </body>
+</html>

server/services/email/templates/assignment_inform_email.json

@@ -0,0 +1,5 @@
+{
+  "subject": "Your Exam Seating Assignment for {{EXAM}}",
+  "body_path": "assignment_inform_email.html",
+  "body_html": "true"
+}