Skip to content

Commit

Permalink
feat!: upgrade pymongo
Browse files Browse the repository at this point in the history
  • Loading branch information
mumarkhan999 committed Jul 25, 2024
1 parent 5198496 commit 2995608
Show file tree
Hide file tree
Showing 16 changed files with 248 additions and 87 deletions.
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *
mv requirements/common_constraints.tmp requirements/common_constraints.txt
sed 's/Django<4.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp
mv requirements/common_constraints.tmp requirements/common_constraints.txt
sed 's/event-tracking<2.4.1//g' requirements/common_constraints.txt > requirements/common_constraints.tmp
mv requirements/common_constraints.tmp requirements/common_constraints.txt
pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in
pip install -r requirements/pip.txt

Expand Down
3 changes: 0 additions & 3 deletions conf/locale/en/LC_MESSAGES/.gitignore

This file was deleted.

2 changes: 1 addition & 1 deletion requirements/common_constraints.txt
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,4 @@ importlib-metadata<7
# We will pin event-tracking to do not break existing installations
# This can be unpinned once https://github.com/openedx/edx-platform/issues/34586
# has been resolved and edx-platform is running with pymongo>=4.4.0
event-tracking<2.4.1

12 changes: 8 additions & 4 deletions requirements/constraints.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,13 @@ django-oauth-toolkit==1.7.1
# incremental upgrade
django-simple-history==3.4.0

# constrained in opaque_keys. migration guide here: https://pymongo.readthedocs.io/en/4.0/migrate-to-pymongo4.html
# Major upgrade will be done in separate ticket.
pymongo<4.0.0
# Adding pin to avoid any major upgrade
pymongo<4.4.1

# To override the constraint of edx-lint
# This can be removed once https://github.com/openedx/edx-platform/issues/34586 is resolved
# and the upstream constraint in edx-lint has been removed.
event-tracking==3.0.0

# greater version has breaking changes and requires some migration steps.
django-webpack-loader==0.7.0
Expand Down Expand Up @@ -125,4 +129,4 @@ numpy<2.0.0
# django-storages==1.14.4 breaks course imports
# Two lines were added in 1.14.4 that make file_exists_in_storage function always return False,
# as the default value of AWS_S3_FILE_OVERWRITE is True
django-storages<1.14.4
django-storages<1.14.4
10 changes: 7 additions & 3 deletions requirements/edx/base.txt
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,10 @@ djangorestframework==3.14.0
# super-csv
djangorestframework-xml==2.0.0
# via edx-enterprise
dnspython==2.6.1
# via
# -r requirements/edx/paver.txt
# pymongo
done-xblock==2.3.0
# via -r requirements/edx/bundled.in
drf-jwt==1.19.2
Expand Down Expand Up @@ -536,9 +540,9 @@ enmerkar==0.7.1
# via enmerkar-underscore
enmerkar-underscore==2.3.0
# via -r requirements/edx/kernel.in
event-tracking==2.4.0
event-tracking==3.0.0
# via
# -c requirements/edx/../common_constraints.txt
# -c requirements/edx/../constraints.txt
# -r requirements/edx/kernel.in
# edx-completion
# edx-proctoring
Expand Down Expand Up @@ -865,7 +869,7 @@ pylti1p3==2.0.0
# via -r requirements/edx/kernel.in
pymemcache==4.0.0
# via -r requirements/edx/paver.txt
pymongo==3.13.0
pymongo==4.4.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/kernel.in
Expand Down
8 changes: 5 additions & 3 deletions requirements/edx/development.txt
Original file line number Diff line number Diff line change
Expand Up @@ -614,8 +614,10 @@ djangorestframework-xml==2.0.0
# edx-enterprise
dnspython==2.6.1
# via
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
# email-validator
# pymongo
docutils==0.21.2
# via
# -r requirements/edx/doc.txt
Expand Down Expand Up @@ -853,9 +855,9 @@ enmerkar-underscore==2.3.0
# via
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
event-tracking==2.4.0
event-tracking==3.0.0
# via
# -c requirements/edx/../common_constraints.txt
# -c requirements/edx/../constraints.txt
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
# edx-completion
Expand Down Expand Up @@ -1535,7 +1537,7 @@ pymemcache==4.0.0
# via
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
pymongo==3.13.0
pymongo==4.4.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/doc.txt
Expand Down
10 changes: 7 additions & 3 deletions requirements/edx/doc.txt
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,10 @@ djangorestframework-xml==2.0.0
# via
# -r requirements/edx/base.txt
# edx-enterprise
dnspython==2.6.1
# via
# -r requirements/edx/base.txt
# pymongo
docutils==0.21.2
# via
# pydata-sphinx-theme
Expand Down Expand Up @@ -614,9 +618,9 @@ enmerkar==0.7.1
# enmerkar-underscore
enmerkar-underscore==2.3.0
# via -r requirements/edx/base.txt
event-tracking==2.4.0
event-tracking==3.0.0
# via
# -c requirements/edx/../common_constraints.txt
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt
# edx-completion
# edx-proctoring
Expand Down Expand Up @@ -1026,7 +1030,7 @@ pylti1p3==2.0.0
# via -r requirements/edx/base.txt
pymemcache==4.0.0
# via -r requirements/edx/base.txt
pymongo==3.13.0
pymongo==4.4.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt
Expand Down
4 changes: 3 additions & 1 deletion requirements/edx/paver.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ charset-normalizer==2.0.12
# via
# -c requirements/edx/../constraints.txt
# requests
dnspython==2.6.1
# via pymongo
edx-opaque-keys==2.10.0
# via -r requirements/edx/paver.in
idna==3.7
Expand All @@ -36,7 +38,7 @@ psutil==6.0.0
# via -r requirements/edx/paver.in
pymemcache==4.0.0
# via -r requirements/edx/paver.in
pymongo==3.13.0
pymongo==4.4.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/paver.in
Expand Down
11 changes: 7 additions & 4 deletions requirements/edx/testing.txt
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,10 @@ djangorestframework-xml==2.0.0
# -r requirements/edx/base.txt
# edx-enterprise
dnspython==2.6.1
# via email-validator
# via
# -r requirements/edx/base.txt
# email-validator
# pymongo
done-xblock==2.3.0
# via -r requirements/edx/base.txt
drf-jwt==1.19.2
Expand Down Expand Up @@ -650,9 +653,9 @@ enmerkar==0.7.1
# enmerkar-underscore
enmerkar-underscore==2.3.0
# via -r requirements/edx/base.txt
event-tracking==2.4.0
event-tracking==3.0.0
# via
# -c requirements/edx/../common_constraints.txt
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt
# edx-completion
# edx-proctoring
Expand Down Expand Up @@ -1141,7 +1144,7 @@ pylti1p3==2.0.0
# via -r requirements/edx/base.txt
pymemcache==4.0.0
# via -r requirements/edx/base.txt
pymongo==3.13.0
pymongo==4.4.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt
Expand Down
4 changes: 3 additions & 1 deletion scripts/structures_pruning/requirements/base.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,13 @@ click==8.1.6
# click-log
click-log==0.4.0
# via -r scripts/structures_pruning/requirements/base.in
dnspython==2.6.1
# via pymongo
edx-opaque-keys==2.10.0
# via -r scripts/structures_pruning/requirements/base.in
pbr==6.0.0
# via stevedore
pymongo==3.13.0
pymongo==4.4.0
# via
# -c scripts/structures_pruning/requirements/../../../requirements/constraints.txt
# -r scripts/structures_pruning/requirements/base.in
Expand Down
6 changes: 5 additions & 1 deletion scripts/structures_pruning/requirements/testing.txt
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ click-log==0.4.0
# via -r scripts/structures_pruning/requirements/base.txt
ddt==1.7.2
# via -r scripts/structures_pruning/requirements/testing.in
dnspython==2.6.1
# via
# -r scripts/structures_pruning/requirements/base.txt
# pymongo
edx-opaque-keys==2.10.0
# via -r scripts/structures_pruning/requirements/base.txt
iniconfig==2.0.0
Expand All @@ -24,7 +28,7 @@ pbr==6.0.0
# stevedore
pluggy==1.5.0
# via pytest
pymongo==3.13.0
pymongo==4.4.0
# via
# -r scripts/structures_pruning/requirements/base.txt
# edx-opaque-keys
Expand Down
65 changes: 53 additions & 12 deletions xmodule/contentstore/mongo.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"""


import hashlib
import json
import os

Expand Down Expand Up @@ -40,23 +41,55 @@ def __init__(
# GridFS will throw an exception if the Database is wrapped in a MongoProxy. So don't wrap it.
# The appropriate methods below are marked as autoretry_read - those methods will handle
# the AutoReconnect errors.
proxy = False
mongo_db = connect_to_mongodb(
db, host,
port=port, tz_aware=tz_aware, user=user, password=password, proxy=proxy, **kwargs
)
self.connection_params = {
'db': db,
'host': host,
'port': port,
'tz_aware': tz_aware,
'user': user,
'password': password,
'proxy': False,
**kwargs
}
self.bucket = bucket
self.do_connection()

def do_connection(self):
"""
Connects to mongodb.
"""
mongo_db = connect_to_mongodb(**self.connection_params)

self.fs = gridfs.GridFS(mongo_db, bucket) # pylint: disable=invalid-name
self.fs = gridfs.GridFS(mongo_db, self.bucket) # pylint: disable=invalid-name

self.fs_files = mongo_db[bucket + ".files"] # the underlying collection GridFS uses
self.chunks = mongo_db[bucket + ".chunks"]
self.fs_files = mongo_db[self.bucket + ".files"] # the underlying collection GridFS uses
self.chunks = mongo_db[self.bucket + ".chunks"]

def close_connections(self):
"""
Closes any open connections to the underlying databases
"""
self.fs_files.database.client.close()

def ensure_connection(self):
"""
Ensure that mongodb connection is open.
"""
if self.check_connection():
return
self.do_connection()

def check_connection(self):
"""
Check if mongodb connection is open or not.
"""
connection = self.fs_files.database.client
try:
connection.admin.command('ping')
return True
except pymongo.errors.InvalidOperation:
return False

def _drop_database(self, database=True, collections=True, connections=True):
"""
A destructive operation to drop the underlying database and close all connections.
Expand All @@ -69,8 +102,8 @@ def _drop_database(self, database=True, collections=True, connections=True):
If connections is True, then close the connection to the database as well.
"""
self.ensure_connection()
connection = self.fs_files.database.client

if database:
connection.drop_database(self.fs_files.database.name)
elif collections:
Expand Down Expand Up @@ -103,16 +136,22 @@ def save(self, content):
# but many more objects have this in python3 and shouldn't be using the chunking logic. For string and
# byte streams we write them directly to gridfs and convert them to byetarrys if necessary.
if hasattr(content.data, '__iter__') and not isinstance(content.data, (bytes, (str,))):
custom_md5 = hashlib.md5()
for chunk in content.data:
fp.write(chunk)
custom_md5.update(chunk)
fp.custom_md5 = custom_md5.hexdigest()
else:
# Ideally we could just ensure that we don't get strings in here and only byte streams
# but being confident of that wolud be a lot more work than we have time for so we just
# handle both cases here.
if isinstance(content.data, str):
fp.write(content.data.encode('utf-8'))
encoded_data = content.data.encode('utf-8')
fp.write(encoded_data)
fp.custom_md5 = hashlib.md5(encoded_data).hexdigest()
else:
fp.write(content.data)
fp.custom_md5 = hashlib.md5(content.data).hexdigest()

return content

Expand Down Expand Up @@ -142,12 +181,13 @@ def find(self, location, throw_on_not_found=True, as_stream=False): # lint-amne
'thumbnail',
thumbnail_location[4]
)

return StaticContentStream(
location, fp.displayname, fp.content_type, fp, last_modified_at=fp.uploadDate,
thumbnail_location=thumbnail_location,
import_path=getattr(fp, 'import_path', None),
length=fp.length, locked=getattr(fp, 'locked', False),
content_digest=getattr(fp, 'md5', None),
content_digest=getattr(fp, 'custom_md5', None),
)
else:
with self.fs.get(content_id) as fp:
Expand All @@ -161,12 +201,13 @@ def find(self, location, throw_on_not_found=True, as_stream=False): # lint-amne
'thumbnail',
thumbnail_location[4]
)

return StaticContent(
location, fp.displayname, fp.content_type, fp.read(), last_modified_at=fp.uploadDate,
thumbnail_location=thumbnail_location,
import_path=getattr(fp, 'import_path', None),
length=fp.length, locked=getattr(fp, 'locked', False),
content_digest=getattr(fp, 'md5', None),
content_digest=getattr(fp, 'custom_md5', None),
)
except NoFile:
if throw_on_not_found: # lint-amnesty, pylint: disable=no-else-raise
Expand Down
Loading

0 comments on commit 2995608

Please sign in to comment.