From 8f43a0333e7cf64dc2569c2c06e4c72b6b5c5aef Mon Sep 17 00:00:00 2001 From: Daniil Lytvyn Date: Tue, 24 Dec 2024 15:46:58 +0200 Subject: [PATCH 1/4] Update to pymongo 4.2 --- common/djangoapps/track/backends/mongodb.py | 12 +++---- .../perf_tests/test_asset_import_export.py | 35 ++++++++----------- .../split_mongo/mongo_connection.py | 9 ++--- common/lib/xmodule/xmodule/mongo_utils.py | 20 +++-------- requirements/edx/base.in | 2 +- requirements/edx/base.txt | 7 ++-- requirements/edx/development.txt | 4 +-- requirements/edx/github.in | 3 +- requirements/edx/paver.in | 2 +- requirements/edx/testing.txt | 6 ++-- 10 files changed, 43 insertions(+), 57 deletions(-) diff --git a/common/djangoapps/track/backends/mongodb.py b/common/djangoapps/track/backends/mongodb.py index 07687eae9899..2ef340087332 100644 --- a/common/djangoapps/track/backends/mongodb.py +++ b/common/djangoapps/track/backends/mongodb.py @@ -63,14 +63,14 @@ def __init__(self, **kwargs): self.connection = MongoClient( host=host, port=port, + username=user, + password=password, + authSource=auth_source, **extra ) database = self.connection[db_name] - if user or password: - database.authenticate(user, password, source=auth_source) - self.collection = database[collection_name] self._create_indexes() @@ -84,13 +84,13 @@ def _create_indexes(self): # TODO: The creation of indexes can be moved to a Django # management command or equivalent. There is also an option to # run the indexing on the background, without locking. - self.collection.ensure_index([('time', pymongo.DESCENDING)], background=True) - self.collection.ensure_index('event_type', background=True) + self.collection.create_index([('time', pymongo.DESCENDING)], background=True) + self.collection.create_index('event_type', background=True) def send(self, event): """Insert the event in to the Mongo collection""" try: - self.collection.insert(event, manipulate=False) + self.collection.insert_one(event) except (PyMongoError, BSONError): # The event will be lost in case of a connection error or any error # that occurs when trying to insert the event into Mongo. diff --git a/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py b/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py index f177ac01c817..22d36fed3f35 100644 --- a/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py +++ b/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py @@ -11,7 +11,6 @@ import ddt import pytest -from bson.code import Code from path import Path as path from xmodule.assetstore import AssetMetadata @@ -172,29 +171,25 @@ def test_asset_sizes(self, source_ms, num_assets): asset_collection = source_ms.asset_collection() # Ensure the asset collection exists. - if asset_collection.name in asset_collection.database.collection_names(): - - # Map gets the size of each structure. - mapper = Code(""" - function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) } - """) - - # Reduce finds the largest structure size and returns only it. - reducer = Code(""" - function(key, values) { - var max_size = 0; - for (var i=0; i < values.length; i++) { - if (values[i] > max_size) { - max_size = values[i]; - } + if asset_collection.name in asset_collection.database.list_collection_names(): + # Get the size of each structure, and finds the largest structure size + pipeline = [ + { + "$project": { + "size": {"$bsonSize": "$$ROOT"} + } + }, + { + "$group": { + "_id": None, + "max_size": {"$max": "$size"} } - return max_size; } - """) + ] - results = asset_collection.map_reduce(mapper, reducer, "size_results") + results = list(asset_collection.aggregate(pipeline)) result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format( - self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()] + self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, results ) with open("bson_sizes.txt", "a") as f: f.write(result_str) diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py index 413880e6ea87..ed42c3251a08 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py @@ -538,7 +538,8 @@ def delete_course_index(self, course_key): key_attr: getattr(course_key, key_attr) for key_attr in ('org', 'course', 'run') } - return self.course_index.remove(query) + result = self.course_index.delete_many(query) + return result.deleted_count def get_definition(self, key, course_context=None): """ @@ -614,9 +615,9 @@ def _drop_database(self, database=True, collections=True, connections=True): self.structures.drop() self.definitions.drop() else: - self.course_index.remove({}) - self.structures.remove({}) - self.definitions.remove({}) + self.course_index.delete_many({}) + self.structures.delete_many({}) + self.definitions.delete_many({}) if connections: connection.close() diff --git a/common/lib/xmodule/xmodule/mongo_utils.py b/common/lib/xmodule/xmodule/mongo_utils.py index 76f476bd43bf..35a8108e3ebd 100644 --- a/common/lib/xmodule/xmodule/mongo_utils.py +++ b/common/lib/xmodule/xmodule/mongo_utils.py @@ -32,18 +32,6 @@ def connect_to_mongodb( handles AutoReconnect errors by retrying read operations, since these exceptions typically indicate a temporary step-down condition for MongoDB. """ - # The MongoReplicaSetClient class is deprecated in Mongo 3.x, in favor of using - # the MongoClient class for all connections. Update/simplify this code when using - # PyMongo 3.x. - if kwargs.get('replicaSet'): - # Enable reading from secondary nodes in the MongoDB replicaset by using the - # MongoReplicaSetClient class. - # The 'replicaSet' parameter in kwargs is required for secondary reads. - # The read_preference should be set to a proper value, like SECONDARY_PREFERRED. - mongo_client_class = pymongo.MongoReplicaSetClient - else: - # No 'replicaSet' in kwargs - so no secondary reads. - mongo_client_class = pymongo.MongoClient # If the MongoDB server uses a separate authentication database that should be specified here auth_source = kwargs.get('authsource', '') or None @@ -67,11 +55,14 @@ def connect_to_mongodb( kwargs['read_preference'] = read_preference mongo_conn = pymongo.database.Database( - mongo_client_class( + pymongo.MongoClient( host=host, port=port, tz_aware=tz_aware, document_class=dict, + username=user, + password=password, + authSource=auth_source, **kwargs ), db @@ -82,9 +73,6 @@ def connect_to_mongodb( mongo_conn, wait_time=retry_wait_time ) - # If credentials were provided, authenticate the user. - if user is not None and password is not None: - mongo_conn.authenticate(user, password, source=auth_source) return mongo_conn diff --git a/requirements/edx/base.in b/requirements/edx/base.in index 9cc3da794c2d..f34196016581 100644 --- a/requirements/edx/base.in +++ b/requirements/edx/base.in @@ -65,6 +65,7 @@ django-user-tasks django-waffle==0.18.0 django-webpack-loader # Used to wire webpack bundles into the django asset pipeline djangorestframework==3.9.4 +dnspython edx-ace edx-analytics-data-api-client edx-api-doc-tools @@ -88,7 +89,6 @@ edx-submissions edx-user-state-client edx-when edxval -event-tracking fs==2.0.18 fs-s3fs==0.1.8 geoip2 # Python API for the GeoIP web services and databases diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 775e61c4ed32..288b3a061bf3 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -90,6 +90,7 @@ djangorestframework-xml==2.0.0 # via edx-enterprise djangorestframework==3.9.4 # via -r requirements/edx/base.in, django-config-models, django-user-tasks, drf-jwt, drf-yasg, edx-api-doc-tools, edx-completion, edx-drf-extensions, edx-enterprise, edx-organizations, edx-proctoring, edx-submissions, ora2, rest-condition, super-csv docopt==0.6.2 # via xmodule docutils==0.16 # via botocore +dnspython==1.16.0 # via -r requirements/edx/base.in drf-jwt==1.14.0 # via -c requirements/edx/../constraints.txt, edx-drf-extensions drf-yasg==1.17.0 # via -c requirements/edx/../constraints.txt, edx-api-doc-tools edx-ace==0.1.15 # via -r requirements/edx/base.in @@ -121,7 +122,7 @@ edx-when==1.2.3 # via -r requirements/edx/base.in, edx-proctoring edxval==1.3.4 # via -r requirements/edx/base.in elasticsearch==1.9.0 # via edx-search enum34==1.1.10 # via edxval -event-tracking==0.3.2 # via -r requirements/edx/base.in, edx-proctoring, edx-search +git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2 # via -r requirements/edx/github.in fs-s3fs==0.1.8 # via -r requirements/edx/base.in, django-pyfs fs==2.0.18 # via -r requirements/edx/base.in, django-pyfs, fs-s3fs, xblock future==0.18.2 # via django-ses, edx-celeryutils, edx-enterprise, pycontracts, pyjwkest @@ -157,7 +158,7 @@ markey==0.8 # via django-babel-underscore markupsafe==1.1.1 # via -r requirements/edx/paver.txt, chem, jinja2, mako, xblock maxminddb==1.5.4 # via geoip2 mock==3.0.5 # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.txt, xblock-drag-and-drop-v2, xblock-poll -git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/github.in +git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/github.in mongoengine==0.10.0 # via -r requirements/edx/base.in more-itertools==8.3.0 # via -r requirements/edx/paver.txt, zipp mpmath==1.1.0 # via sympy @@ -190,7 +191,7 @@ pycryptodomex==3.9.7 # via -r requirements/edx/base.in, edx-proctoring, pyj pygments==2.6.1 # via -r requirements/edx/base.in pyjwkest==1.4.2 # via -r requirements/edx/base.in, edx-drf-extensions pyjwt==1.5.2 # via -r requirements/edx/base.in, drf-jwt, edx-rest-api-client, social-auth-core -pymongo==3.9.0 # via -r requirements/edx/base.in, -r requirements/edx/paver.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine +pymongo==4.2.0 # via -r requirements/edx/base.in, -r requirements/edx/paver.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine pynliner==0.8.0 # via -r requirements/edx/base.in pyparsing==2.4.7 # via chem, openedx-calc, packaging, pycontracts pysrt==1.1.2 # via -r requirements/edx/base.in, edxval diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 616984a8598e..dc9c71f384d1 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -135,7 +135,7 @@ edx-when==1.2.3 # via -r requirements/edx/testing.txt, edx-proctoring edxval==1.3.4 # via -r requirements/edx/testing.txt elasticsearch==1.9.0 # via -r requirements/edx/testing.txt, edx-search enum34==1.1.10 # via -r requirements/edx/testing.txt, edxval -event-tracking==0.3.2 # via -r requirements/edx/testing.txt, edx-proctoring, edx-search +git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2 # via -r requirements/edx/testing.txt, edx-proctoring, edx-search execnet==1.7.1 # via -r requirements/edx/testing.txt, pytest-xdist factory-boy==2.8.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt faker==4.1.0 # via -r requirements/edx/testing.txt, factory-boy @@ -190,7 +190,7 @@ maxminddb==1.5.4 # via -r requirements/edx/testing.txt, geoip2 mccabe==0.6.1 # via -r requirements/edx/testing.txt, flake8, pylint mistune==0.8.4 # via m2r mock==3.0.5 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, xblock-drag-and-drop-v2, xblock-poll -git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/testing.txt +git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/testing.txt mongoengine==0.10.0 # via -r requirements/edx/testing.txt more-itertools==8.3.0 # via -r requirements/edx/testing.txt, pytest, zipp mpmath==1.1.0 # via -r requirements/edx/testing.txt, sympy diff --git a/requirements/edx/github.in b/requirements/edx/github.in index 9a8b2966e725..408892c416ed 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -56,7 +56,8 @@ git+https://github.com/edx/openedx-chem.git@ff4e3a03d3c7610e47a9af08eb648d8aabe2 # Third-party: -e git+https://github.com/edx/django-wiki.git@0.0.27#egg=django-wiki -git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 +git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2 +git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2 -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev -e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme diff --git a/requirements/edx/paver.in b/requirements/edx/paver.in index 1a4266a2bee7..c25f0317da60 100644 --- a/requirements/edx/paver.in +++ b/requirements/edx/paver.in @@ -18,7 +18,7 @@ mock # Stub out code with mock objects and make a path # Easier manipulation of filesystem paths paver # Build, distribution and deployment scripting tool psutil==1.2.1 # Library for retrieving information on running processes and system utilization -pymongo==3.9.0 # via edx-opaque-keys +pymongo==4.2.0 # via edx-opaque-keys python-memcached # Python interface to the memcached memory cache daemon requests # Simple interface for making HTTP requests stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index ef360465d765..abc896a67cdb 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -131,7 +131,7 @@ edx-when==1.2.3 # via -r requirements/edx/base.txt, edx-proctoring edxval==1.3.4 # via -r requirements/edx/base.txt elasticsearch==1.9.0 # via -r requirements/edx/base.txt, edx-search enum34==1.1.10 # via -r requirements/edx/base.txt, edxval -event-tracking==0.3.2 # via -r requirements/edx/base.txt, edx-proctoring, edx-search +git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2 # via -r requirements/edx/base.txt, edx-proctoring, edx-search execnet==1.7.1 # via pytest-xdist factory-boy==2.8.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.in faker==4.1.0 # via factory-boy @@ -182,7 +182,7 @@ markupsafe==1.1.1 # via -r requirements/edx/base.txt, -r requirements/ed maxminddb==1.5.4 # via -r requirements/edx/base.txt, geoip2 mccabe==0.6.1 # via flake8, pylint mock==3.0.5 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, xblock-drag-and-drop-v2, xblock-poll -git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/base.txt +git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/base.txt mongoengine==0.10.0 # via -r requirements/edx/base.txt more-itertools==8.3.0 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, pytest, zipp mpmath==1.1.0 # via -r requirements/edx/base.txt, sympy @@ -227,7 +227,7 @@ pylint-celery==0.3 # via edx-lint pylint-django==2.0.11 # via edx-lint pylint-plugin-utils==0.6 # via pylint-celery, pylint-django pylint==2.4.2 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils -pymongo==3.9.0 # via -r requirements/edx/base.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine +pymongo==4.2.0 # via -r requirements/edx/base.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine pynliner==0.8.0 # via -r requirements/edx/base.txt pyparsing==2.4.7 # via -r requirements/edx/base.txt, chem, openedx-calc, packaging, pycontracts pyquery==1.4.1 # via -r requirements/edx/testing.in From 4b2ad35038cea056036d9be7815622197fa72835 Mon Sep 17 00:00:00 2001 From: Daniil Lytvyn Date: Tue, 24 Dec 2024 16:38:28 +0200 Subject: [PATCH 2/4] Fix code styling --- common/djangoapps/track/backends/mongodb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/djangoapps/track/backends/mongodb.py b/common/djangoapps/track/backends/mongodb.py index 2ef340087332..1969f2f344ee 100644 --- a/common/djangoapps/track/backends/mongodb.py +++ b/common/djangoapps/track/backends/mongodb.py @@ -65,7 +65,7 @@ def __init__(self, **kwargs): port=port, username=user, password=password, - authSource=auth_source, + authSource=auth_source, **extra ) From 7668db8881c84ef6944a73da8536f7141ca59e6a Mon Sep 17 00:00:00 2001 From: Daniil Lytvyn Date: Tue, 24 Dec 2024 16:39:08 +0200 Subject: [PATCH 3/4] Update python version for tests in GH actions --- .github/workflows/tests.yml | 3 +-- tox.ini | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b96fe22dde49..fd94a505c0e8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,7 +8,6 @@ on: jobs: test: - runs-on: ubuntu-20.04 services: mongodb: @@ -17,7 +16,7 @@ jobs: - 27017:27017 strategy: matrix: - python-version: [3.5] + python-version: [3.7] tox-env: - pep8 - common diff --git a/tox.ini b/tox.ini index b76bb0d699b5..10724fae2538 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ skipsdist=True toxworkdir={env:TOX_WORKDIR:{homedir}/edxapp_toxenv} [testenv] -basepython=python3.5 +basepython=python3.7 # TODO: Remove tox-pip-version once we upgrade to Koa+, or whenever we have addressed pip 20.3 strict issues. pip_version=pip==20.2.4 # This ensures "-e ." is installed, so that a link back to the top-level From 62159c1935988268e6efdfc7d0f470dc949a6611 Mon Sep 17 00:00:00 2001 From: Daniil Lytvyn Date: Mon, 30 Dec 2024 17:47:55 +0200 Subject: [PATCH 4/4] Fix python 3.7 compatability issues https://docs.python.org/3/whatsnew/3.7.html https://github.com/python/cpython/pull/1646/files#diff-daea55f2fca784d9a64ca8f190403904 --- common/djangoapps/pipeline_mako/templates/mako/js.html | 9 +-------- common/lib/xmodule/xmodule/modulestore/__init__.py | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/common/djangoapps/pipeline_mako/templates/mako/js.html b/common/djangoapps/pipeline_mako/templates/mako/js.html index ffc9355675db..d9c180329730 100644 --- a/common/djangoapps/pipeline_mako/templates/mako/js.html +++ b/common/djangoapps/pipeline_mako/templates/mako/js.html @@ -1,8 +1 @@ - + diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 00a123e1d108..74b52b11961d 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -899,7 +899,7 @@ def _value_matches(self, target, criteria): """ if isinstance(target, list): return any(self._value_matches(ele, criteria) for ele in target) - elif isinstance(criteria, re._pattern_type): # pylint: disable=protected-access + elif isinstance(criteria, re.Pattern): return criteria.search(target) is not None elif callable(criteria): return criteria(target)