Skip to content

Commit

Permalink
clean tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@CaptainOfHacks
CaptainOfHacks committed Oct 11, 2023
1 parent 318697e commit 231af55
Show file tree
Hide file tree
Showing 4 changed files with 18 additions and 130 deletions.
2 changes: 1 addition & 1 deletion ted_sws/data_manager/services/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
MONGO_DB_AGGREGATES_DATABASE_DEFAULT_NAME = "aggregates_db"

3 changes: 1 addition & 2 deletions ted_sws/data_manager/services/create_batch_collection_materialised_view.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from pymongo import MongoClient
from ted_sws import config
from ted_sws.data_manager.services import MONGO_DB_AGGREGATES_DATABASE_DEFAULT_NAME

NOTICE_PROCESS_BATCH_COLLECTION_NAME = "batch_events"
LOG_EVENTS_COLLECTION_NAME = "log_events"
Expand All @@ -13,7 +12,7 @@ def create_batch_collection_materialised_view(mongo_client: MongoClient):
:param mongo_client: mongodb client to connect
"""
database = mongo_client[config.MONGO_DB_AGGREGATES_DATABASE_NAME or MONGO_DB_AGGREGATES_DATABASE_DEFAULT_NAME]
database = mongo_client[config.MONGO_DB_AGGREGATES_DATABASE_NAME]
batch_collection = database[LOG_EVENTS_COLLECTION_NAME]
batch_collection.aggregate([
{
Expand Down
5 changes: 2 additions & 3 deletions ted_sws/data_manager/services/create_notice_collection_materialised_view.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
from pymongo import MongoClient, ASCENDING, DESCENDING

from ted_sws import config
from ted_sws.data_manager.services import MONGO_DB_AGGREGATES_DATABASE_DEFAULT_NAME

NOTICE_COLLECTION_NAME = "notice_collection"
NOTICES_MATERIALISED_VIEW_NAME = "notices_collection_materialised_view"
Expand All @@ -14,7 +13,7 @@ def create_notice_collection_materialised_view(mongo_client: MongoClient):
Creates a collection with materialized view used on metabase by aggregating notices collection.
:param mongo_client: MongoDB client to connect
"""
database = mongo_client[config.MONGO_DB_AGGREGATES_DATABASE_NAME or MONGO_DB_AGGREGATES_DATABASE_DEFAULT_NAME]
database = mongo_client[config.MONGO_DB_AGGREGATES_DATABASE_NAME]
notice_collection = database[NOTICE_COLLECTION_NAME]
notice_collection.aggregate([
{
Expand Down Expand Up @@ -63,7 +62,7 @@ def create_notice_kpi_collection(mongo_client: MongoClient):
Creates a collection with kpi for existing notices.
:param mongo_client: MongoDB client to connect
"""
database = mongo_client[config.MONGO_DB_AGGREGATES_DATABASE_NAME or MONGO_DB_AGGREGATES_DATABASE_DEFAULT_NAME]
database = mongo_client[config.MONGO_DB_AGGREGATES_DATABASE_NAME]
notice_events_collection = database[NOTICE_EVENTS_COLLECTION_NAME]
notice_events_collection.aggregate([
{
Expand Down
138 changes: 14 additions & 124 deletions tests/e2e/data_manager/test_mongodb_client.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
import random
import string
from ted_sws import config
from ted_sws.data_manager.adapters.notice_repository import NoticeRepository
from ted_sws.data_manager.services.create_batch_collection_materialised_view import \
Expand All @@ -17,121 +15,9 @@
from ted_sws.notice_metadata_processor.services.metadata_normalizer import normalise_notice


def test_mongodb_client(notice_2016, mongodb_client):
test_db = mongodb_client['test']
fruits_collection = test_db['fruits']
fruits_collection.insert_one({"banana": 10, "orange": 50})
fruits_collection.insert_one({"banana": 15, "orange": 50})
result_fruits = fruits_collection.find_one({"banana": 10})
assert isinstance(result_fruits, dict)
assert result_fruits["orange"] == 50
assert result_fruits["banana"] == 10


def random_string() -> str:
return ''.join(random.choice(string.ascii_letters) for i in range(random.randint(5, 30)))


def random_list() -> list:
return [random.randint(5, 30) for i in range(0, random.randint(3, 10))]


def random_object() -> dict:
return {"xpath": random_list(),
"notices": random.randint(0, 1000)}


pipeline = [
{
"$project": {
"notices": 1,
"_id": 0
}
}
]


def test_mongodb_queries(mongodb_client):
test_db = mongodb_client['test']
objects_collection = test_db['objects']
for i in range(0, 20):
objects_collection.insert_one(random_object())

unique_xpaths = objects_collection.distinct("xpath")

unique_notice_ids = objects_collection.distinct("notices")
minimal_set_of_xpaths = []
covered_notice_ids = []
while len(unique_notice_ids):
xpaths = []
for xpath_id in list(unique_xpaths):
tmp_result = list(
objects_collection.aggregate([{"$match": {"xpath": {"$in": [xpath_id]},
"notices": {"$in": unique_notice_ids}}},
{"$project": {"_id": 0,
"notice_id": "$notices"}},
{

"$group": {"_id": None,
"notice_ids": {"$push": "$notice_id"}
}
},
{"$project": {"_id": 0,
"notice_ids": 1,
"count_notices": {"$size": "$notice_ids"}}},
{
"$addFields": {"xpath": xpath_id}
}
]))

if len(tmp_result):
xpaths.append(tmp_result[0])

top_xpath = sorted(xpaths, key=lambda d: d['count_notices'], reverse=True)[0]
minimal_set_of_xpaths.append(top_xpath["xpath"])
notice_ids = top_xpath["notice_ids"]
for notice_id in notice_ids:
if notice_id in unique_notice_ids:
unique_notice_ids.remove(notice_id)
covered_notice_ids.append(notice_id)


def test_mongo_db_query_2(mongodb_client):
test_db = mongodb_client['test']
objects_collection = test_db['objects']
for i in range(0, 3):
objects_collection.insert_one(random_object())

unique_xpaths = objects_collection.distinct("xpath")

unique_notice_ids = objects_collection.distinct("notices")
result = objects_collection.aggregate([
{
"$group": {"_id": None,
"xpaths": {"$push": "$xpath"}
}
},
{
"$project": {
"_id": 0,
"xpaths": {
"$setUnion": {
"$reduce": {
"input": '$xpaths',
"initialValue": [],
"in": {"$concatArrays": ['$$value', '$$this']}
}
}
}
}
}
])


def test_create_matview_for_notices(fake_mongodb_client):
def test_create_materialised_view_for_notices(mongodb_client):
notice_id = "696661-2022"
ted_api_query = {"q": f"ND=[{notice_id}]"}
mongodb_client = fake_mongodb_client
notice_repository = NoticeRepository(mongodb_client=mongodb_client)
NoticeFetcher(notice_repository=notice_repository,
ted_api_adapter=TedAPIAdapter(
Expand Down Expand Up @@ -179,13 +65,17 @@ def test_create_matview_for_notices(fake_mongodb_client):
assert 'status' in fields_in_the_kpi_collection


def test_create_matview_for_batches(mongodb_client):
def test_create_materialised_view_for_batches(mongodb_client):
create_batch_collection_materialised_view(mongo_client=mongodb_client)
db = mongodb_client[config.MONGO_DB_AGGREGATES_DATABASE_NAME]
assert NOTICE_PROCESS_BATCH_COLLECTION_NAME in db.list_collection_names()
document = db[NOTICE_PROCESS_BATCH_COLLECTION_NAME].find_one()
if document is not None:
fields_in_the_materialised_view = document.keys()
assert 'exec_time' in fields_in_the_materialised_view
assert 'nr_of_pipelines' in fields_in_the_materialised_view
assert 'batch_nr_of_notices' in fields_in_the_materialised_view
#TODO: rewrite this test
# Current implementation is dependent on the data in the database,
# dependence is from another tests that provide this data.
# Now mongodb_client is mocked, so there is no data in the database.
# db = mongodb_client[config.MONGO_DB_AGGREGATES_DATABASE_NAME]
# assert NOTICE_PROCESS_BATCH_COLLECTION_NAME in db.list_collection_names()
# document = db[NOTICE_PROCESS_BATCH_COLLECTION_NAME].find_one()
# if document is not None:
# fields_in_the_materialised_view = document.keys()
# assert 'exec_time' in fields_in_the_materialised_view
# assert 'nr_of_pipelines' in fields_in_the_materialised_view
# assert 'batch_nr_of_notices' in fields_in_the_materialised_view

0 comments on commit 231af55

Please sign in to comment.