added integration tests and repository pattern for image, model and visual communication DTOs

shared/repositories/tests/integration/conftest.py

@@ -0,0 +1,444 @@
+"""Integration tests configuration."""
+
+import os
+import random
+from collections.abc import Iterator
+from io import BytesIO
+from pathlib import Path
+
+import minio
+import pytest
+from PIL import Image
+from pymongo import MongoClient
+from testcontainers.minio import MinioContainer
+from testcontainers.mongodb import MongoDbContainer
+
+from model.src.models import VisualCommunicationModel
+from shared.repositories import (
+    HexadecimalString,
+    ImageData,
+    ImageRepository,
+    ModelData,
+    ModelRepository,
+    VisualCommunicationData,
+    VisualCommunicationRepository,
+    VisualCommunicationValues,
+)
+from shared.repositories.src.implementations import (
+    MinioImplementation,
+    MongoImplementation,
+)
+
+# set random seed for reproducibility
+random.seed(13)
+
+# define test static variables
+MINIO_PORT = 9000
+MINIO_ACCESS_KEY = 'minioadmin'
+MINIO_SECRET_KEY = 'minioadmin'
+MONGO_PORT = 27017
+MONGO_USERNAME = 'admin'
+MONGO_PASSWORD = 'password'
+
+env_var_map = {
+    'MINIO_ENDPOINT': f'localhost:{MINIO_PORT}',  # updated when container is running
+    'MINIO_ACCESS_KEY': MINIO_ACCESS_KEY,
+    'MINIO_SECRET_KEY': MINIO_SECRET_KEY,
+    'MINIO_BUCKET_NAME': 'test-bucket',
+    'MINIO_OBJECT_NAME': 'test-object',
+    'MINIO_IMAGE_NAME': 'test-image',
+    'MONGO_ENDPOINT': (
+        f'mongodb://{MONGO_USERNAME}:{MONGO_PASSWORD}' f'@localhost:{MONGO_PORT}/'
+    ),  # updated when container is running
+    'MONGO_DB': 'test-db',
+    'MONGO_COLLECTION': 'test-collection',
+}
+container_map = {
+    'minio': MinioContainer(
+        port=MINIO_PORT,
+        access_key=MINIO_ACCESS_KEY,
+        secret_key=MINIO_SECRET_KEY,
+    ),
+    'mongo': MongoDbContainer(
+        port=MONGO_PORT,
+        username=MONGO_USERNAME,
+        password=MONGO_PASSWORD,
+        dbname=env_var_map['MONGO_DB'],
+    ),
+}
+
+
+@pytest.fixture(scope='session', autouse=True)
+def setup_infrastructure(request: pytest.FixtureRequest) -> None:
+    """Prepare infrastructure for integration test."""
+    # prepare infrastructure
+    for container in container_map.values():
+        container.start()
+    # update env var map
+    minio_host = container_map['minio'].get_container_host_ip()
+    minio_port = container_map['minio'].get_exposed_port(MINIO_PORT)
+    env_var_map['MINIO_ENDPOINT'] = f'{minio_host}:{minio_port}'
+    mongo_host = container_map['mongo'].get_container_host_ip()
+    mongo_port = container_map['mongo'].get_exposed_port(MONGO_PORT)
+    env_var_map['MONGO_ENDPOINT'] = (
+        f'mongodb://{MONGO_USERNAME}:{MONGO_PASSWORD}@{mongo_host}:{mongo_port}/'
+    )
+
+    # ensure cleanup
+    def cleanup_infrastructure():
+        for container in container_map.values():
+            container.stop()
+
+    request.addfinalizer(cleanup_infrastructure)
+
+
+@pytest.fixture(scope='session', autouse=True)
+def populate_env(
+    request: pytest.FixtureRequest,
+    setup_infrastructure,
+) -> None:
+    """Populate environment with variables used for testing."""
+    # update env
+    for key, val in env_var_map.items():
+        os.environ[key] = val
+
+    # ensure cleanup
+    def cleanup_env():
+        for key in env_var_map:
+            _ = os.environ.pop(key, default=None)
+
+    request.addfinalizer(cleanup_env)
+
+
+@pytest.fixture(scope='session')
+def raw_minio_client(
+    setup_infrastructure,
+    populate_env,
+) -> Iterator[minio.Minio]:
+    """Raw Minio client fixture."""
+    # prepare arguments
+    minio_endpoint = str(os.getenv('MINIO_ENDPOINT'))
+    minio_access_key = str(os.getenv('MINIO_ACCESS_KEY'))
+    minio_secret_key = str(os.getenv('MINIO_SECRET_KEY'))
+    minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
+    # connect client
+    client = minio.Minio(
+        endpoint=minio_endpoint,
+        access_key=minio_access_key,
+        secret_key=minio_secret_key,
+        secure=False,
+    )
+    # ensure bucket exists
+    if not client.bucket_exists(bucket_name=minio_bucket_name):
+        client.make_bucket(bucket_name=minio_bucket_name)
+    # expose client
+    yield client
+    # cleanup
+    object_list = client.list_objects(minio_bucket_name, recursive=True)
+    for obj in object_list:
+        client.remove_object(
+            bucket_name=obj.bucket_name,
+            object_name=obj.object_name,
+        )
+    client.remove_bucket(minio_bucket_name)
+
+
+@pytest.fixture
+def minio_client(
+    setup_infrastructure,
+    populate_env,
+) -> Iterator[MinioImplementation]:
+    """MinioImplementation fixture."""
+    # instantiate and connect client
+    minio_client = MinioImplementation()
+    minio_client.connect()
+    # expose client
+    yield minio_client
+    # cleanup
+    object_name_list = minio_client._list_objects(Path('*'))
+    for name in object_name_list:
+        minio_client._delete(Path(name))
+    minio_client.close()
+
+
+@pytest.fixture
+def buffer() -> Iterator[BytesIO]:
+    """Bytes buffer fixture."""
+    # generate reproducible random data
+    data = random.randbytes(n=2**21)  # 2 MB
+    # convert data
+    buffer = BytesIO(data)
+    # expose buffer
+    yield buffer
+
+
+@pytest.fixture
+def buffer_in_minio(
+    raw_minio_client: minio.Minio,
+    buffer: BytesIO,
+) -> Iterator[tuple[Path, BytesIO]]:
+    """Buffer in Minio fixture."""
+    # prepare arguments
+    minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
+    minio_object_name = str(os.getenv('MINIO_OBJECT_NAME'))
+    # prepare for saving
+    num_bytes = len(buffer.getvalue())
+    buffer.seek(0)
+    # put data in bucket
+    raw_minio_client.put_object(
+        bucket_name=minio_bucket_name,
+        object_name=minio_object_name,
+        length=num_bytes,
+        data=buffer,
+    )
+    # expose data
+    yield Path(minio_object_name), buffer
+    # cleanup
+    raw_minio_client.remove_object(
+        bucket_name=minio_bucket_name,
+        object_name=minio_object_name,
+    )
+
+
+@pytest.fixture
+def image_data() -> Iterator[ImageData]:
+    """Image data fixture."""
+    # prepare arguments
+    name = str(os.getenv('MINIO_IMAGE_NAME'))
+    image = Image.new(mode='RGB', size=(480, 480))
+    # instantiate data
+    image_data = ImageData(image=image, name=name)
+    # expose data
+    yield image_data
+
+
+@pytest.fixture
+def image_data_in_minio(
+    raw_minio_client: minio.Minio,
+    image_data: ImageData,
+) -> Iterator[ImageData]:
+    """Image data in Minio fixture."""
+    # prepare arguments
+    minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
+    image_name = image_data.name
+    # build object path
+    object_path = ImageRepository._build_path(image_name)
+    # save image to buffer
+    buffer = BytesIO()
+    image_data.image.save(buffer, 'png')
+    # prepare for saving
+    num_bytes = len(buffer.getvalue())
+    buffer.seek(0)
+    # put data in bucket
+    raw_minio_client.put_object(
+        bucket_name=minio_bucket_name,
+        object_name=object_path.as_posix(),
+        length=num_bytes,
+        data=buffer,
+    )
+    # expose data
+    yield image_data
+    # cleanup
+    raw_minio_client.remove_object(
+        bucket_name=minio_bucket_name,
+        object_name=object_path.as_posix(),
+    )
+
+
+@pytest.fixture(scope='session')
+def image_repo(
+    setup_infrastructure,
+    populate_env,
+) -> Iterator[ImageRepository]:
+    """Image repository fixture."""
+    repo = ImageRepository()
+    repo.connect()
+    yield repo
+    repo.close()
+
+
+@pytest.fixture
+def model_data() -> Iterator[ModelData]:
+    """Model data fixture."""
+    # prepare arguments
+    vis_com_model = VisualCommunicationModel().to('cpu')
+    class_name = type(vis_com_model).__name__
+    buffer = ModelData.model_to_buffer(vis_com_model)
+    buffer_checksum = HexadecimalString('77dcab1769563654a6e24f92d40f29bd')
+    # instantiate data
+    model_data = ModelData(
+        buffer=buffer,
+        buffer_checksum=buffer_checksum,
+        class_name=class_name,
+    )
+    # expose model
+    yield model_data
+
+
+@pytest.fixture
+def model_data_in_minio(
+    raw_minio_client: minio.Minio,
+    model_data: ModelData,
+) -> Iterator[ModelData]:
+    """Model data in Minio fixture."""
+    # prepare arguments
+    minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
+    object_name = ModelRepository._build_object_name(model_data)
+    buffer = model_data.buffer
+    # build object path
+    object_path = ModelRepository._prefix() / object_name
+    # prepare for saving
+    num_bytes = len(buffer.getvalue())
+    buffer.seek(0)
+    # put data in bucket
+    raw_minio_client.put_object(
+        bucket_name=minio_bucket_name,
+        object_name=object_path.as_posix(),
+        length=num_bytes,
+        data=buffer,
+    )
+    # expose data
+    yield model_data
+    # cleanup
+    raw_minio_client.remove_object(
+        bucket_name=minio_bucket_name,
+        object_name=object_path.as_posix(),
+    )
+
+
+@pytest.fixture(scope='session')
+def model_repo(
+    setup_infrastructure,
+    populate_env,
+) -> Iterator[ModelRepository]:
+    """Model repository fixture."""
+    repo = ModelRepository()
+    repo.connect()
+    yield repo
+    repo.close()
+
+
+@pytest.fixture
+def raw_mongo_client(
+    setup_infrastructure,
+    populate_env,
+) -> Iterator[MongoClient]:
+    """Raw mongo client fixture."""
+    # prepare arguments
+    mongo_endpoint = str(os.getenv('MONGO_ENDPOINT'))
+    mongo_database = str(os.getenv('MONGO_DB'))
+    # connect client
+    client = MongoClient(mongo_endpoint)
+    _ = client[mongo_database]
+    # expose client
+    yield client
+    # cleanup
+    client.drop_database(mongo_database)
+
+
+@pytest.fixture
+def mongo_client(
+    setup_infrastructure,
+    populate_env,
+) -> Iterator[MongoImplementation]:
+    """MongoImplementation fixture."""
+    # instantiate and connect client
+    mongo_client = MongoImplementation()
+    mongo_client.connect()
+    # expose client
+    yield mongo_client
+    # cleanup
+    mongo_client._collection.drop()
+    mongo_client.close()
+
+
+@pytest.fixture
+def dictionary() -> Iterator[dict]:
+    """Dictionary fixture."""
+    # prepare data
+    data = {
+        'name': 'test-dictionary',
+        'str_key': 'value',
+        'int_key': 100,
+        'float_key': 3.14,
+        'list_key': [1, 2, 3],
+        'dict_key': {
+            'nested_key': 'nested_value',
+        },
+    }
+    # expose data
+    yield data
+
+
+@pytest.fixture
+def dictionary_in_mongo(
+    raw_mongo_client: MongoClient,
+    dictionary: dict,
+) -> Iterator[dict]:
+    """Dictionary in Mongo fixture."""
+    # prepare arguments
+    database = str(os.getenv('MONGO_DB'))
+    collection = str(os.getenv('MONGO_COLLECTION'))
+    # save data
+    _ = raw_mongo_client[database][collection].insert_one(dictionary.copy())
+    # expose data
+    yield dictionary
+    # cleanup
+    raw_mongo_client[database][collection].delete_one(dictionary)
+
+
+@pytest.fixture
+def visual_communication_values() -> Iterator[VisualCommunicationValues]:
+    """Visual communication values fixture."""
+    # instantiate with random values
+    visual_communication_values = VisualCommunicationValues.from_random()
+    # expose values
+    yield visual_communication_values
+
+
+@pytest.fixture
+def visual_communication_data(
+    visual_communication_values: VisualCommunicationValues,
+) -> Iterator[VisualCommunicationData]:
+    """Visual communication data fixture."""
+    # prepare arguments
+    name = 'test-visual-communication'
+    annotation = visual_communication_values
+    # instantiate data
+    visual_communication_data = VisualCommunicationData(
+        name=name,
+        annotation=annotation,
+    )
+    # expose data
+    yield visual_communication_data
+
+
+@pytest.fixture
+def visual_communication_data_in_mongo(
+    raw_mongo_client: MongoClient,
+    visual_communication_data: VisualCommunicationData,
+) -> Iterator[VisualCommunicationData]:
+    """Visual communication data in Mongo fixture."""
+    # prepare arguments
+    database = str(os.getenv('MONGO_DB'))
+    collection = str(os.getenv('MONGO_COLLECTION'))
+    # convert data
+    dictionary = visual_communication_data.model_dump(mode='dict')
+    # save data
+    _ = raw_mongo_client[database][collection].insert_one(dictionary.copy())
+    # expose data
+    yield visual_communication_data
+    # cleanup
+    raw_mongo_client[database][collection].delete_one(dictionary)
+
+
+@pytest.fixture(scope='session')
+def visual_communication_repo(
+    setup_infrastructure,
+    populate_env,
+) -> Iterator[VisualCommunicationRepository]:
+    """Visual communication repository fixture."""
+    repo = VisualCommunicationRepository()
+    repo.connect()
+    yield repo
+    repo.close()