updated to use new vis com class

other/transfer_images_mongo_minio.py

@@ -0,0 +1,122 @@
+"""Script to move all images from MongoDB to MinIO."""
+from __future__ import annotations
+
+import logging
+from io import BytesIO
+from pathlib import Path
+
+from bson import ObjectId
+from dotenv import load_dotenv
+from pymongo.collection import Collection
+
+from shared.data_store import connect as connect_minio
+from shared.data_store import put
+from shared.database import connect
+from shared.database import VisualCommunication
+from shared.utils import check_env
+from shared.utils import setup_logging
+
+
+def list_mongo_document_ids(
+    collection: Collection,
+) -> list[ObjectId]:
+    """Get list of all VisualCommunication documents in MongoDB."""
+    # prepare query
+    query = collection.find(
+        filter={},
+        projection={'_id': True},
+    )
+    # execute query
+    res_list = list(query)
+    logging.debug('got %s document(s)', len(res_list))
+    # convert result
+    id_list = [elem['_id'] for elem in res_list]
+    return id_list
+
+
+def get_visual_communication(
+    collection: Collection,
+    doc_id: ObjectId,
+) -> VisualCommunication:
+    """Get image from MongoDB."""
+    # prepare query
+    query = collection.find(
+        filter={'_id': doc_id},
+        projection={'_id': False},
+    )
+    # execute query
+    res_list = list(query)
+    logging.debug('got %s document(s)', len(res_list))
+    # convert result
+    vis_com = VisualCommunication.model_validate(res_list[0])
+    return vis_com
+
+
+def update_visual_communication(
+    collection: Collection,
+    vis_com: VisualCommunication,
+) -> None:
+    """Update VisualCommunication document in MongoDB."""
+    try:
+        collection.update_one(
+            filter={
+                'name': vis_com.name,
+            },
+            update={
+                '$set': vis_com.model_dump(),
+            },
+        )
+    except Exception as exc:
+        logging.error('failed saving %s', vis_com.name)
+        logging.debug(exc)
+        raise exc
+
+
+if __name__ == '__main__':
+    # load in env file
+    env_path = Path(__file__).parent.parent / 'server.env'
+    assert env_path.exists()
+    load_dotenv(env_path)
+    # ensure env vars set
+    check_env()
+    # setup logging
+    setup_logging()
+    # connect to minIO
+    minio_client = connect_minio()
+    # connect to MongoDB
+    collection, db, client = connect()
+    # list documents in mongoDB
+    id_list = list_mongo_document_ids(collection)
+    for doc_id in id_list:
+        try:
+            # get image from MongoDB
+            vis_com = get_visual_communication(collection, doc_id)
+        except Exception as exc:
+            logging.debug(exc)
+            logging.error('failed getting image from document: %s', doc_id)
+            continue
+        try:
+            # save image to buffer
+            buffer = BytesIO()
+            vis_com.image.save(buffer, 'png')
+            # put buffer in minio
+            object_name = put(
+                client=minio_client,
+                buffer=buffer,
+            )
+        except Exception as exc:
+            logging.debug(exc)
+            logging.error('failed saving image to minio')
+            continue
+        # update visual communication in mongodb
+        try:
+            vis_com.image = None    # type: ignore
+            vis_com.object_name = object_name
+            update_visual_communication(collection, vis_com)
+        except Exception as exc:
+            logging.debug(exc)
+            logging.error(
+                'failed updating visual communication %s', vis_com.name,
+            )
+            continue
+        logging.debug('updated visual communication %s', vis_com.name)

shared/data_store/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from .connect import connect
+from .get import get
+from .put import put

shared/data_store/connect.py

@@ -0,0 +1,30 @@
+"""Definition of connect function."""
+from __future__ import annotations
+
+import os
+
+from minio import Minio
+
+
+def connect(
+) -> Minio:
+    """Connect to MinIO server"""
+    minio_endpoint = os.getenv('MINIO_ENDPOINT', default=None)
+    assert isinstance(minio_endpoint, str)
+    minio_access_key = os.getenv('MINIO_ACCESS_KEY', default=None)
+    assert isinstance(minio_access_key, str)
+    minio_secret_key = os.getenv('MINIO_SECRET_KEY', default=None)
+    assert isinstance(minio_secret_key, str)
+    minio_bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None)
+    assert isinstance(minio_bucket_name, str)
+    # connect client
+    client = Minio(
+        endpoint=minio_endpoint,
+        access_key=minio_access_key,
+        secret_key=minio_secret_key,
+        secure=False,
+    )
+    # ensure bucket exists
+    if not client.bucket_exists(bucket_name=minio_bucket_name):
+        client.make_bucket(bucket_name=minio_bucket_name)
+    return client

shared/data_store/get.py

@@ -0,0 +1,27 @@
+"""Definition of get function."""
+from __future__ import annotations
+
+import os
+from io import BytesIO
+
+from minio import Minio
+
+
+def get(
+    client: Minio,
+    object_name: str,
+) -> BytesIO:
+    """Get buffer from bucket in MinIO."""
+    assert isinstance(client, Minio)
+    assert isinstance(object_name, str)
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None)
+    assert isinstance(bucket_name, str)
+    # get buffer
+    try:
+        response = client.get_object(bucket_name, object_name)
+        buffer = BytesIO(response.data)
+    finally:
+        response.close()
+        response.release_conn()
+    buffer.seek(0)
+    return buffer

shared/data_store/put.py

@@ -0,0 +1,38 @@
+"""Definition of put function."""
+from __future__ import annotations
+
+import logging
+import os
+from hashlib import md5
+from io import BytesIO
+
+from minio import Minio
+
+
+def put(
+    client: Minio,
+    buffer: BytesIO,
+) -> str:
+    """Put buffer in bucket in MinIO and return MD5 checksum as object name."""
+    assert isinstance(client, Minio)
+    assert isinstance(buffer, BytesIO)
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None)
+    assert isinstance(bucket_name, str)
+    # get md5 of image
+    checksum = md5(buffer.getbuffer()).hexdigest()
+    # prepare for saving
+    num_bytes = buffer.tell()
+    buffer.seek(0)
+    # send data to bucket
+    try:
+        client.put_object(
+            bucket_name=bucket_name,
+            object_name=checksum,
+            length=num_bytes,
+            data=buffer,
+        )
+    except Exception as exc:
+        logging.error('failed saving data to MinIO')
+        raise exc
+    logging.debug('saved data to %s', checksum)
+    return checksum

shared/database/classes/visual_communication.py

@@ -7,11 +7,12 @@ from base64 import b64encode
 from io import BytesIO
 from pathlib import Path
 
+from minio import Minio
 from PIL import Image
 from pydantic import BaseModel
-from pydantic import field_serializer
-from pydantic import field_validator
 
+from shared.data_store import get
+from shared.data_store import put
 from shared.dto import ModelData
 
 
@@ -19,7 +20,7 @@ class VisualCommunication(BaseModel):
     """Visual communication model."""
 
     name: str
-    image: Image.Image
+    object_name: str
     annotation: ModelData | None = None
     prediction: ModelData | None = None
 
@@ -33,12 +34,22 @@ class VisualCommunication(BaseModel):
         return cls.__name__
 
     @classmethod
-    def from_file(cls, path: Path) -> VisualCommunication:
+    def from_file(cls, path: Path, minio_client: Minio) -> VisualCommunication:
         """Instantiate from file."""
+        assert isinstance(path, Path)
+        assert isinstance(minio_client, Minio)
+        # determine name
         name = path.stem
+        # open and upload image to minio
         image = Image.open(path)
         image.load()
-        return VisualCommunication(name=name, image=image)
+        buffer = BytesIO()
+        image.save(buffer, 'png')
+        object_name = put(
+            client=minio_client,
+            buffer=buffer,
+        )
+        return VisualCommunication(name=name, object_name=object_name)
 
     @classmethod
     def decode_image(cls, content: str) -> Image.Image:
@@ -46,35 +57,26 @@ class VisualCommunication(BaseModel):
         _, content_data = content.split(',')
         return Image.open(BytesIO(b64decode(content_data)))
 
-    @field_serializer('image')
+    def get_image(self, minio_client: Minio) -> Image.Image:
-    @classmethod
+        """Load image data from minio."""
-    def serialize_image(cls, image: Image.Image) -> bytes:   # type: ignore
+        assert isinstance(minio_client, Minio)
-        """Convert image to bytes for storage in database."""
+        # get buffer from minio
-        buffer = BytesIO()
+        buffer = get(
-        image.save(buffer, format='JPEG')
+            client=minio_client,
-        return buffer.getvalue()
+            object_name=self.object_name,
+        )
+        # convert data to image
+        im = Image.open(buffer)
+        return im
 
-    @field_validator('image', mode='before')
+    def webencoded_image(self, minio_client: Minio) -> str:
-    @classmethod
-    def convert_to_image(
-        cls,
-        image: Image.Image | BytesIO | bytes,
-    ) -> Image.Image:
-        """Convert bytes input from database into image."""
-        if isinstance(image, bytes):
-            image = BytesIO(image)
-        if isinstance(image, BytesIO):
-            image = Image.open(image)
-        return image
-
-    def __repr__(self) -> str:
-        return f"{self.classname()}(name='{self.name}')"
-
-    def webencoded_image(self) -> str:
         """Convert image to be displayed on webpage."""
+        assert isinstance(minio_client, Minio)
+        # get image from minio
+        image = self.get_image(minio_client)
         # convert images to bytes string
         buffer = BytesIO()
-        self.image.save(buffer, format='png')
+        image.save(buffer, format='png')
         img_enc = b64encode(buffer.getvalue()).decode('utf-8')
         return f"data:image/png;base64, {img_enc}"
 

shared/utils/setup_logging.py

@@ -12,10 +12,14 @@ def setup_logging() -> None:
     fmt = (
         '%(asctime)s | '
         '%(levelname)s | '
-        '%(filename)s | '
+        '%(name)s | '
         '%(funcName)s | '
         '%(message)s'
     )
     datefmt = '%Y-%m-%d %H:%M:%S'
     logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
+    # change levels for modules that spam the log
+    logging.getLogger('pymongo').setLevel(logging.WARNING)
+    logging.getLogger('urllib3').setLevel(logging.INFO)
+    logging.getLogger('PIL').setLevel(logging.INFO)
     logging.debug('finished')

tests/get_visual_communication_test.py

@@ -1,22 +1,32 @@
 from __future__ import annotations
 
-import os
 from pathlib import Path
 
 from dotenv import load_dotenv
 
+from shared.data_store import (
+    connect as connect_minio,
+)
 from shared.database import connect
 from shared.database import get_visual_communication
+from shared.utils import check_env
+from shared.utils import setup_logging
 
 if __name__ == '__main__':
-    # prepare env vars
+    # load in env file
-    env_path = Path(__file__).parent.parent / 'local.env'
+    env_path = Path(__file__).parent.parent / 'server.env'
     assert env_path.exists()
     load_dotenv(env_path)
-    os.environ['MONGO_HOST'] = 'localhost'
+    # ensure env vars set
-    # connect to database
+    check_env()
+    # setup logging
+    setup_logging()
+    # connect to minIO
+    minio_client = connect_minio()
+    # connect to MongoDB
     collection, db, client = connect()
-    print(client.server_info())
     # get visual communication
     vis_com = get_visual_communication(collection)
-    print(vis_com)
+    print(repr(vis_com))
+    image = vis_com.get_image(minio_client=minio_client)
+    image.show()

web_ui/src/app/init_app.py

@@ -11,6 +11,7 @@ from dash import Input
 from dash import Output
 from dash import State
 from dash_auth import BasicAuth
+from minio import Minio
 from pydantic import ValidationError
 from pymongo.collection import Collection
 
@@ -25,7 +26,8 @@ from shared.dto import ModelData
 
 
 def init_app(
-    collection: Collection,
+    mongo_collection: Collection,
+    minio_client: Minio,
 ) -> Dash:
     """Initialise web UI application."""
     # setup app
@@ -92,11 +94,11 @@ def init_app(
             assert isinstance(filename_list, list)
         # get values from database
         num_total = count_documents(
-            collection=collection,
+            collection=mongo_collection,
             only_with_annotation=False,
         )
         num_handled = count_documents(
-            collection=collection,
+            collection=mongo_collection,
             only_with_annotation=True,
         )
         limit = int(num_total/20)
@@ -138,7 +140,7 @@ def init_app(
             vis_com_list.append(vis_com)
         # upsert documents
         success = upsert_visual_communication(
-            collection=collection,
+            collection=mongo_collection,
             visual_communication_list=vis_com_list,
         )
         if success:
@@ -217,7 +219,7 @@ def init_app(
                 annotations = ModelData.from_annotations(**annotation_map)
                 # save data to database
                 upsert_annotation(
-                    collection=collection,
+                    collection=mongo_collection,
                     vis_com_name=vis_com_name,
                     annotations=annotations,
                 )
@@ -231,12 +233,12 @@ def init_app(
         try:
             # get data
             vis_com = get_visual_communication(
-                collection=collection,
+                collection=mongo_collection,
                 with_annotation=False,
             )
             # set variables
             vis_com_name = vis_com.name
-            image_src = vis_com.webencoded_image()
+            image_src = vis_com.webencoded_image(minio_client=minio_client)
             if vis_com.prediction is not None:
                 # TODO: update to use optional predictions
                 pass

web_ui/src/main.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 import os
 
 from .app import init_app
+from shared.data_store import connect as connect_minio
 from shared.database import connect
 from shared.utils import check_env
 from shared.utils import setup_logging
@@ -17,7 +18,13 @@ setup_logging()
 # connect to database
 collection, db, client = connect()
 
+# connect to minio
+minio_client = connect_minio()
+
 # initialise application
-app = init_app(collection=collection)
+app = init_app(
+    mongo_collection=collection,
+    minio_client=minio_client,
+)
 server = app.server
 server.config.update(SECRET_KEY=os.urandom(24))