updated to use new class

other/transfer_images_mongo_minio.py

@@ -98,7 +98,7 @@ if __name__ == '__main__':
         try:
             # save image to buffer
             buffer = BytesIO()
-            vis_com.image.save(buffer, 'png')
+            vis_com.image.save(buffer, 'png')   # type: ignore
             # put buffer in minio
             object_name = put(
                 client=minio_client,

shared/data_store/__init__.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
 from .connect import connect
+from .delete import delete
 from .get import get
 from .put import put

shared/data_store/delete.py

@@ -0,0 +1,29 @@
+"""Definition of delete function."""
+from __future__ import annotations
+
+import logging
+import os
+
+from minio import Minio
+
+
+def delete(
+    client: Minio,
+    object_name: str,
+) -> None:
+    """Delete object from MinIO."""
+    assert isinstance(client, Minio)
+    assert isinstance(object_name, str)
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None)
+    assert isinstance(bucket_name, str)
+    # remove object
+    try:
+        client.remove_object(
+            bucket_name=bucket_name,
+            object_name=object_name,
+        )
+    except Exception as exc:
+        logging.debug(exc)
+        logging.error('failed deleting %s', object_name)
+    else:
+        logging.debug('deleted %s', object_name)

shared/database/classes/visual_communication.py

@@ -10,6 +10,7 @@ from pathlib import Path
 from minio import Minio
 from PIL import Image
 from pydantic import BaseModel
+from pymongo.collection import Collection
 
 from shared.data_store import get
 from shared.data_store import put
@@ -34,23 +35,60 @@ class VisualCommunication(BaseModel):
         return cls.__name__
 
     @classmethod
-    def from_file(cls, path: Path, minio_client: Minio) -> VisualCommunication:
-        """Instantiate from file."""
-        assert isinstance(path, Path)
+    def upload_image_to_minio(
+        cls,
+        image: Image.Image,
+        minio_client: Minio,
+    ) -> str:
+        """Upload image to MinIO and return MD5 checksum of hashed image."""
+        assert isinstance(image, Image.Image)
         assert isinstance(minio_client, Minio)
-        # determine name
-        name = path.stem
-        # open and upload image to minio
-        image = Image.open(path)
-        image.load()
         buffer = BytesIO()
         image.save(buffer, 'png')
         object_name = put(
             client=minio_client,
             buffer=buffer,
         )
+        return object_name
+
+    @classmethod
+    def from_name_and_image(
+        cls,
+        name: str,
+        image: Image.Image,
+        minio_client: Minio,
+    ) -> VisualCommunication:
+        """
+        Instantiate from filename and image
+        that is automatically uploaded to MinIO.
+        """
+        assert isinstance(name, str)
+        assert isinstance(image, Image.Image)
+        assert isinstance(minio_client, Minio)
+        # upload file to minio
+        object_name = VisualCommunication.upload_image_to_minio(
+            image=image,
+            minio_client=minio_client,
+        )
         return VisualCommunication(name=name, object_name=object_name)
 
+    @classmethod
+    def from_file(cls, path: Path, minio_client: Minio) -> VisualCommunication:
+        """Instantiate from file."""
+        assert isinstance(path, Path)
+        assert isinstance(minio_client, Minio)
+        # determine name
+        name = path.stem
+        # open image
+        image = Image.open(path)
+        image.load()
+        # instantiate object
+        return VisualCommunication.from_name_and_image(
+            name=name,
+            image=image,
+            minio_client=minio_client,
+        )
+
     @classmethod
     def decode_image(cls, content: str) -> Image.Image:
         """Extract image from webencoded content."""
@@ -69,6 +107,13 @@ class VisualCommunication(BaseModel):
         im = Image.open(buffer)
         return im
 
+    def save_to_mongo(self, collection: Collection) -> None:
+        """Save self as document in MongoDB."""
+        res = collection.insert_one(
+            document=self.model_dump(),
+        )
+        assert res.acknowledged
+
     def webencoded_image(self, minio_client: Minio) -> str:
         """Convert image to be displayed on webpage."""
         assert isinstance(minio_client, Minio)

tests/generate_random_prediction_test.py

@@ -2,10 +2,27 @@ from __future__ import annotations
 
 from pathlib import Path
 
-from shared.database.classes import VisualCommunication
+from dotenv import load_dotenv
 
+from shared.data_store import connect as connect_minio
+from shared.database import connect as connect_mongo
+from shared.database.classes import VisualCommunication
+from shared.utils import check_env
+from shared.utils import setup_logging
 
 if __name__ == '__main__':
+    # load in env file
+    env_path = Path(__file__).parent.parent / 'server.env'
+    assert env_path.exists()
+    load_dotenv(env_path)
+    # ensure env vars set
+    check_env()
+    # setup logging
+    setup_logging()
+    # connect to minIO
+    minio_client = connect_minio()
+    # connect to MongoDB
+    collection, db, client = connect_mongo()
     # get list of image paths
     test_dir = Path(__file__).parent
     img_dir = test_dir / 'imgs'
@@ -13,7 +30,7 @@ if __name__ == '__main__':
     print(img_path_list)
     # instantiate data object
     vis_com_list = [
-        VisualCommunication.from_file(path)
+        VisualCommunication.from_file(path, minio_client=minio_client)
         for path
         in img_path_list
     ]

tests/image_upload_test.py

@@ -1,15 +1,29 @@
 from __future__ import annotations
 
-import os
 from pathlib import Path
 
 from dotenv import load_dotenv
 from pymongo.errors import DuplicateKeyError
 
-from shared.database import connect
+from shared.data_store import connect as connect_minio
+from shared.database import connect as connect_mongo
 from shared.database.classes import VisualCommunication
+from shared.utils import check_env
+from shared.utils import setup_logging
 
 if __name__ == '__main__':
+    # load in env file
+    env_path = Path(__file__).parent.parent / 'server.env'
+    assert env_path.exists()
+    load_dotenv(env_path)
+    # ensure env vars set
+    check_env()
+    # setup logging
+    setup_logging()
+    # connect to minIO
+    minio_client = connect_minio()
+    # connect to MongoDB
+    collection, db, client = connect_mongo()
     # get list of image paths
     test_dir = Path(__file__).parent
     img_dir = test_dir / 'imgs'
@@ -17,20 +31,12 @@ if __name__ == '__main__':
     print(img_path_list)
     # instantiate data object
     vis_com_list = [
-        VisualCommunication.from_file(path)
+        VisualCommunication.from_file(path, minio_client=minio_client)
         for path
         in img_path_list
     ]
     for vis_com in vis_com_list:
         print(repr(vis_com))
-    # prepare env vars
-    env_path = test_dir.parent / 'local.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    os.environ['MONGO_HOST'] = 'localhost'
-    # connect to database
-    collection, db, client = connect()
-    print(client.server_info())
     # upload images
     for vis_com in vis_com_list:
         try:

tests/image_upload_to_server_test.py

@@ -5,10 +5,25 @@ from pathlib import Path
 from dotenv import load_dotenv
 from pymongo.errors import DuplicateKeyError
 
-from shared.database import connect
+from shared.data_store import connect as connect_minio
+from shared.database import connect as connect_mongo
 from shared.database import VisualCommunication
+from shared.utils import check_env
+from shared.utils import setup_logging
 
 if __name__ == '__main__':
+    # load in env file
+    env_path = Path(__file__).parent.parent / 'server.env'
+    assert env_path.exists()
+    load_dotenv(env_path)
+    # ensure env vars set
+    check_env()
+    # setup logging
+    setup_logging()
+    # connect to minIO
+    minio_client = connect_minio()
+    # connect to MongoDB
+    collection, db, client = connect_mongo()
     # get list of image paths
     ext_img_dir = Path('/Volumes/BW-PSSD/Mixed Methods/')
     assert ext_img_dir.exists()
@@ -20,19 +35,11 @@ if __name__ == '__main__':
     print(f"found {len(img_path_list)} images")
     # create visual communication objects
     vis_com_list = [
-        VisualCommunication.from_file(path)
+        VisualCommunication.from_file(path, minio_client=minio_client)
         for path
         in img_path_list
     ]
     print(f"created {len(vis_com_list)} visual communication objects")
-    # prepare env vars
-    env_path = Path(__file__).parent.parent / 'server.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    # connect to database
-    collection, db, client = connect()
-    assert client.server_info() is not None
-    print('connected to database')
     # upload images
     for vis_com in vis_com_list:
         try:

tests/prediction_upload_test.py

@@ -1,46 +1,42 @@
 from __future__ import annotations
 
-import logging
-import os
 from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.database import connect
+from shared.data_store import connect as connect_minio
+from shared.database import connect as connect_mongo
 from shared.database import upsert_prediction
 from shared.database.classes import VisualCommunication
+from shared.utils import check_env
+from shared.utils import setup_logging
 
 if __name__ == '__main__':
+    # load in env file
+    env_path = Path(__file__).parent.parent / 'server.env'
+    assert env_path.exists()
+    load_dotenv(env_path)
+    # ensure env vars set
+    check_env()
     # setup logging
-    fmt = (
-        '%(asctime)s | '
-        '%(levelname)s | '
-        '%(filename)s | '
-        '%(funcName)s | '
-        '%(message)s'
-    )
-    datefmt = '%Y-%m-%d %H:%M:%S'
-    logging.basicConfig(format=fmt, datefmt=datefmt, level=logging.INFO)
+    setup_logging()
+    # connect to minIO
+    minio_client = connect_minio()
+    # connect to MongoDB
+    collection, db, client = connect_mongo()
     # get list of image paths
     test_dir = Path(__file__).parent
     img_dir = test_dir / 'imgs'
     img_path_list = [path for path in img_dir.glob('*.jpeg') if path.is_file()]
     # instantiate data object
     vis_com_list = [
-        VisualCommunication.from_file(path)
+        VisualCommunication.from_file(path, minio_client=minio_client)
         for path
         in img_path_list
     ]
     # generate random predictions
     for vis_com in vis_com_list:
         vis_com.generate_random_prediction()
-    # prepare env vars
-    env_path = test_dir.parent / 'local.env'
-    assert env_path.exists()
-    load_dotenv(env_path)
-    os.environ['MONGO_HOST'] = 'localhost'
-    # connect to database
-    collection, db, client = connect()
     # upload visual communication
     for vis_com in vis_com_list:
         if vis_com.prediction is None:

web_ui/src/app/init_app.py

@@ -16,11 +16,11 @@ from pydantic import ValidationError
 from pymongo.collection import Collection
 
 from .layout import app_layout
+from shared.data_store import delete as delete_from_minio
 from shared.database import count_documents
 from shared.database import get_visual_communication
 from shared.database import NoDocumentFoundException
 from shared.database import upsert_annotation
-from shared.database import upsert_visual_communication
 from shared.database import VisualCommunication
 from shared.dto import ModelData
 
@@ -120,34 +120,52 @@ def init_app(
         filename_list: list[str] | None,
     ) -> tuple[str | None, str | None]:
         """Upload image to database through web ui."""
-        # stop early if possible
-        if content_list is None or filename_list is None:
-            logging.info('nothing to upload.')
-            return None, 'nothing to upload'.title()
-        # build list of visual communication
-        vis_com_list = []
+        try:
+            # stop if no input
+            assert (
+                content_list is not None
+            ) and (
+                filename_list is not None
+            ), 'nothing to upload'
+            # handle input
+            failed_filename_list = []
             for content, filename in zip(content_list, filename_list):
                 try:
+                    # decode image content
                     image = VisualCommunication.decode_image(content=content)
-                vis_com = VisualCommunication(
+                except Exception:
+                    logging.debug('failed decoding %s', filename)
+                    failed_filename_list.append(filename)
+                    continue
+                try:
+                    # instantiate to upload image to minio
+                    vis_com = VisualCommunication.from_name_and_image(
                         name=filename,
                         image=image,
+                        minio_client=minio_client,
                     )
                 except Exception as exc:
-                logging.error('failed handling data from file: %s', filename)
                     logging.debug(exc)
+                    failed_filename_list.append(filename)
                     continue
-            vis_com_list.append(vis_com)
-        # upsert documents
-        success = upsert_visual_communication(
-            collection=mongo_collection,
-            visual_communication_list=vis_com_list,
+                try:
+                    # save to mongodb
+                    vis_com.save_to_mongo(collection=mongo_collection)
+                except Exception as exc:
+                    logging.debug(exc)
+                    failed_filename_list.append(filename)
+                    # remove document from minio
+                    delete_from_minio(
+                        client=minio_client,
+                        object_name=vis_com.object_name,
                     )
-        if success:
-            logging.info('uploaded %s files to database', len(vis_com_list))
+            assert len(failed_filename_list) == 0, f"failed uploading:{
+                '\n'.join(failed_filename_list)
+            }"
+        except Exception as exc:
+            logging.debug(exc)
+            return None, str(exc).title()
         return 'successfully uploaded images'.title(), None
-        logging.error('failed uploading images to database')
-        return None, 'failed uploading images'.title()
 
     # define callback: cycle visual communication data
     @app.callback(