fixed mypy types

model/src/utils/get_class.py

@@ -1,8 +1,9 @@
 from importlib import import_module
-from types import ModuleType
+
+from torch.nn import Module
 
 
-def get_class(path: str) -> ModuleType:
+def get_class(path: str) -> Module:
     parts = path.split('.')
     module_path = '.'.join(parts[:-1])
     class_name = parts[-1]

model/src/utils/load_model.py

@@ -5,8 +5,8 @@ from pathlib import Path
 
 import torch
 from minio import Minio
-from src.models import VisualCommunicationModel
 
+from model.src.models import VisualCommunicationModel
 from shared.data_store import get_model
 
 from .get_model_name import get_model_name

model/train.py

@@ -13,11 +13,11 @@ from ignite.handlers.param_scheduler import create_lr_scheduler_with_warmup
 from ignite.handlers.tensorboard_logger import TensorboardLogger
 from ignite.handlers.tqdm_logger import ProgressBar
 from ignite.metrics import Average, Loss, RunningAverage
-from src.utils import VCDADataset, get_class
 from torch import nn
 from torch.optim.lr_scheduler import ExponentialLR
 from torch.utils.data import DataLoader
 
+from model.src.utils import VCDADataset, get_class
 from shared.data_store import connect_minio
 
 
@@ -90,7 +90,7 @@ val_metrics = {
 }
 evaluator = create_supervised_evaluator(
     model=model,
-    metrics=val_metrics,
+    metrics=val_metrics,  # type: ignore
     device=device,
 )
 ProgressBar(desc='Val', ncols=80).attach(evaluator)

other/transfer_images_minio_subfolder.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from dotenv import load_dotenv
 from PIL import Image
 
-from shared.data_store import connect, get, put_image
+from shared.data_store import connect_minio, get, put_image
 from shared.utils import setup_logging
 
 if __name__ == '__main__':
@@ -16,7 +16,7 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minio
-    minio_client = connect()
+    minio_client = connect_minio()
     # list images in bucket
     BUCKET_NAME = 'visual-critical-discourse-analysis'
     obj_list = minio_client.list_objects(

other/transfer_images_mongo_minio.py

@@ -1,4 +1,5 @@
 """Script to move all images from MongoDB to MinIO."""
+
 from __future__ import annotations
 
 import logging
@@ -9,12 +10,9 @@ from bson import ObjectId
 from dotenv import load_dotenv
 from pymongo.collection import Collection
 
-from shared.data_store import connect as connect_minio
+from shared.data_store import connect_minio, put
-from shared.data_store import put
+from shared.database import VisualCommunication, connect_mongodb
-from shared.database import connect
+from shared.utils import check_env, setup_logging
-from shared.database import VisualCommunication
-from shared.utils import check_env
-from shared.utils import setup_logging
 
 
 def list_mongo_document_ids(
@@ -84,7 +82,7 @@ if __name__ == '__main__':
     # connect to minIO
     minio_client = connect_minio()
     # connect to MongoDB
-    collection, db, client = connect()
+    collection, db, client = connect_mongodb()
     # list documents in mongoDB
     id_list = list_mongo_document_ids(collection)
     for doc_id in id_list:
@@ -98,7 +96,7 @@ if __name__ == '__main__':
         try:
             # save image to buffer
             buffer = BytesIO()
-            vis_com.image.save(buffer, 'png')   # type: ignore
+            vis_com.image.save(buffer, 'png')  # type: ignore
             # put buffer in minio
             object_name = put(
                 client=minio_client,
@@ -110,13 +108,14 @@ if __name__ == '__main__':
             continue
         # update visual communication in mongodb
         try:
-            vis_com.image = None    # type: ignore
+            vis_com.image = None  # type: ignore
             vis_com.object_name = object_name
             update_visual_communication(collection, vis_com)
         except Exception as exc:
             logging.debug(exc)
             logging.error(
-                'failed updating visual communication %s', vis_com.name,
+                'failed updating visual communication %s',
+                vis_com.name,
             )
             continue
         logging.debug('updated visual communication %s', vis_com.name)

poetry.lock

@@ -2553,6 +2553,22 @@ type = "legacy"
 url = "http://192.168.1.2:5001/index"
 reference = "threadripper"
 
+[[package]]
+name = "types-tqdm"
+version = "4.66.0.20240417"
+description = "Typing stubs for tqdm"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "types-tqdm-4.66.0.20240417.tar.gz", hash = "sha256:16dce9ef522ea8d40e4f5b8d84dd8a1166eefc13ceee7a7e158bf0f1a1421a31"},
+    {file = "types_tqdm-4.66.0.20240417-py3-none-any.whl", hash = "sha256:248aef1f9986b7b8c2c12b3cb4399fc17dba0a29e7e3f3f9cd704babb879383d"},
+]
+
+[package.source]
+type = "legacy"
+url = "http://192.168.1.2:5001/index"
+reference = "threadripper"
+
 [[package]]
 name = "typing-extensions"
 version = "4.12.1"
@@ -2695,4 +2711,4 @@ reference = "threadripper"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "64b30e34cd397481936704022e9e7e8a256a85215b180199c56ca8770c0bdc0e"
+content-hash = "e311eaec3058b444c5bb980a3700ca9de273d7948ace5d5b94363f4f967b622a"

pyproject.toml

@@ -20,6 +20,7 @@ types-requests = "^2.32.0.20240602"
 types-retry = "^0.9.9.4"
 flake8-pyproject = "^1.2.3"
 pandas-stubs = "^2.2.2.240603"
+types-tqdm = "^4.66.0.20240417"
 
 
 [tool.poetry.group.dev.dependencies]

shared/data_store/connect_minio.py

@@ -23,10 +23,10 @@ def connect_minio() -> Minio:
             env_var
         }"
     # prepare arguments
-    minio_endpoint = os.getenv('MINIO_ENDPOINT')
+    minio_endpoint = os.getenv('MINIO_ENDPOINT', default='')
-    minio_access_key = os.getenv('MINIO_ACCESS_KEY')
+    minio_access_key = os.getenv('MINIO_ACCESS_KEY', default='')
-    minio_secret_key = os.getenv('MINIO_SECRET_KEY')
+    minio_secret_key = os.getenv('MINIO_SECRET_KEY', default='')
-    minio_bucket_name = os.getenv('MINIO_BUCKET_NAME')
+    minio_bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
     # connect client
     client = Minio(
         endpoint=minio_endpoint,

shared/data_store/delete.py

@@ -1,4 +1,5 @@
 """Definition of delete function."""
+
 from __future__ import annotations
 
 import logging
@@ -14,8 +15,8 @@ def delete(
     """Delete object from MinIO."""
     assert isinstance(client, Minio)
     assert isinstance(object_name, str)
-    bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None)
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
-    assert isinstance(bucket_name, str)
+    assert len(bucket_name) > 0
     # remove object
     try:
         client.remove_object(

shared/data_store/get.py

@@ -15,7 +15,7 @@ def get(
     """Get buffer from bucket in MinIO."""
     assert isinstance(client, Minio)
     assert isinstance(object_name, str)
-    bucket_name = os.getenv('MINIO_BUCKET_NAME')
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
     # get buffer
     try:
         response = client.get_object(

shared/data_store/get_image.py

@@ -17,7 +17,8 @@ def get_image(
     assert isinstance(object_name, str)
     assert len(object_name) > 0
     # prepare arguments
-    bucket_name = os.getenv('MINIO_BUCKET_NAME')
+    assert 'MINIO_BUCKET_NAME' in os.environ
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
     subfolder = 'images'
     # get object from bucket
     object_name = f'{subfolder}/{object_name}'

shared/data_store/put.py

@@ -1,4 +1,5 @@
 """Definition of put function."""
+
 from __future__ import annotations
 
 import logging
@@ -16,8 +17,8 @@ def put(
     """Put buffer in bucket in MinIO and return MD5 checksum as object name."""
     assert isinstance(client, Minio)
     assert isinstance(buffer, BytesIO)
-    bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None)
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
-    assert isinstance(bucket_name, str)
+    assert len(bucket_name) > 0
     # get md5 of image
     checksum = md5(buffer.getbuffer()).hexdigest()
     # prepare for saving

shared/data_store/put_image.py

@@ -17,8 +17,8 @@ def put_image(
     used as object name."""
     assert isinstance(client, Minio)
     assert isinstance(image, Image.Image)
-    bucket_name = os.getenv('MINIO_BUCKET_NAME', default=None)
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
-    assert isinstance(bucket_name, str)
+    assert len(bucket_name) > 0
     subfolder = 'images'
     # save image to buffer
     buffer = BytesIO()

shared/data_store/put_model.py

@@ -18,7 +18,8 @@ def put_model(
     used as object name."""
     assert isinstance(client, Minio)
     assert isinstance(model, Module)
-    bucket_name = os.getenv('MINIO_BUCKET_NAME')
+    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
+    assert len(bucket_name) > 0
     subfolder = 'models'
     # save image to buffer
     buffer = BytesIO()

shared/database/__init__.py

@@ -1,10 +1,11 @@
 """Database module content."""
+
 from __future__ import annotations
 
 from .classes.dataset import Dataset
 from .classes.exceptions import NoDocumentFoundException
 from .classes.visual_communication import VisualCommunication
-from .utils.connect import connect
+from .utils.connect_mongodb import connect_mongodb
 from .utils.count_documents import count_documents
 from .utils.get_visual_communication import get_visual_communication
 from .utils.list_names import list_names

shared/database/utils/save_dataset.py

@@ -20,10 +20,12 @@ def save_dataset(
 
 if __name__ == '__main__':
     from dotenv import load_dotenv
+
     load_dotenv('local.env')
-    from shared.database import list_names, connect
+    from shared.database import connect_mongodb, list_names
+
     # connect to database
-    collection, db, client = connect()
+    collection, db, client = connect_mongodb()
     print(client.server_info())
 
     name_list = list_names(collection=collection, only_with_annotation=True)

tests/generate_random_prediction_test.py

@@ -4,11 +4,10 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.data_store import connect as connect_minio
+from shared.data_store import connect_minio
-from shared.database import connect as connect_mongo
+from shared.database import connect_mongodb
 from shared.database.classes import VisualCommunication
-from shared.utils import check_env
+from shared.utils import check_env, setup_logging
-from shared.utils import setup_logging
 
 if __name__ == '__main__':
     # load in env file
@@ -22,7 +21,7 @@ if __name__ == '__main__':
     # connect to minIO
     minio_client = connect_minio()
     # connect to MongoDB
-    collection, db, client = connect_mongo()
+    collection, db, client = connect_mongodb()
     # get list of image paths
     test_dir = Path(__file__).parent
     img_dir = test_dir / 'imgs'
@@ -31,8 +30,7 @@ if __name__ == '__main__':
     # instantiate data object
     vis_com_list = [
         VisualCommunication.from_file(path, minio_client=minio_client)
-        for path
+        for path in img_path_list
-        in img_path_list
     ]
     # generate random predictions
     for vis_com in vis_com_list:

tests/get_visual_communication_test.py

@@ -4,13 +4,9 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.data_store import (
+from shared.data_store import connect_minio
-    connect as connect_minio,
+from shared.database import connect_mongodb, get_visual_communication
-)
+from shared.utils import check_env, setup_logging
-from shared.database import connect
-from shared.database import get_visual_communication
-from shared.utils import check_env
-from shared.utils import setup_logging
 
 if __name__ == '__main__':
     # load in env file
@@ -24,7 +20,7 @@ if __name__ == '__main__':
     # connect to minIO
     minio_client = connect_minio()
     # connect to MongoDB
-    collection, db, client = connect()
+    collection, db, client = connect_mongodb()
     # get visual communication
     vis_com = get_visual_communication(collection)
     print(repr(vis_com))

tests/image_upload_test.py

@@ -5,11 +5,10 @@ from pathlib import Path
 from dotenv import load_dotenv
 from pymongo.errors import DuplicateKeyError
 
-from shared.data_store import connect as connect_minio
+from shared.data_store import connect_minio
-from shared.database import connect as connect_mongo
+from shared.database import connect_mongodb
 from shared.database.classes import VisualCommunication
-from shared.utils import check_env
+from shared.utils import check_env, setup_logging
-from shared.utils import setup_logging
 
 if __name__ == '__main__':
     # load in env file
@@ -23,7 +22,7 @@ if __name__ == '__main__':
     # connect to minIO
     minio_client = connect_minio()
     # connect to MongoDB
-    collection, db, client = connect_mongo()
+    collection, db, client = connect_mongodb()
     # get list of image paths
     test_dir = Path(__file__).parent
     img_dir = test_dir / 'imgs'
@@ -32,8 +31,7 @@ if __name__ == '__main__':
     # instantiate data object
     vis_com_list = [
         VisualCommunication.from_file(path, minio_client=minio_client)
-        for path
+        for path in img_path_list
-        in img_path_list
     ]
     for vis_com in vis_com_list:
         print(repr(vis_com))

tests/image_upload_to_server_test.py

@@ -5,11 +5,9 @@ from pathlib import Path
 from dotenv import load_dotenv
 from pymongo.errors import DuplicateKeyError
 
-from shared.data_store import connect as connect_minio
+from shared.data_store import connect_minio
-from shared.database import connect as connect_mongo
+from shared.database import VisualCommunication, connect_mongodb
-from shared.database import VisualCommunication
+from shared.utils import check_env, setup_logging
-from shared.utils import check_env
-from shared.utils import setup_logging
 
 if __name__ == '__main__':
     # load in env file
@@ -23,21 +21,22 @@ if __name__ == '__main__':
     # connect to minIO
     minio_client = connect_minio()
     # connect to MongoDB
-    collection, db, client = connect_mongo()
+    collection, db, client = connect_mongodb()
     # get list of image paths
     ext_img_dir = Path('/Volumes/BW-PSSD/Mixed Methods/')
     assert ext_img_dir.exists()
     img_path_list = [
-        path for path in ext_img_dir.glob(
+        path
+        for path in ext_img_dir.glob(
             '*.jpg',
-        ) if path.is_file()
+        )
+        if path.is_file()
     ]
     print(f"found {len(img_path_list)} images")
     # create visual communication objects
     vis_com_list = [
         VisualCommunication.from_file(path, minio_client=minio_client)
-        for path
+        for path in img_path_list
-        in img_path_list
     ]
     print(f"created {len(vis_com_list)} visual communication objects")
     # upload images

tests/model_io_test.py

@@ -7,7 +7,7 @@ from dotenv import load_dotenv
 from torchinfo import summary
 
 from model.src.models import VisualCommunicationModel
-from shared.data_store import connect, put_model
+from shared.data_store import connect_minio, put_model
 from shared.utils import setup_logging
 
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
@@ -20,7 +20,7 @@ if __name__ == '__main__':
     # setup logging
     setup_logging()
     # connect to minio
-    client = connect()
+    client = connect_minio()
     # instantiate model
     model = VisualCommunicationModel().to(DEVICE)
     # show model weights

tests/prediction_upload_test.py

@@ -4,12 +4,10 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.data_store import connect as connect_minio
+from shared.data_store import connect_minio
-from shared.database import connect as connect_mongo
+from shared.database import connect_mongodb, upsert_prediction
-from shared.database import upsert_prediction
 from shared.database.classes import VisualCommunication
-from shared.utils import check_env
+from shared.utils import check_env, setup_logging
-from shared.utils import setup_logging
 
 if __name__ == '__main__':
     # load in env file
@@ -23,7 +21,7 @@ if __name__ == '__main__':
     # connect to minIO
     minio_client = connect_minio()
     # connect to MongoDB
-    collection, db, client = connect_mongo()
+    collection, db, client = connect_mongodb()
     # get list of image paths
     test_dir = Path(__file__).parent
     img_dir = test_dir / 'imgs'
@@ -31,8 +29,7 @@ if __name__ == '__main__':
     # instantiate data object
     vis_com_list = [
         VisualCommunication.from_file(path, minio_client=minio_client)
-        for path
+        for path in img_path_list
-        in img_path_list
     ]
     # generate random predictions
     for vis_com in vis_com_list:

tests/total_annotated_test.py

@@ -5,8 +5,7 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.database import connect
+from shared.database import connect_mongodb, count_documents
-from shared.database import count_documents
 
 if __name__ == '__main__':
     # prepare env vars
@@ -15,7 +14,7 @@ if __name__ == '__main__':
     load_dotenv(env_path)
     os.environ['MONGO_HOST'] = 'localhost'
     # connect to database
-    collection, db, client = connect()
+    collection, db, client = connect_mongodb()
     # get visual communication
     num_docs = count_documents(
         collection=collection,

tests/total_documents_test.py

@@ -5,8 +5,7 @@ from pathlib import Path
 
 from dotenv import load_dotenv
 
-from shared.database import connect
+from shared.database import connect_mongodb, count_documents
-from shared.database import count_documents
 
 if __name__ == '__main__':
     # prepare env vars
@@ -15,7 +14,7 @@ if __name__ == '__main__':
     load_dotenv(env_path)
     os.environ['MONGO_HOST'] = 'localhost'
     # connect to database
-    collection, db, client = connect()
+    collection, db, client = connect_mongodb()
     # get visual communication
     num_docs = count_documents(
         collection=collection,

web_ui/src/main.py

@@ -1,13 +1,14 @@
 """Definition of web_ui main script."""
+
 from __future__ import annotations
 
 import os
 
+from shared.data_store import connect_minio
+from shared.database import connect_mongodb
+from shared.utils import check_env, setup_logging
+
 from .app import init_app
-from shared.data_store import connect as connect_minio
-from shared.database import connect
-from shared.utils import check_env
-from shared.utils import setup_logging
 
 # ensure env vars set
 check_env()
@@ -16,7 +17,7 @@ check_env()
 setup_logging()
 
 # connect to database
-collection, db, client = connect()
+collection, db, client = connect_mongodb()
 
 # connect to minio
 minio_client = connect_minio()