updated to do prediction and print result

Dockerfile.model

@@ -1,5 +1,5 @@
 # build stage
-FROM python:3.12-slim-bookworm as BUILDER
+FROM python:3.12-slim-bookworm AS builder
 
 # set environment variables
 ENV PYTHONUNBUFFERED=1 \

Dockerfile.web_ui

@@ -1,5 +1,5 @@
 # build stage
-FROM python:3.12-slim-bookworm as BUILDER
+FROM python:3.12-slim-bookworm AS builder
 
 # set environment variables
 ENV PYTHONUNBUFFERED=1 \

model/src/main.py

@@ -1,25 +1,48 @@
 """Main script to be run by service."""
 
-from pathlib import Path
+import json
+import logging
+from traceback import print_exc
 
-from dotenv import load_dotenv
+import torch
 from models import VisualCommunicationModel
-from torchinfo import summary
+from tqdm import tqdm
-from utils import load_model
+from utils import DEVICE, VCDADataset, load_model
 
-from shared.data_store import connect
+from shared.data_store import connect_minio
+from shared.dto import ModelData
 from shared.utils import setup_logging
 
 if __name__ == '__main__':
-    # load in env file
-    env_path = Path(__file__).parent.parent.parent / 'server.env'
-    load_dotenv(env_path)
     # setup logging
     setup_logging()
     # connect to minio
-    minio_client = connect()
+    minio_client = connect_minio()
     # instantiate model
     model: VisualCommunicationModel = load_model(client=minio_client)
-    # show model weights
+    model.eval()
-    summary(model)
+    # setup dataset
-    print('loaded model')
+    dataset = VCDADataset(
+        minio_client=minio_client,
+        data_name_list=[
+            '02dbaf48d713e4e6d3a6b98fd2dc866e',
+        ],
+        do_augment=False,
+    )
+
+    # make prediction
+    with torch.no_grad():
+        for i in tqdm(range(len(dataset))):
+            try:
+                # get image
+                image = dataset[i]
+                image = torch.unsqueeze(image, 0)  # add artificial batch dimension
+                image = image.to(DEVICE)
+                # make prediction
+                pred: ModelData = model(image)
+            except Exception:
+                print_exc()
+                continue
+            else:
+                print(json.dumps(pred.model_dump(), indent=4))
+    logging.debug('finished')

model/src/model_name.txt

@@ -1 +1 @@
-fb75db772ba8b280bdaf1774d5f73f58
+2bbe2be17a663f4299657378ac5e993e

model/src/models/angle.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/models/contact.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/models/distance.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/models/framing.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/models/fully_connected.py

@@ -1,26 +1,24 @@
-from __future__ import annotations
+from torch import nn
-
-import torch.nn as nn
 
 
 class FullyConnectedModel(nn.Module):
+    """Fully connected layers model template for intrepreting feature space-
+    output from ResNet18 head."""
+
     def __init__(self, num_out_features: int):
         super().__init__()
         # define layers
-        self.fc1 = nn.Linear(in_features=16*16*512, out_features=512)
+        self.fc1 = nn.Linear(in_features=512, out_features=128)
         self.af1 = nn.ReLU()
-        self.fc2 = nn.Linear(in_features=512, out_features=128)
+        self.fc2 = nn.Linear(in_features=128, out_features=32)
         self.af2 = nn.ReLU()
-        self.fc3 = nn.Linear(in_features=128, out_features=32)
+        self.fc3 = nn.Linear(in_features=32, out_features=num_out_features)
-        self.af3 = nn.ReLU()
-        self.fc4 = nn.Linear(in_features=32, out_features=num_out_features)
 
     def forward(self, x):
+        """Pass input through model."""
         x = self.fc1(x)
         x = self.af1(x)
         x = self.fc2(x)
         x = self.af2(x)
         x = self.fc3(x)
-        x = self.af3(x)
-        x = self.fc4(x)
         return x

model/src/models/information_value.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/models/modality_color.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/models/modality_depth.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/models/modality_lighting.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/models/point_of_view.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/models/resnet18_head.py

@@ -5,11 +5,15 @@ import torchvision
 
 
 class ResNet18Head(nn.Module):
-    def __init__(self):
+    def __init__(self, download_resnet_weights: bool = False):
         super().__init__()
         # copy out parts from ResNet18 with weights
+        if download_resnet_weights:
+            weights = torchvision.models.ResNet18_Weights.IMAGENET1K_V1
+        else:
+            weights = None
         resnet18 = torchvision.models.resnet18(
-            weights=torchvision.models.ResNet18_Weights.IMAGENET1K_V1,
+            weights=weights,
         )
         # save relevant layers
         self.conv1 = resnet18.conv1
@@ -21,7 +25,7 @@ class ResNet18Head(nn.Module):
         self.layer3 = resnet18.layer3
         self.layer4 = resnet18.layer4
         self.avgpool = resnet18.avgpool
-        self.flat = nn.Flatten()    # size 512
+        self.flat = nn.Flatten()  # size 512
 
     def forward(self, x):
         x = self.conv1(x)

model/src/models/salience.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/models/visual_communication.py

@@ -4,19 +4,7 @@ from __future__ import annotations
 
 from torch import nn
 
-from shared.dto import (
+from shared.dto import ModelData
-    AngleData,
-    ContactData,
-    DistanceData,
-    FramingData,
-    InformationValueData,
-    ModalityColorData,
-    ModalityDepthData,
-    ModalityLightingData,
-    PointOfViewData,
-    SalienceData,
-    VisualSyntaxData,
-)
 
 from .angle import AngleTail
 from .contact import ContactTail
@@ -35,10 +23,10 @@ from .visual_syntax import VisualSyntaxTail
 class VisualCommunicationModel(nn.Module):
     """Visual communication model."""
 
-    def __init__(self):
+    def __init__(self, download_resnet_weights: bool = False):
         super().__init__()
         # store other models
-        self.resnet_head = ResNet18Head()
+        self.resnet_head = ResNet18Head(download_resnet_weights)
         self.visual_syntax_tail = VisualSyntaxTail()
         self.contact_tail = ContactTail()
         self.angle_tail = AngleTail()
@@ -51,76 +39,24 @@ class VisualCommunicationModel(nn.Module):
         self.framing_tail = FramingTail()
         self.salience_tail = SalienceTail()
 
-    def forward(self, x):
+    def forward(self, x) -> ModelData:
         """Calculate model output on data."""
         # generate visual representation
-        vis_rep = self.resnet_head(x)
+        features = self.resnet_head(x)
-        # prepare result map
+        # make predictions
-        results = {}
+        prediction_dict = {
-        # predict visual syntax
+            'visual_syntax': self.visual_syntax_tail(features),
-        results['visual_syntax'] = VisualSyntaxData.from_list(
+            'contact': self.contact_tail(features),
-            self.visual_syntax_tail(
+            'angle': self.angle_tail(features),
-                vis_rep,
+            'point_of_view': self.point_of_view_tail(features),
-            ).cpu(),
+            'distance': self.distance_tail(features),
-        )
+            'modality_lighting': self.modality_lighting_tail(features),
-        # predict contact
+            'modality_color': self.modality_color_tail(features),
-        results['contact'] = ContactData.from_list(
+            'modality_depth': self.modality_depth_tail(features),
-            self.contact_tail(
+            'information_value': self.information_value_tail(features),
-                vis_rep,
+            'framing': self.framing_tail(features),
-            ).cpu(),
+            'salience': self.salience_tail(features),
-        )
+        }
-        # predict angle
+        # convert to respective classes
-        results['angle'] = AngleData.from_list(
+        data = ModelData.from_prediction_dict(prediction_dict)
-            self.angle_tail(
+        return data
-                vis_rep,
-            ).cpu(),
-        )
-        # predict point of view
-        results['point_of_view'] = PointOfViewData.from_list(
-            self.point_of_view_tail(
-                vis_rep,
-            ).cpu(),
-        )
-        # predict distance
-        results['distance'] = DistanceData.from_list(
-            self.distance_tail(
-                vis_rep,
-            ).cpu(),
-        )
-        # predict modality lighting
-        results['modality_lighting'] = ModalityLightingData.from_list(
-            self.modality_lighting_tail(
-                vis_rep,
-            ).cpu(),
-        )
-        # predict modality color
-        results['modality_color'] = ModalityColorData.from_list(
-            self.modality_color_tail(
-                vis_rep,
-            ).cpu(),
-        )
-        # predict modality depth
-        results['modality_depth'] = ModalityDepthData.from_list(
-            self.modality_depth_tail(
-                vis_rep,
-            ).cpu(),
-        )
-        # predict information value
-        results['information_value'] = InformationValueData.from_list(
-            self.information_value_tail(
-                vis_rep,
-            ).cpu(),
-        )
-        # predict framing
-        results['framing'] = FramingData.from_list(
-            self.framing_tail(
-                vis_rep,
-            ).cpu(),
-        )
-        # predict salience
-        results['salience'] = SalienceData.from_list(
-            self.salience_tail(
-                vis_rep,
-            ).cpu(),
-        )
-        return results

model/src/models/visual_syntax.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 from .fully_connected import FullyConnectedModel
 
 

model/src/utils/__init__.py

@@ -1 +1,2 @@
-from .load_model import load_model
+from .load_model import DEVICE, load_model
+from .vcda_dataset import VCDADataset

model/src/utils/train.py

@@ -6,7 +6,7 @@ import torch
 import torch.nn as nn
 from dataloader import VCDADataset  # noqa: F401
 
-from .models import VisualCommunicationModel
+from ..models import VisualCommunicationModel
 
 PRE_WARMUP_LR = 1e-10
 POST_WARMUP_LR = 1e-5

model/src/utils/vcda_dataset.py

@@ -2,6 +2,7 @@
 
 import random
 
+from minio import Minio
 from PIL import Image
 from torch import Tensor
 from torch.utils.data import Dataset
@@ -15,7 +16,7 @@ from torchvision.transforms.functional import (
     to_tensor,
 )
 
-from shared.data_store import connect, get_image
+from shared.data_store import get_image
 
 # resnet18 original normalization values
 RESNET_NORMALIZE_MEAN = [0.485, 0.456, 0.406]
@@ -27,11 +28,13 @@ class VCDADataset(Dataset):
 
     def __init__(
         self,
+        minio_client: Minio,
         data_name_list: list[str],
         do_augment: bool = False,
         random_annotations: bool = False,
     ):
         super().__init__()
+        self.minio_client = minio_client
         self.data_name_list = data_name_list
         self.do_augment = do_augment
         self.random_annotations = random_annotations
@@ -47,9 +50,6 @@ class VCDADataset(Dataset):
             contrast=8e-2,
             saturation=8e-2,
         )
-        # connect to minio
-        minio_client = connect()
-        self.minio_client = minio_client
 
     def __len__(self):
         return len(self.data_name_list)

poetry.lock

@@ -2228,6 +2228,31 @@ type = "legacy"
 url = "http://192.168.1.2:5001/index"
 reference = "threadripper"
 
+[[package]]
+name = "tqdm"
+version = "4.66.4"
+description = "Fast, Extensible Progress Meter"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "tqdm-4.66.4-py3-none-any.whl", hash = "sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644"},
+    {file = "tqdm-4.66.4.tar.gz", hash = "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[package.extras]
+dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"]
+notebook = ["ipywidgets (>=6)"]
+slack = ["slack-sdk"]
+telegram = ["requests"]
+
+[package.source]
+type = "legacy"
+url = "http://192.168.1.2:5001/index"
+reference = "threadripper"
+
 [[package]]
 name = "trio"
 version = "0.25.1"
@@ -2481,4 +2506,4 @@ reference = "threadripper"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "adcc6b2cdc33ee2aa56b419d6123e3d1553b08641cd81c22b187ecd0096812b1"
+content-hash = "8caa4d24aec4c8cd5b1e726f2e00bc0cf8d865350aa5dc4282fc88d53d1b944c"

pyproject.toml

@@ -34,6 +34,7 @@ torch = "^2.2.1"
 torchvision = "^0.17.1"
 torchinfo = "^1.8.0"
 minio = "^7.2.7"
+tqdm = "^4.66.4"
 
 
 [tool.poetry.group.shared.dependencies]

shared/data_store/__init__.py

@@ -1,4 +1,4 @@
-from .connect import connect
+from .connect_minio import connect_minio
 from .delete import delete
 from .get import get
 from .get_image import get_image

shared/data_store/connect_minio.py

@@ -6,7 +6,7 @@ import os
 from minio import Minio
 
 
-def connect() -> Minio:
+def connect_minio() -> Minio:
     """Connect to MinIO server."""
     # ensure necessary env vars available
     env_var_list = [

shared/database/utils/__init__.py

@@ -1,4 +1,3 @@
 """Database utils module content."""
-from __future__ import annotations
 
-from .connect import connect
+from .connect_mongodb import connect_mongodb

shared/database/utils/connect_mongodb.py

@@ -1,8 +1,5 @@
-"""
+"""Definition of function to connect to database using environment
-Definition of function to connect to database
+variables."""
-using environment variables.
-"""
-from __future__ import annotations
 
 import logging
 import os
@@ -11,7 +8,7 @@ from dotenv import load_dotenv
 from pymongo import MongoClient
 
 
-def connect():
+def connect_mongodb():
     """Connect to MongoDB using env vars."""
     # load env vars
     load_dotenv()

shared/dto/data_model.py

@@ -1,10 +1,9 @@
 """Definition of DataModel base class."""
-from __future__ import annotations
 
 import random
 
-from pydantic import BaseModel
+from pydantic import BaseModel, ValidationError
-from pydantic import ValidationError
+from torch import Tensor
 
 
 class DataModel(BaseModel):
@@ -33,26 +32,27 @@ class DataModel(BaseModel):
             raise ValidationError()
         assert isinstance(option, str), 'option is not a string'
         allowed_options_list = cls.list_fields()
-        assert option in allowed_options_list, \
+        assert (
-            f"{option} is not among allowed fields {allowed_options_list}"
+            option in allowed_options_list
+        ), f"{option} is not among allowed fields {allowed_options_list}"
         kwargs = {field: 0 for field in cls.list_fields()}
         kwargs[option] = 1
         return cls(**kwargs)
 
     @classmethod
-    def from_list(cls, data_list: list[float]):
+    def from_tensor(cls, tensor: Tensor):
         """Instantiate from list of values."""
-        kwargs = {key: val for key, val in zip(cls.list_fields(), data_list)}
+        assert tensor.size(dim=0) == 1, f'tensor batch larger than 1: {tensor}'
+        data_list = [float(t.item()) for t in tensor[0]]
+        kwargs = dict(zip(cls.list_fields(), data_list))
         return cls(**kwargs)
 
     def __repr__(self) -> str:
         model_dict = self.model_dump()
-        model_repr_str = f"{self.classname()}("
+        model_repr_str = f'{self.classname()}('
-        model_repr_str += ', '.join([
+        model_repr_str += ', '.join(
-            f"{field}={value:.3f}"
+            [f'{field}={value:.3f}' for field, value in model_dict.items()],
-            for field, value
+        )
-            in model_dict.items()
-        ])
         model_repr_str += ')'
         return model_repr_str
 

shared/dto/model_data.py

@@ -1,4 +1,5 @@
 """Definition of ModelData data model."""
+
 from __future__ import annotations
 
 from .angle import AngleData
@@ -17,6 +18,7 @@ from .visual_syntax import VisualSyntaxData
 
 class ModelData(DataModel):
     """ModelData model for data IO with combined ML model."""
+
     visual_syntax: VisualSyntaxData
     contact: ContactData
     angle: AngleData
@@ -34,8 +36,50 @@ class ModelData(DataModel):
         """Instantiate with random numbers."""
         kwargs = {
             field: field_info.annotation.from_random()  # type: ignore
-            for field, field_info
+            for field, field_info in cls.model_fields.items()
-            in cls.model_fields.items()
+        }
+        return cls(**kwargs)
+
+    @classmethod
+    def from_prediction_dict(
+        cls,
+        prediction_dict: dict,
+    ) -> ModelData:
+        """Instantiate from prediction dictionary."""
+        kwargs = {
+            'visual_syntax': VisualSyntaxData.from_tensor(
+                prediction_dict['visual_syntax'],
+            ),
+            'contact': ContactData.from_tensor(
+                prediction_dict['contact'],
+            ),
+            'angle': AngleData.from_tensor(
+                prediction_dict['angle'],
+            ),
+            'point_of_view': PointOfViewData.from_tensor(
+                prediction_dict['point_of_view'],
+            ),
+            'distance': DistanceData.from_tensor(
+                prediction_dict['distance'],
+            ),
+            'modality_lighting': ModalityLightingData.from_tensor(
+                prediction_dict['modality_lighting'],
+            ),
+            'modality_color': ModalityColorData.from_tensor(
+                prediction_dict['modality_color'],
+            ),
+            'modality_depth': ModalityDepthData.from_tensor(
+                prediction_dict['modality_depth'],
+            ),
+            'information_value': InformationValueData.from_tensor(
+                prediction_dict['information_value'],
+            ),
+            'framing': FramingData.from_tensor(
+                prediction_dict['framing'],
+            ),
+            'salience': SalienceData.from_tensor(
+                prediction_dict['salience'],
+            ),
         }
         return cls(**kwargs)
 
@@ -56,27 +100,16 @@ class ModelData(DataModel):
     ) -> ModelData:
         """Instantiate from annotation."""
         kwargs = {
-            'visual_syntax': VisualSyntaxData
+            'visual_syntax': VisualSyntaxData.from_choice(visual_syntax),
-            .from_choice(visual_syntax),
+            'contact': ContactData.from_choice(contact),
-            'contact': ContactData
+            'angle': AngleData.from_choice(angle),
-            .from_choice(contact),
+            'point_of_view': PointOfViewData.from_choice(point_of_view),
-            'angle': AngleData
+            'distance': DistanceData.from_choice(distance),
-            .from_choice(angle),
+            'modality_lighting': ModalityLightingData.from_choice(modality_lighting),
-            'point_of_view': PointOfViewData
+            'modality_color': ModalityColorData.from_choice(modality_color),
-            .from_choice(point_of_view),
+            'modality_depth': ModalityDepthData.from_choice(modality_depth),
-            'distance': DistanceData
+            'information_value': InformationValueData.from_choice(information_value),
-            .from_choice(distance),
+            'framing': FramingData.from_choice(framing),
-            'modality_lighting': ModalityLightingData
+            'salience': SalienceData.from_choice(salience),
-            .from_choice(modality_lighting),
-            'modality_color': ModalityColorData
-            .from_choice(modality_color),
-            'modality_depth': ModalityDepthData
-            .from_choice(modality_depth),
-            'information_value': InformationValueData
-            .from_choice(information_value),
-            'framing': FramingData
-            .from_choice(framing),
-            'salience': SalienceData
-            .from_choice(salience),
         }
         return cls(**kwargs)

tests/new_model_to_minio_test.py

@@ -0,0 +1,32 @@
+"""Definition of function to generate a new randomly initialized model and save
+it in Minio datastore."""
+
+from pathlib import Path
+
+from dotenv import load_dotenv
+from torchinfo import summary
+
+from model.src.models import VisualCommunicationModel
+from shared.data_store import connect_minio, put_model
+from shared.utils import setup_logging
+
+if __name__ == '__main__':
+    # load in env file
+    env_path = Path(__file__).parent.parent / 'server.env'
+    assert env_path.exists()
+    load_dotenv(env_path)
+    # setup logging
+    setup_logging()
+    # connect to minio
+    client = connect_minio()
+    # instantiate model
+    model = VisualCommunicationModel(download_resnet_weights=True)
+    # show model weights
+    summary(model)
+    # put buffer in minio bucket
+    hash_str = put_model(
+        client=client,
+        model=model,
+    )
+    print(f"hash string: {hash_str}")
+    print('saved to Minio')