began defining visual communication model

core/data_models/__init__.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from .angle import AngleData
+from .contact import ContactData
+from .distance import DistanceData
+from .framing import FramingData
+from .information_value import InformationValueData
+from .modality_color import ModalityColorData
+from .modality_depth import ModalityDepthData
+from .modality_lighting import ModalityLightingData
+from .point_of_view import PointOfViewData
+from .salience import SalienceData
+from .visual_syntax import VisualSyntaxData

core/data_models/angle.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class AngleData(DataModel):
+    high: float
+    eye_level: float
+    low: float

core/data_models/contact.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class ContactData(DataModel):
+    offer: float
+    demand: float

core/data_models/data_model.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import random
+
+from pydantic import BaseModel
+from pydantic import ValidationError
+
+
+class DataModel(BaseModel):
+
+    @classmethod
+    def classname(cls) -> str:
+        """Return classname."""
+        return cls.__name__
+
+    @classmethod
+    def list_fields(cls) -> list[str]:
+        """List options that are stored as attributes."""
+        return list(cls.model_fields.keys())
+
+    @classmethod
+    def from_random(cls):
+        """Instantiate with random numbers."""
+        kwargs = {field: random.random() for field in cls.list_fields()}
+        return cls(**kwargs)
+
+    @classmethod
+    def from_choice(cls, option: str):
+        """Instantiate from choice."""
+        if option is None:
+            raise ValidationError()
+        assert isinstance(option, str), 'option is not a string'
+        allowed_options_list = cls.list_fields()
+        assert option in allowed_options_list, \
+            f"{option} is not among allowed fields {allowed_options_list}"
+        kwargs = {field: 0 for field in cls.list_fields()}
+        kwargs[option] = 1
+        return cls(**kwargs)
+
+    @classmethod
+    def from_list(cls, data_list: list[float]):
+        """Instantiate from list of values."""
+        kwargs = {key: val for key, val in zip(cls.list_fields(), data_list)}
+        return cls(**kwargs)
+
+    def __repr__(self) -> str:
+        model_dict = self.model_dump()
+        model_repr_str = f"{self.classname()}("
+        model_repr_str += ', '.join([
+            f"{field}={value:.3f}"
+            for field, value
+            in model_dict.items()
+        ])
+        model_repr_str += ')'
+        return model_repr_str
+
+    def highest_score_field(self) -> str:
+        """Return name of field with highest score."""
+        model_dict = self.model_dump()
+        return max(model_dict, key=lambda k: model_dict[k])
+
+    def highest_score_value(self) -> float:
+        """Return value of field with highest score."""
+        model_dict = self.model_dump()
+        return max(model_dict.values())

core/data_models/distance.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class DistanceData(DataModel):
+    long: float
+    medium: float
+    close: float

core/data_models/framing.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class FramingData(DataModel):
+    frame_lines: float
+    empty_space: float
+    colour_contrast: float
+    form_contrast: float

core/data_models/information_value.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class InformationValueData(DataModel):
+    given_new: float
+    ideal_real: float
+    central_marginal: float

core/data_models/modality_color.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class ModalityColorData(DataModel):
+    high: float
+    medium: float
+    low: float

core/data_models/modality_depth.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class ModalityDepthData(DataModel):
+    high: float
+    medium: float
+    low: float

core/data_models/modality_lighting.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class ModalityLightingData(DataModel):
+    high: float
+    medium: float
+    low: float

core/data_models/point_of_view.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class PointOfViewData(DataModel):
+    frontal: float
+    oblique: float

core/data_models/salience.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class SalienceData(DataModel):
+    size: float
+    colour: float
+    tone: float
+    form: float
+    positioning: float

core/data_models/visual_syntax.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from .data_model import DataModel
+
+
+class VisualSyntaxData(DataModel):
+    non_transactional_action: float
+    non_transactional_reaction: float
+    unidirectional_transactional_action: float
+    unidirectional_transactional_reaction: float
+    bidirectional_transactional_action: float
+    bidirectional_transactional_reaction: float
+    conversion: float
+    speech_process: float
+    classification_overt_taxonomy: float
+    analytical_exhaustive: float
+    analytical_disarranged: float
+    analytical_temporal: float
+    analytical_distributed: float
+    analytical_topological: float
+    analytical_exploded: float
+    analytical_inclusive: float
+    symbolic_suggestive: float
+    symbolic_attributive: float

model/src/dataloader.py

@@ -15,7 +15,6 @@ from torchvision.transforms.functional import resize
 from torchvision.transforms.functional import rotate
 
 from core.database import connect
-from core.database import list_names
 
 # resnet18 original normalization values
 RESNET_NORMALIZE_MEAN = [0.485, 0.456, 0.406]
@@ -25,12 +24,14 @@ RESNET_NORMALIZE_STD = [0.229, 0.224, 0.225]
 class VCDADataset(Dataset):
     def __init__(
         self,
+        data_name_list: list[str],
         do_augment: bool = False,
         random_annotations: bool = False,
         normalize_mean: list[float] = RESNET_NORMALIZE_MEAN,
         normalize_std: list[float] = RESNET_NORMALIZE_STD,
     ):
         super().__init__()
+        self.data_name_list = data_name_list
         self.do_augment = do_augment
         self.random_annotations = random_annotations
         self.normalize_mean = normalize_mean
@@ -48,12 +49,6 @@ class VCDADataset(Dataset):
         # connect to database
         collection, _, _ = connect()
         self.collection = collection
-        # load data names
-        has_annotation = False if self.random_annotations else True
-        self.data_name_list = list_names(
-            collection=self.collection,
-            has_annotation=has_annotation,
-        )
 
     def __len__(self):
         return len(self.data_name_list)

model/src/loss_fn.py

@@ -1,7 +0,0 @@
-from __future__ import annotations
-
-import torch.nn as nn
-
-
-def setup_criterion():
-    return nn.MSELoss()

model/src/models/angle.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class AngleTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=3)

model/src/models/contact.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class ContactTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=2)

model/src/models/distance.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class DistanceTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=3)

model/src/models/framing.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class FramingTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=4)

model/src/models/fully_connected.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import torch.nn as nn
+
+
+class FullyConnectedModel(nn.Module):
+    def __init__(self, num_out_features: int):
+        super().__init__()
+        # define layers
+        self.fc1 = nn.Linear(in_features=16*16*512, out_features=512)
+        self.af1 = nn.ReLU()
+        self.fc2 = nn.Linear(in_features=512, out_features=128)
+        self.af2 = nn.ReLU()
+        self.fc3 = nn.Linear(in_features=128, out_features=32)
+        self.af3 = nn.ReLU()
+        self.fc4 = nn.Linear(in_features=32, out_features=num_out_features)
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.af1(x)
+        x = self.fc2(x)
+        x = self.af2(x)
+        x = self.fc3(x)
+        x = self.af3(x)
+        x = self.fc4(x)
+        return x

model/src/models/information_value.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class InformationValueTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=3)

model/src/models/modality_color.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class ModalityColorTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=3)

model/src/models/modality_depth.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class ModalityDepthTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=3)

model/src/models/modality_lighting.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class ModalityLightingTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=3)

model/src/models/point_of_view.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class PointOfViewTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=2)

model/src/models/salience.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class SalienceTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=5)

model/src/models/visual_communication.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import torch.nn as nn
+
+from .resnet18_head import ResNet18Head
+from .visual_syntax import VisualSyntaxTail
+from core.data_models import VisualSyntaxData
+
+
+class VisualCommunicationModel(nn.Module):
+    def __init__(self):
+        super().__init__()
+        # store other models
+        self.resnet_head = ResNet18Head()
+        self.visual_syntax_tail = VisualSyntaxTail()
+
+    def forward(self, x):
+        # generate visual representation
+        vis_rep = self.resnet_head(x)
+        # prepare result map
+        results = {}
+        # predict visual syntax
+        visual_syntax_pred = self.visual_syntax_tail(vis_rep).cpu()
+        # visual_syntax_pred = float()
+        results['visual_syntax'] = VisualSyntaxData.from_list(
+            visual_syntax_pred,
+        )
+
+        return results

model/src/models/visual_syntax.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from .fully_connected import FullyConnectedModel
+
+
+class VisualSyntaxTail(FullyConnectedModel):
+    def __init__(self):
+        super().__init__(num_out_features=18)