extended tests

poetry.lock

@@ -552,6 +552,33 @@ type = "legacy"
 url = "http://192.168.1.2:5001/index"
 reference = "threadripper"
 
+[[package]]
+name = "docker"
+version = "7.1.0"
+description = "A Python library for the Docker Engine API."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"},
+    {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"},
+]
+
+[package.dependencies]
+pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""}
+requests = ">=2.26.0"
+urllib3 = ">=1.26.0"
+
+[package.extras]
+dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"]
+docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"]
+ssh = ["paramiko (>=2.4.3)"]
+websockets = ["websocket-client (>=1.3.0)"]
+
+[package.source]
+type = "legacy"
+url = "http://192.168.1.2:5001/index"
+reference = "threadripper"
+
 [[package]]
 name = "filelock"
 version = "3.14.0"
@@ -2114,6 +2141,38 @@ type = "legacy"
 url = "http://192.168.1.2:5001/index"
 reference = "threadripper"
 
+[[package]]
+name = "pywin32"
+version = "308"
+description = "Python for Window Extensions"
+optional = false
+python-versions = "*"
+files = [
+    {file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"},
+    {file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"},
+    {file = "pywin32-308-cp310-cp310-win_arm64.whl", hash = "sha256:a5ab5381813b40f264fa3495b98af850098f814a25a63589a8e9eb12560f450c"},
+    {file = "pywin32-308-cp311-cp311-win32.whl", hash = "sha256:5d8c8015b24a7d6855b1550d8e660d8daa09983c80e5daf89a273e5c6fb5095a"},
+    {file = "pywin32-308-cp311-cp311-win_amd64.whl", hash = "sha256:575621b90f0dc2695fec346b2d6302faebd4f0f45c05ea29404cefe35d89442b"},
+    {file = "pywin32-308-cp311-cp311-win_arm64.whl", hash = "sha256:100a5442b7332070983c4cd03f2e906a5648a5104b8a7f50175f7906efd16bb6"},
+    {file = "pywin32-308-cp312-cp312-win32.whl", hash = "sha256:587f3e19696f4bf96fde9d8a57cec74a57021ad5f204c9e627e15c33ff568897"},
+    {file = "pywin32-308-cp312-cp312-win_amd64.whl", hash = "sha256:00b3e11ef09ede56c6a43c71f2d31857cf7c54b0ab6e78ac659497abd2834f47"},
+    {file = "pywin32-308-cp312-cp312-win_arm64.whl", hash = "sha256:9b4de86c8d909aed15b7011182c8cab38c8850de36e6afb1f0db22b8959e3091"},
+    {file = "pywin32-308-cp313-cp313-win32.whl", hash = "sha256:1c44539a37a5b7b21d02ab34e6a4d314e0788f1690d65b48e9b0b89f31abbbed"},
+    {file = "pywin32-308-cp313-cp313-win_amd64.whl", hash = "sha256:fd380990e792eaf6827fcb7e187b2b4b1cede0585e3d0c9e84201ec27b9905e4"},
+    {file = "pywin32-308-cp313-cp313-win_arm64.whl", hash = "sha256:ef313c46d4c18dfb82a2431e3051ac8f112ccee1a34f29c263c583c568db63cd"},
+    {file = "pywin32-308-cp37-cp37m-win32.whl", hash = "sha256:1f696ab352a2ddd63bd07430080dd598e6369152ea13a25ebcdd2f503a38f1ff"},
+    {file = "pywin32-308-cp37-cp37m-win_amd64.whl", hash = "sha256:13dcb914ed4347019fbec6697a01a0aec61019c1046c2b905410d197856326a6"},
+    {file = "pywin32-308-cp38-cp38-win32.whl", hash = "sha256:5794e764ebcabf4ff08c555b31bd348c9025929371763b2183172ff4708152f0"},
+    {file = "pywin32-308-cp38-cp38-win_amd64.whl", hash = "sha256:3b92622e29d651c6b783e368ba7d6722b1634b8e70bd376fd7610fe1992e19de"},
+    {file = "pywin32-308-cp39-cp39-win32.whl", hash = "sha256:7873ca4dc60ab3287919881a7d4f88baee4a6e639aa6962de25a98ba6b193341"},
+    {file = "pywin32-308-cp39-cp39-win_amd64.whl", hash = "sha256:71b3322d949b4cc20776436a9c9ba0eeedcbc9c650daa536df63f0ff111bb920"},
+]
+
+[package.source]
+type = "legacy"
+url = "http://192.168.1.2:5001/index"
+reference = "threadripper"
+
 [[package]]
 name = "requests"
 version = "2.32.3"
@@ -2353,6 +2412,63 @@ type = "legacy"
 url = "http://192.168.1.2:5001/index"
 reference = "threadripper"
 
+[[package]]
+name = "testcontainers"
+version = "4.8.2"
+description = "Python library for throwaway instances of anything that can run in a Docker container"
+optional = false
+python-versions = ">=3.9,<4.0"
+files = [
+    {file = "testcontainers-4.8.2-py3-none-any.whl", hash = "sha256:9e19af077cd96e1957c13ee466f1f32905bc6c5bc1bc98643eb18be1a989bfb0"},
+    {file = "testcontainers-4.8.2.tar.gz", hash = "sha256:dd4a6a2ea09e3c3ecd39e180b6548105929d0bb78d665ce9919cb3f8c98f9853"},
+]
+
+[package.dependencies]
+docker = "*"
+typing-extensions = "*"
+urllib3 = "*"
+wrapt = "*"
+
+[package.extras]
+arangodb = ["python-arango (>=7.8,<8.0)"]
+aws = ["boto3", "httpx"]
+azurite = ["azure-storage-blob (>=12.19,<13.0)"]
+chroma = ["chromadb-client"]
+clickhouse = ["clickhouse-driver"]
+cosmosdb = ["azure-cosmos"]
+db2 = ["ibm_db_sa", "sqlalchemy"]
+generic = ["httpx", "redis"]
+google = ["google-cloud-datastore (>=2)", "google-cloud-pubsub (>=2)"]
+influxdb = ["influxdb", "influxdb-client"]
+k3s = ["kubernetes", "pyyaml"]
+keycloak = ["python-keycloak"]
+localstack = ["boto3"]
+mailpit = ["cryptography"]
+minio = ["minio"]
+mongodb = ["pymongo"]
+mssql = ["pymssql", "sqlalchemy"]
+mysql = ["pymysql[rsa]", "sqlalchemy"]
+nats = ["nats-py"]
+neo4j = ["neo4j"]
+opensearch = ["opensearch-py"]
+oracle = ["oracledb", "sqlalchemy"]
+oracle-free = ["oracledb", "sqlalchemy"]
+qdrant = ["qdrant-client"]
+rabbitmq = ["pika"]
+redis = ["redis"]
+registry = ["bcrypt"]
+scylla = ["cassandra-driver (==3.29.1)"]
+selenium = ["selenium"]
+sftp = ["cryptography"]
+test-module-import = ["httpx"]
+trino = ["trino"]
+weaviate = ["weaviate-client (>=4.5.4,<5.0.0)"]
+
+[package.source]
+type = "legacy"
+url = "http://192.168.1.2:5001/index"
+reference = "threadripper"
+
 [[package]]
 name = "torch"
 version = "2.2.2"
@@ -2730,6 +2846,90 @@ type = "legacy"
 url = "http://192.168.1.2:5001/index"
 reference = "threadripper"
 
+[[package]]
+name = "wrapt"
+version = "1.16.0"
+description = "Module for decorators, wrappers and monkey patching."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"},
+    {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"},
+    {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"},
+    {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"},
+    {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"},
+    {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"},
+    {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"},
+    {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"},
+    {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"},
+    {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"},
+    {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"},
+    {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"},
+    {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"},
+    {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"},
+    {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"},
+    {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"},
+    {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"},
+    {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"},
+    {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"},
+    {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"},
+    {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"},
+    {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"},
+    {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"},
+    {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"},
+    {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"},
+    {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"},
+    {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"},
+    {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"},
+    {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"},
+    {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"},
+    {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"},
+    {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"},
+    {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"},
+    {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"},
+    {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"},
+    {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"},
+    {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"},
+    {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"},
+    {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"},
+    {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"},
+    {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"},
+    {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"},
+    {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"},
+    {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"},
+    {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"},
+    {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"},
+    {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"},
+    {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"},
+    {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"},
+    {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"},
+    {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"},
+    {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"},
+    {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"},
+    {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"},
+    {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"},
+    {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"},
+    {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"},
+    {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"},
+    {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"},
+    {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"},
+    {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"},
+    {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"},
+    {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"},
+    {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"},
+    {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"},
+    {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"},
+    {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"},
+    {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"},
+    {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"},
+    {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"},
+]
+
+[package.source]
+type = "legacy"
+url = "http://192.168.1.2:5001/index"
+reference = "threadripper"
+
 [[package]]
 name = "wsproto"
 version = "1.2.0"
@@ -2772,4 +2972,4 @@ reference = "threadripper"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "33dbc73047d591ae63dd2d934519882654074a1f3c9ca757d0571d869b8541c2"
+content-hash = "c5e7d72d26658899174ec00fe2cb435b0e571efe4efe17988b0c94a80c893597"

pyproject.toml

@@ -22,6 +22,7 @@ flake8-pyproject = "^1.2.3"
 pandas-stubs = "^2.2.2.240603"
 types-tqdm = "^4.66.0.20240417"
 pytest = "^8.3.3"
+testcontainers = "^4.8.2"
 
 
 [tool.poetry.group.dev.dependencies]

shared/datastore/__init__.py

@@ -1,8 +1,10 @@
-from .connect_minio import connect_minio
+from .src import (
-from .delete import delete
+    connect_minio,
-from .get import get
+    delete,
-from .get_image import get_image
+    get,
-from .get_model import get_model
+    get_image,
-from .put import put
+    get_model,
-from .put_image import put_image
+    put,
-from .put_model import put_model
+    put_image,
+    put_model,
+)

shared/datastore/get_model.py

@@ -1,30 +0,0 @@
-"""Definition of get_model function."""
-
-import logging
-from collections import OrderedDict
-
-import torch
-from minio import Minio
-
-from .get import get
-
-
-def get_model(
-    client: Minio,
-    object_name: str,
-) -> OrderedDict:
-    """Get model from model subfolder in bucket in Minio."""
-    assert isinstance(client, Minio)
-    assert isinstance(object_name, str)
-    subfolder = 'models'
-    object_name = f'{subfolder}/{object_name}'
-    # get buffer
-    buffer = get(
-        client=client,
-        object_name=object_name,
-    )
-    # convert data to model checkpoint
-    buffer.seek(0)
-    model_content = torch.load(buffer)
-    logging.debug('finished')
-    return model_content

shared/datastore/src/__init__.py

@@ -0,0 +1,8 @@
+from .connect_minio import connect_minio
+from .delete import delete
+from .get import get
+from .get_image import get_image
+from .get_model import get_model
+from .put import put
+from .put_image import put_image
+from .put_model import put_model

shared/datastore/src/connect_minio.py

@@ -5,28 +5,24 @@ import os
 
 from minio import Minio
 
+from shared.utils import check_env
+
 
 def connect_minio() -> Minio:
     """Connect to MinIO server."""
     # ensure necessary env vars available
-    env_var_list = [
+    var_list = {
         'MINIO_ENDPOINT',
         'MINIO_ACCESS_KEY',
         'MINIO_SECRET_KEY',
         'MINIO_BUCKET_NAME',
-    ]
+    }
-    for env_var in env_var_list:
+    check_env(var_list)
-        # ensure env var set
-        assert (
-            env_var in os.environ
-        ), f"environment variable not set: {
-            env_var
-        }"
     # prepare arguments
-    minio_endpoint = os.getenv('MINIO_ENDPOINT', default='')
+    minio_endpoint = str(os.getenv('MINIO_ENDPOINT'))
-    minio_access_key = os.getenv('MINIO_ACCESS_KEY', default='')
+    minio_access_key = str(os.getenv('MINIO_ACCESS_KEY'))
-    minio_secret_key = os.getenv('MINIO_SECRET_KEY', default='')
+    minio_secret_key = str(os.getenv('MINIO_SECRET_KEY'))
-    minio_bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
+    minio_bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
     # connect client
     client = Minio(
         endpoint=minio_endpoint,

shared/datastore/src/get.py

@@ -1,7 +1,6 @@
 """Definition of get function."""
 
 import logging
-import os
 from io import BytesIO
 from traceback import print_exc
 
@@ -10,19 +9,22 @@ from minio import Minio
 
 def get(
     client: Minio,
+    bucket_name: str,
     object_name: str,
 ) -> BytesIO:
     """Get buffer from bucket in MinIO."""
     assert isinstance(client, Minio)
     assert isinstance(object_name, str)
-    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
+    assert isinstance(bucket_name, str)
-    # get buffer
+    assert len(bucket_name) > 0
     try:
+        # make request
         response = client.get_object(
             bucket_name=bucket_name,
             object_name=object_name,
         )
         assert response.status == 200
+        # get buffer
         buffer = BytesIO()
         chunk_size = 2**14
         while chunk := response.read(chunk_size):

shared/datastore/src/get_image.py

@@ -7,6 +7,8 @@ from io import BytesIO
 from minio import Minio
 from PIL import Image
 
+from shared.utils import check_env
+
 
 def get_image(
     client: Minio,
@@ -17,11 +19,10 @@ def get_image(
     assert isinstance(object_name, str)
     assert len(object_name) > 0
     # prepare arguments
-    assert 'MINIO_BUCKET_NAME' in os.environ
+    check_env({'MINIO_BUCKET_NAME'})
-    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
+    bucket_name = str(os.getenv('MINIO_BUCKET_NAME'))
-    subfolder = 'images'
+    object_name = f'images/{object_name}'
     # get object from bucket
-    object_name = f'{subfolder}/{object_name}'
     try:
         response = client.get_object(
             bucket_name=bucket_name,

shared/datastore/src/get_model.py

@@ -0,0 +1,43 @@
+"""Definition of get_model function."""
+
+import logging
+import os
+from collections import OrderedDict
+from io import BytesIO
+
+import torch
+from minio import Minio
+
+from shared.utils import check_env
+
+
+def get_model(
+    client: Minio,
+    object_name: str,
+) -> OrderedDict:
+    """Get model from model subfolder in bucket in Minio."""
+    assert isinstance(client, Minio)
+    assert isinstance(object_name, str)
+    assert len(object_name) > 0
+    # prepare arguments
+    check_env({'MINIO_BUCKET_NAME_MODELS'})
+    bucket_name = str(os.getenv('MINIO_BUCKET_NAME_MODELS'))
+    object_name = f'models/{object_name}'
+    # get object from bucket
+    try:
+        response = client.get_object(
+            bucket_name=bucket_name,
+            object_name=object_name,
+        )
+        buffer = BytesIO(response.data)
+    except Exception as exc:
+        logging.error('failed getting data from MinIO')
+        raise exc
+    finally:
+        response.close()
+        response.release_conn()
+    # convert data to model checkpoint
+    buffer.seek(0)
+    model_content = torch.load(buffer)
+    logging.debug('finished')
+    return model_content

shared/datastore/src/put.py

@@ -1,10 +1,6 @@
 """Definition of put function."""
 
-from __future__ import annotations
-
 import logging
-import os
-from hashlib import md5
 from io import BytesIO
 
 from minio import Minio
@@ -13,14 +9,16 @@ from minio import Minio
 def put(
     client: Minio,
     buffer: BytesIO,
-) -> str:
+    bucket_name: str,
+    object_name: str,
+) -> None:
     """Put buffer in bucket in MinIO and return MD5 checksum as object name."""
     assert isinstance(client, Minio)
     assert isinstance(buffer, BytesIO)
-    bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
+    assert isinstance(bucket_name, str)
     assert len(bucket_name) > 0
-    # get md5 of image
+    assert isinstance(object_name, str)
-    checksum = md5(buffer.getbuffer()).hexdigest()
+    assert len(object_name) > 0
     # prepare for saving
     num_bytes = buffer.tell()
     buffer.seek(0)
@@ -28,12 +26,11 @@ def put(
     try:
         client.put_object(
             bucket_name=bucket_name,
-            object_name=checksum,
+            object_name=object_name,
             length=num_bytes,
             data=buffer,
         )
     except Exception as exc:
         logging.error('failed saving data to MinIO')
         raise exc
-    logging.debug('saved data to %s', checksum)
+    logging.debug('saved data to %s', object_name)
-    return checksum

shared/datastore/src/put_image.py

@@ -8,6 +8,8 @@ from io import BytesIO
 from minio import Minio
 from PIL import Image
 
+from .put import put
+
 
 def put_image(
     client: Minio,
@@ -17,28 +19,23 @@ def put_image(
     used as object name."""
     assert isinstance(client, Minio)
     assert isinstance(image, Image.Image)
+    # get bucket name from env
     bucket_name = os.getenv('MINIO_BUCKET_NAME', default='')
     assert len(bucket_name) > 0
-    subfolder = 'images'
     # save image to buffer
     buffer = BytesIO()
     image.save(buffer, 'png')
-    # get md5 of image
+    # get md5 of buffer
     checksum = md5(buffer.getbuffer()).hexdigest()
-    # prepare for saving
+    # determine object name
-    num_bytes = buffer.tell()
+    subfolder = 'images'
-    buffer.seek(0)
-    # send data to bucket
     object_name = f'{subfolder}/{checksum}'
-    try:
+    # send data to bucket
-        client.put_object(
+    put(
-            bucket_name=bucket_name,
+        client=client,
-            object_name=object_name,
+        buffer=buffer,
-            length=num_bytes,
+        bucket_name=bucket_name,
-            data=buffer,
+        object_name=object_name,
-        )
+    )
-    except Exception as exc:
+    logging.debug('finished')
-        logging.error('failed saving data to MinIO')
-        raise exc
-    logging.debug('saved data to %s', object_name)
     return checksum

shared/datastore/tests/integration_test.py

@@ -0,0 +1,110 @@
+"""Integration tests for functions exposed by the datastore module."""
+
+import os
+from io import BytesIO
+from random import randbytes
+
+import pytest
+from minio import Minio
+from testcontainers.minio import MinioContainer
+
+from shared.datastore import connect_minio, get, put
+from shared.utils import setup_logging
+
+minio = MinioContainer()
+MINIO_BUCKET_NAME = 'visual-critical-discourse-analysis'
+
+
+@pytest.fixture(scope='module', autouse=True)
+def setup(request):
+    """Setup function for datastore module integration tests."""
+    # start minio
+    minio.start()
+
+    # ensure minio is stopped after testing
+    def remove_container():
+        minio.stop()
+
+    request.addfinalizer(remove_container)
+    # extract information
+    minio_ip = minio.get_container_host_ip()
+    minio_port = minio.get_exposed_port(port=9000)
+    os.environ['MINIO_ENDPOINT'] = f'{minio_ip}:{minio_port}'
+    os.environ['MINIO_ACCESS_KEY'] = minio.access_key
+    os.environ['MINIO_SECRET_KEY'] = minio.secret_key
+    os.environ['MINIO_BUCKET_NAME'] = MINIO_BUCKET_NAME
+
+
+def test_connect_minio():
+    """Test function connect_minio."""
+    minio_client = connect_minio()
+    assert isinstance(minio_client, Minio)
+
+
+def test_put():
+    """Test function put."""
+    # connect to minio
+    minio_client = connect_minio()
+    # generate random bytes
+    buffer = BytesIO(randbytes(2**20))
+    # put data in Minio
+    put(
+        client=minio_client,
+        buffer=buffer,
+        bucket_name=MINIO_BUCKET_NAME,
+        object_name='test_put_data',
+    )
+
+
+def test_get():
+    """Test function get."""
+    # connect to minio
+    minio_client = connect_minio()
+    # generate random bytes
+    buffer = BytesIO(randbytes(2**20))
+    # put data in minio
+    put(
+        client=minio_client,
+        buffer=buffer,
+        bucket_name=MINIO_BUCKET_NAME,
+        object_name='test_get_data',
+    )
+    # get data back from minio
+    buffer_rec = get(
+        client=minio_client,
+        bucket_name=MINIO_BUCKET_NAME,
+        object_name='test_get_data',
+    )
+    # check that data are the same
+    assert buffer.read() == buffer_rec.read()
+
+
+def test_delete():
+    """Test function delete."""
+    raise NotImplementedError()
+
+
+def test_get_image():
+    """Test function get_image."""
+    raise NotImplementedError()
+
+
+def test_put_image():
+    """Test function put_image."""
+    raise NotImplementedError()
+
+
+def test_get_model():
+    """Test function get_model."""
+    raise NotImplementedError()
+
+
+def test_put_model():
+    """Test function put_model."""
+    raise NotImplementedError()
+
+
+if __name__ == '__main__':
+    os.environ['LOG_LEVEL'] = 'debug'
+    setup_logging()
+    pytest.main()