Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fa4ff1f03c | ||
|
|
0a39151ba0 | ||
|
|
82a702db0c | ||
|
|
b886a7c147 | ||
|
|
4518a93a3a | ||
|
|
a8b7b59484 | ||
|
|
0e6941dccf | ||
|
|
a6428ccd29 | ||
|
|
655fdc4ee8 | ||
|
|
e031c549e5 | ||
|
|
49b6238b7d | ||
|
|
0549358fe1 | ||
|
|
33efbd1005 | ||
|
|
24126e4044 | ||
|
|
f3b6cdbc9e | ||
|
|
fa2554ef5f | ||
|
|
b21fd247c5 | ||
|
|
441a21204d | ||
|
|
f2a8581de2 | ||
|
|
3b05260323 | ||
|
|
369295b8ae | ||
|
|
95c78596e7 | ||
|
|
42e885edd5 | ||
|
|
bd4794d1c0 | ||
|
|
3dc412cd15 | ||
|
|
b8d88e6703 | ||
|
|
67bb88fbb4 |
@@ -0,0 +1,16 @@
|
|||||||
|
.git
|
||||||
|
.github
|
||||||
|
.gitea
|
||||||
|
.mypy_cache
|
||||||
|
.pytest_cache
|
||||||
|
.ruff_cache
|
||||||
|
.venv
|
||||||
|
__pycache__
|
||||||
|
*.pyc
|
||||||
|
.coverage
|
||||||
|
coverage-unit
|
||||||
|
coverage-integration
|
||||||
|
coverage.txt
|
||||||
|
dist
|
||||||
|
build
|
||||||
|
*.egg-info
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# Build and publish the per-repo CI base image to the Gitea container registry.
|
||||||
|
|
||||||
|
name: Build CI Image
|
||||||
|
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: "0 2 * * *"
|
||||||
|
workflow_dispatch:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
paths:
|
||||||
|
- pyproject.toml
|
||||||
|
- uv.lock
|
||||||
|
- docker/ci/**
|
||||||
|
|
||||||
|
env:
|
||||||
|
REGISTRY: gitea.lille-vemmelund.dk
|
||||||
|
REGISTRY_USER: ci-bot
|
||||||
|
IMAGE: gitea.lille-vemmelund.dk/lillevemmelund/python-repositories-ci
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
build-and-push:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Log in to Gitea container registry
|
||||||
|
env:
|
||||||
|
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: |
|
||||||
|
echo "$CI_RUNNER_TOKEN" | docker login "$REGISTRY" -u "$REGISTRY_USER" --password-stdin
|
||||||
|
|
||||||
|
- name: Build CI image
|
||||||
|
env:
|
||||||
|
CI_RUNNER_TOKEN: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: |
|
||||||
|
echo "$CI_RUNNER_TOKEN" > /tmp/uv_token
|
||||||
|
docker build --network=host -f docker/ci/Dockerfile \
|
||||||
|
--secret id=uv_token,src=/tmp/uv_token \
|
||||||
|
-t "${IMAGE}:latest" \
|
||||||
|
.
|
||||||
|
rm -f /tmp/uv_token
|
||||||
|
|
||||||
|
- name: Tag and push CI image
|
||||||
|
run: |
|
||||||
|
STAMP="$(date -u +%Y%m%d%H%M)"
|
||||||
|
docker tag "${IMAGE}:latest" "${IMAGE}:${STAMP}"
|
||||||
|
docker push "${IMAGE}:latest"
|
||||||
|
docker push "${IMAGE}:${STAMP}"
|
||||||
|
echo "Pushed ${IMAGE}:latest and ${IMAGE}:${STAMP}"
|
||||||
@@ -8,23 +8,17 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
code-quality:
|
code-quality:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Sync dependencies
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version-file: .python-version
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
env:
|
||||||
UV_LINK_MODE: copy
|
UV_LINK_MODE: copy
|
||||||
run: uv sync --all-extras
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
|
||||||
- name: Type check with mypy
|
- name: Type check with mypy
|
||||||
run: uv run mypy .
|
run: uv run mypy .
|
||||||
|
|||||||
@@ -7,22 +7,17 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
update-check:
|
update-check:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
token: ${{ secrets.CI_RUNNER_TOKEN }}
|
token: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version-file: .python-version
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Upgrade dependencies
|
- name: Upgrade dependencies
|
||||||
|
env:
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
run: uv lock --upgrade
|
run: uv lock --upgrade
|
||||||
|
|
||||||
- name: Commit and push changes
|
- name: Commit and push changes
|
||||||
|
|||||||
@@ -9,25 +9,24 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-and-publish:
|
build-and-publish:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version-file: .python-version
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Update version in pyproject.toml to match tag
|
- name: Update version in pyproject.toml to match tag
|
||||||
|
env:
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
run: scripts/ci/bump-version.sh --from-tag "${GITHUB_REF##*/}"
|
run: scripts/ci/bump-version.sh --from-tag "${GITHUB_REF##*/}"
|
||||||
|
|
||||||
- name: Build package
|
- name: Build package
|
||||||
|
env:
|
||||||
|
UV_LINK_MODE: copy
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
run: |
|
run: |
|
||||||
uv sync --no-dev
|
uv sync --no-dev --frozen
|
||||||
uv build
|
uv build
|
||||||
|
|
||||||
- name: Publish to Gitea Package Registry
|
- name: Publish to Gitea Package Registry
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ on:
|
|||||||
jobs:
|
jobs:
|
||||||
release:
|
release:
|
||||||
if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
|
if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
@@ -22,19 +22,12 @@ jobs:
|
|||||||
COMMIT_MSG: ${{ github.event.head_commit.message }}
|
COMMIT_MSG: ${{ github.event.head_commit.message }}
|
||||||
run: scripts/ci/parse-merge-commit.sh "$COMMIT_MSG"
|
run: scripts/ci/parse-merge-commit.sh "$COMMIT_MSG"
|
||||||
|
|
||||||
- name: Set up Python
|
|
||||||
if: steps.meta.outputs.bump != 'skip'
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version-file: .python-version
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
if: steps.meta.outputs.bump != 'skip'
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Bump version
|
- name: Bump version
|
||||||
if: steps.meta.outputs.bump != 'skip'
|
if: steps.meta.outputs.bump != 'skip'
|
||||||
id: bump
|
id: bump
|
||||||
|
env:
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
run: scripts/ci/bump-version.sh "${{ steps.meta.outputs.bump }}"
|
run: scripts/ci/bump-version.sh "${{ steps.meta.outputs.bump }}"
|
||||||
|
|
||||||
- name: Generate release notes
|
- name: Generate release notes
|
||||||
|
|||||||
@@ -7,23 +7,17 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
safety:
|
safety:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Sync dependencies
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version-file: .python-version
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
env:
|
||||||
UV_LINK_MODE: copy
|
UV_LINK_MODE: copy
|
||||||
run: uv sync
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --frozen
|
||||||
|
|
||||||
- name: Run safety check
|
- name: Run safety check
|
||||||
run: uv run safety check
|
run: uv run safety check
|
||||||
|
|||||||
+15
-33
@@ -8,23 +8,17 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
unit-tests:
|
unit-tests:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Sync dependencies
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version-file: .python-version
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
env:
|
||||||
UV_LINK_MODE: copy
|
UV_LINK_MODE: copy
|
||||||
run: uv sync --all-extras
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
|
||||||
- name: Run unit tests
|
- name: Run unit tests
|
||||||
run: |
|
run: |
|
||||||
@@ -43,23 +37,17 @@ jobs:
|
|||||||
compression-level: 0
|
compression-level: 0
|
||||||
|
|
||||||
integration-tests:
|
integration-tests:
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Sync dependencies
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version-file: .python-version
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
env:
|
||||||
UV_LINK_MODE: copy
|
UV_LINK_MODE: copy
|
||||||
run: uv sync --all-extras
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
|
||||||
- name: Verify Docker
|
- name: Verify Docker
|
||||||
run: docker info
|
run: docker info
|
||||||
@@ -83,24 +71,18 @@ jobs:
|
|||||||
coverage-report:
|
coverage-report:
|
||||||
# Merges unit + integration coverage and enforces fail_under from pyproject.toml.
|
# Merges unit + integration coverage and enforces fail_under from pyproject.toml.
|
||||||
needs: [unit-tests, integration-tests]
|
needs: [unit-tests, integration-tests]
|
||||||
runs-on: ubuntu-latest
|
runs-on: python-repositories-ci
|
||||||
if: github.event_name == 'pull_request' || github.event_name == 'push'
|
if: github.event_name == 'pull_request' || github.event_name == 'push'
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Python
|
- name: Sync dependencies
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version-file: .python-version
|
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: pip install uv
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
env:
|
env:
|
||||||
UV_LINK_MODE: copy
|
UV_LINK_MODE: copy
|
||||||
run: uv sync --all-extras
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
|
||||||
- name: Download unit coverage
|
- name: Download unit coverage
|
||||||
uses: https://github.com/christopherHX/gitea-download-artifact@v4
|
uses: https://github.com/christopherHX/gitea-download-artifact@v4
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ Subclass an adapter in your own repository to add domain-specific methods while
|
|||||||
| **Adapters** | Technology-specific base classes (`RedisAdapter`, `MinioAdapter`) |
|
| **Adapters** | Technology-specific base classes (`RedisAdapter`, `MinioAdapter`) |
|
||||||
| **Your project** | Subclass an adapter and add domain methods |
|
| **Your project** | Subclass an adapter and add domain methods |
|
||||||
|
|
||||||
Connection adapters expose `connect()`, `disconnect()`, and `is_connected()`. The latter verifies backend reachability with a cached health probe (default TTL: 1 second). Subclasses may override `health_check_ttl_seconds`.
|
Connection adapters expose `connect()`, `disconnect()`, and `is_connected()`. The latter verifies backend reachability with a cached health probe (default TTL: 1 second). Subclasses may override `health_check_ttl_seconds`. `connect()` is idempotent: calling it while already connected and healthy is a no-op.
|
||||||
|
|
||||||
## Optional dependencies
|
## Optional dependencies
|
||||||
|
|
||||||
@@ -123,16 +123,18 @@ with ArtifactObjectRepository() as repo:
|
|||||||
### Subclassing in your own project
|
### Subclassing in your own project
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from python_repositories import RedisAdapter
|
from python_repositories import RedisAdapter
|
||||||
|
|
||||||
class UserRepository(RedisAdapter):
|
class UserRepository(RedisAdapter):
|
||||||
def _key(self, user_id: str) -> str:
|
def _key(self, user_id: str) -> str:
|
||||||
return f"user:{user_id}"
|
return f"user:{user_id}"
|
||||||
|
|
||||||
def get_user(self, user_id: str) -> dict | None:
|
def get_user(self, user_id: str) -> dict[str, Any] | None:
|
||||||
return self.get(self._key(user_id))
|
return self.get(self._key(user_id))
|
||||||
|
|
||||||
def save_user(self, user_id: str, user: dict) -> None:
|
def save_user(self, user_id: str, user: dict[str, Any]) -> None:
|
||||||
self.set(self._key(user_id), user)
|
self.set(self._key(user_id), user)
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -164,7 +166,14 @@ uv run pytest -v # full suite (requires Doc
|
|||||||
|
|
||||||
Integration tests are marked with `@pytest.mark.integration` and require Docker (testcontainers). Run unit tests alone for quick local feedback.
|
Integration tests are marked with `@pytest.mark.integration` and require Docker (testcontainers). Run unit tests alone for quick local feedback.
|
||||||
|
|
||||||
CI runs unit and integration tests in parallel with coverage, then merges `.coverage` artifacts in a follow-up job (via [christopherhx/gitea-\*-artifact@v4](https://github.com/christopherHX/gitea-upload-artifact) for Gitea 1.26 compatibility). Combined coverage must be at least **90%**; the floor is set by [`fail_under` in `pyproject.toml`](pyproject.toml#L45-L48) and enforced after merging unit and integration coverage, not on unit-only runs.
|
### CI base image
|
||||||
|
|
||||||
|
Gitea Actions jobs use a pre-built image (`python-repositories-ci`) with Python 3.12,
|
||||||
|
`uv`, and locked dependencies baked in. The image is rebuilt nightly and when
|
||||||
|
`uv.lock` changes; see [`docs/ci-image.md`](docs/ci-image.md) for runner setup and
|
||||||
|
bootstrap order.
|
||||||
|
|
||||||
|
CI runs unit and integration tests in parallel with coverage, then merges `.coverage` artifacts in a follow-up job (via [christopherhx/gitea-\*-artifact@v4](https://github.com/christopherHX/gitea-upload-artifact) for Gitea 1.26 compatibility). Combined coverage must meet the floor in [`fail_under` in `pyproject.toml`](pyproject.toml#L45-L48); enforcement happens after merging unit and integration coverage, not on unit-only runs.
|
||||||
|
|
||||||
To check coverage locally (requires Docker for the full suite):
|
To check coverage locally (requires Docker for the full suite):
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
# CI job image for Gitea Actions (act_runner requires Node.js in job containers).
|
||||||
|
FROM node:20-bookworm
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
ca-certificates \
|
||||||
|
curl \
|
||||||
|
git \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=docker:27-cli /usr/local/bin/docker /usr/local/bin/docker
|
||||||
|
COPY --from=ghcr.io/astral-sh/uv:0.7.0 /uv /usr/local/bin/uv
|
||||||
|
|
||||||
|
ENV UV_PYTHON_INSTALL_DIR=/opt/uv-python \
|
||||||
|
UV_LINK_MODE=copy
|
||||||
|
|
||||||
|
COPY .python-version /tmp/.python-version
|
||||||
|
RUN uv python install "$(cat /tmp/.python-version)"
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY pyproject.toml uv.lock .python-version ./
|
||||||
|
|
||||||
|
ENV PATH="/app/.venv/bin:${PATH}"
|
||||||
|
|
||||||
|
RUN --mount=type=secret,id=uv_token \
|
||||||
|
UV_INDEX_GITEA_USERNAME=ci-bot \
|
||||||
|
UV_INDEX_GITEA_PASSWORD="$(cat /run/secrets/uv_token)" \
|
||||||
|
uv sync --all-extras --frozen --no-install-project
|
||||||
@@ -0,0 +1,169 @@
|
|||||||
|
# CI base image
|
||||||
|
|
||||||
|
This repository uses a **per-repo Docker image** for Gitea Actions jobs instead of
|
||||||
|
installing Python and `uv` on every run. Infrastructure images (redis, minio, etc.)
|
||||||
|
are cached cluster-wide via Harbor (see homelab-platform
|
||||||
|
[`docs/harbor-registry-mirror.md`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/docs/harbor-registry-mirror.md)).
|
||||||
|
|
||||||
|
## Image contents
|
||||||
|
|
||||||
|
Built from [`docker/ci/Dockerfile`](../docker/ci/Dockerfile):
|
||||||
|
|
||||||
|
- Node.js 20 (required by act_runner job containers)
|
||||||
|
- Python 3.12 (installed via `uv python install` from [`.python-version`](../.python-version)) and pinned `uv` 0.7.0
|
||||||
|
- Docker CLI (integration tests via testcontainers)
|
||||||
|
- Dev dependencies from `uv.lock` (`uv sync --all-extras --no-install-project`)
|
||||||
|
|
||||||
|
Published to the Gitea container registry (owner segment must be lowercase for Docker):
|
||||||
|
|
||||||
|
- `gitea.lille-vemmelund.dk/lillevemmelund/python-repositories-ci:latest`
|
||||||
|
- `gitea.lille-vemmelund.dk/lillevemmelund/python-repositories-ci:YYYYMMDDHHmm` (timestamped rollback tag)
|
||||||
|
|
||||||
|
## Rebuild triggers
|
||||||
|
|
||||||
|
[`ci-image.yml`](../.gitea/workflows/ci-image.yml) runs on:
|
||||||
|
|
||||||
|
- Nightly cron (`0 2 * * *` UTC)
|
||||||
|
- Manual `workflow_dispatch`
|
||||||
|
- Push to `main` when `pyproject.toml`, `uv.lock`, or `docker/ci/**` change
|
||||||
|
|
||||||
|
## Workflow usage
|
||||||
|
|
||||||
|
Python jobs use `runs-on: python-repositories-ci` and a fast incremental sync:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
runs-on: python-repositories-ci
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- name: Sync dependencies
|
||||||
|
env:
|
||||||
|
UV_LINK_MODE: copy
|
||||||
|
UV_INDEX_GITEA_USERNAME: ci-bot
|
||||||
|
UV_INDEX_GITEA_PASSWORD: ${{ secrets.CI_RUNNER_TOKEN }}
|
||||||
|
run: uv sync --all-extras --frozen
|
||||||
|
```
|
||||||
|
|
||||||
|
`ci-image.yml` uses `runs-on: ubuntu-latest` so it can bootstrap before the custom
|
||||||
|
image exists.
|
||||||
|
|
||||||
|
## Registry authentication
|
||||||
|
|
||||||
|
act_runner pulls the job image **before any workflow step runs**, so a `docker
|
||||||
|
login` step inside a job cannot authenticate that pull. Authentication must be
|
||||||
|
configured on the runner (or via a job-level `container.credentials` block — not
|
||||||
|
used here).
|
||||||
|
|
||||||
|
### k8s Gitea runners (automatic)
|
||||||
|
|
||||||
|
The homelab-platform runners mount Gitea registry credentials automatically:
|
||||||
|
|
||||||
|
- Secret: `gitea-runners/gitea-registry-dockerconfig` (created by
|
||||||
|
[`scripts/create-gitea-registry-secret.sh`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/scripts/create-gitea-registry-secret.sh))
|
||||||
|
- Mounted at `/root/.docker/config.json` on the **runner** container (not DinD)
|
||||||
|
- Configured in
|
||||||
|
[`platform/gitea-runners/values.yaml`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/platform/gitea-runners/values.yaml)
|
||||||
|
|
||||||
|
No manual `docker login` is required on the in-cluster runners once the secret
|
||||||
|
exists. To create or rotate credentials:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export KUBECONFIG=/path/to/homelab-cluster/talos/_out/kubeconfig
|
||||||
|
export GITEA_REGISTRY_USERNAME='ci-bot'
|
||||||
|
export GITEA_REGISTRY_PASSWORD='personal-access-token-with-read-package'
|
||||||
|
bash scripts/create-gitea-registry-secret.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Then restart runner pods so they pick up the updated secret. See homelab-platform
|
||||||
|
[`docs/gitea-actions-runners.md`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/docs/gitea-actions-runners.md)
|
||||||
|
for full runner setup.
|
||||||
|
|
||||||
|
Verify on a running pod:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
kubectl -n gitea-runners exec homelab-cluster-gitea-runner-0 -c runner -- \
|
||||||
|
test -f /root/.docker/config.json && echo "registry auth mounted"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Standalone runners (e.g. Unraid `homelab`)
|
||||||
|
|
||||||
|
The 4th runner is outside the k8s cluster and does **not** get the automatic
|
||||||
|
mount. Configure registry auth manually on that host:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker login gitea.lille-vemmelund.dk -u ci-bot -p <token>
|
||||||
|
```
|
||||||
|
|
||||||
|
Also add the `python-repositories-ci` label to that runner's act_runner config.
|
||||||
|
|
||||||
|
### `ci-image.yml` push login
|
||||||
|
|
||||||
|
The build workflow still runs `docker login` before `docker push`. That step
|
||||||
|
authenticates **DinD inside the job** for pushing the image to Gitea — a different
|
||||||
|
code path from act_runner pulling the job container.
|
||||||
|
|
||||||
|
## Runner label
|
||||||
|
|
||||||
|
Jobs use the `python-repositories-ci` act_runner label, configured in
|
||||||
|
homelab-platform
|
||||||
|
[`platform/gitea-runners/values.yaml`](https://gitea.lille-vemmelund.dk/LilleVemmelund/homelab-platform/src/branch/main/platform/gitea-runners/values.yaml):
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
python-repositories-ci:docker://gitea.lille-vemmelund.dk/lillevemmelund/python-repositories-ci:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
After label changes, roll runner pods so they re-register with Gitea.
|
||||||
|
|
||||||
|
## Bootstrap order
|
||||||
|
|
||||||
|
1. Ensure homelab-platform runners have `gitea-registry-dockerconfig` and the
|
||||||
|
`python-repositories-ci` label (see homelab-platform docs).
|
||||||
|
2. Merge `ci-image.yml`, `docker/ci/Dockerfile`, and workflow migrations to `main`.
|
||||||
|
3. Seed the registry with a first image (see below).
|
||||||
|
4. Confirm a test workflow job starts on `python-repositories-ci`.
|
||||||
|
|
||||||
|
Until step 3 completes, jobs targeting `python-repositories-ci` will fail because
|
||||||
|
the image does not exist in the Gitea registry yet.
|
||||||
|
|
||||||
|
### Seed the image
|
||||||
|
|
||||||
|
**After merge to `main`:** Actions → **Build CI Image** → **Run workflow**.
|
||||||
|
|
||||||
|
**Before merge (e.g. PR branch):** the workflow file is not on `main` yet — build
|
||||||
|
and push locally with [`scripts/ci/build-ci-image.sh`](../scripts/ci/build-ci-image.sh):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||||
|
bash scripts/ci/build-ci-image.sh --push
|
||||||
|
```
|
||||||
|
|
||||||
|
Run from the repo root on the branch you want to test. Runners pull
|
||||||
|
`gitea.lille-vemmelund.dk/lillevemmelund/python-repositories-ci:latest` from the registry;
|
||||||
|
they do not care which git branch built it.
|
||||||
|
|
||||||
|
Override registry settings if needed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export REGISTRY=gitea.lille-vemmelund.dk
|
||||||
|
export REGISTRY_USER=ci-bot
|
||||||
|
export CI_RUNNER_TOKEN='...'
|
||||||
|
bash scripts/ci/build-ci-image.sh --push
|
||||||
|
```
|
||||||
|
|
||||||
|
## Local build
|
||||||
|
|
||||||
|
Build only (no registry login or push):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||||
|
bash scripts/ci/build-ci-image.sh --local
|
||||||
|
```
|
||||||
|
|
||||||
|
Or manually:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
echo "$CI_RUNNER_TOKEN" > /tmp/uv_token
|
||||||
|
docker build -f docker/ci/Dockerfile \
|
||||||
|
--secret id=uv_token,src=/tmp/uv_token \
|
||||||
|
-t python-repositories-ci:local .
|
||||||
|
rm -f /tmp/uv_token
|
||||||
|
```
|
||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "python-repositories"
|
name = "python-repositories"
|
||||||
version = "2.0.0"
|
version = "2.0.3"
|
||||||
description = "Various python repository interfaces exposed as a python package."
|
description = "Various python repository interfaces exposed as a python package."
|
||||||
authors = [
|
authors = [
|
||||||
{ name = "Brian Bjarke Jensen", email = "schnitzelen@gmail.com" }
|
{ name = "Brian Bjarke Jensen", email = "schnitzelen@gmail.com" }
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ class ConnectionAwareAdapter(ConnectionAwareInterface, ContextAwareInterface):
|
|||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
self.logger = structlog.get_logger(self.__class__.__name__)
|
self.logger = structlog.get_logger(self.__class__.__name__)
|
||||||
|
self._client_injected = False
|
||||||
self._health_check_at: float | None = None
|
self._health_check_at: float | None = None
|
||||||
self._health_check_ok: bool = False
|
self._health_check_ok: bool = False
|
||||||
|
|
||||||
@@ -57,6 +58,27 @@ class ConnectionAwareAdapter(ConnectionAwareInterface, ContextAwareInterface):
|
|||||||
def _probe_connection(self) -> bool:
|
def _probe_connection(self) -> bool:
|
||||||
"""Backend-specific liveness check; called only when client is ready."""
|
"""Backend-specific liveness check; called only when client is ready."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def _validate_injected_client(self) -> None:
|
||||||
|
"""Verify an injected client is reachable; raise ConnectionError on failure."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def _establish_connection(self) -> None:
|
||||||
|
"""Create a backend client and set internal connection state."""
|
||||||
|
|
||||||
|
def connect(self) -> None:
|
||||||
|
"""Connect to the backend; idempotent when already connected and healthy."""
|
||||||
|
if self._client_injected:
|
||||||
|
self._validate_injected_client()
|
||||||
|
self._invalidate_health_cache()
|
||||||
|
return
|
||||||
|
if self._is_client_ready() and self.is_connected():
|
||||||
|
self.logger.info("Already connected", connection_name=self.connection_name)
|
||||||
|
return
|
||||||
|
self.disconnect()
|
||||||
|
self._establish_connection()
|
||||||
|
self._invalidate_health_cache()
|
||||||
|
|
||||||
def is_connected(self) -> bool:
|
def is_connected(self) -> bool:
|
||||||
"""Check if connected to the backend."""
|
"""Check if connected to the backend."""
|
||||||
if not self._is_client_ready():
|
if not self._is_client_ready():
|
||||||
|
|||||||
@@ -58,23 +58,17 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
def _is_client_ready(self) -> bool:
|
def _is_client_ready(self) -> bool:
|
||||||
return self._client is not None and self._bucket_name is not None
|
return self._client is not None and self._bucket_name is not None
|
||||||
|
|
||||||
def connect(self) -> None:
|
def _validate_injected_client(self) -> None:
|
||||||
"""Connect to the Minio server."""
|
if self._client is None:
|
||||||
if self._client_injected:
|
|
||||||
if self._client is not None:
|
|
||||||
try:
|
|
||||||
_ = self._client.list_buckets()
|
|
||||||
except Exception as exc: # pylint: disable=broad-except
|
|
||||||
raise ConnectionError(
|
|
||||||
f"Could not connect to Minio at {self._config.endpoint}"
|
|
||||||
) from exc
|
|
||||||
self._invalidate_health_cache()
|
|
||||||
return
|
return
|
||||||
if self._client is not None and self.is_connected():
|
try:
|
||||||
self.logger.info("Already connected to Minio")
|
_ = self._client.list_buckets()
|
||||||
return
|
except Exception as exc: # pylint: disable=broad-except
|
||||||
if self._client is not None:
|
raise ConnectionError(
|
||||||
self.disconnect()
|
f"Could not connect to Minio at {self._config.endpoint}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def _establish_connection(self) -> None:
|
||||||
endpoint = self._config.endpoint
|
endpoint = self._config.endpoint
|
||||||
access_key = self._config.access_key
|
access_key = self._config.access_key
|
||||||
secret_key = self._config.secret_key
|
secret_key = self._config.secret_key
|
||||||
@@ -94,11 +88,10 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
raise ConnectionError(
|
raise ConnectionError(
|
||||||
f"Bucket '{bucket}' does not exist on Minio at {endpoint}"
|
f"Bucket '{bucket}' does not exist on Minio at {endpoint}"
|
||||||
)
|
)
|
||||||
self.logger.info(f"Creating bucket '{bucket}'")
|
self.logger.info("Creating bucket", bucket=bucket)
|
||||||
client.make_bucket(bucket)
|
client.make_bucket(bucket)
|
||||||
self._client = client
|
self._client = client
|
||||||
self._bucket_name = bucket
|
self._bucket_name = bucket
|
||||||
self._invalidate_health_cache()
|
|
||||||
|
|
||||||
def disconnect(self) -> None:
|
def disconnect(self) -> None:
|
||||||
"""Disconnect from the Minio server."""
|
"""Disconnect from the Minio server."""
|
||||||
@@ -145,7 +138,7 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
content_type=content_type,
|
content_type=content_type,
|
||||||
)
|
)
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Put object '{object_name}' into bucket '{self._bucket_name}'"
|
"Put object", object_name=object_name, bucket=self._bucket_name
|
||||||
)
|
)
|
||||||
|
|
||||||
def get(self, object_name: str) -> BytesIO | None:
|
def get(self, object_name: str) -> BytesIO | None:
|
||||||
@@ -170,13 +163,15 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
buffer.write(chunk)
|
buffer.write(chunk)
|
||||||
buffer.seek(0)
|
buffer.seek(0)
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Got object '{object_name}' from bucket '{self._bucket_name}'"
|
"Got object", object_name=object_name, bucket=self._bucket_name
|
||||||
)
|
)
|
||||||
return buffer
|
return buffer
|
||||||
except minio.S3Error as exc:
|
except minio.S3Error as exc:
|
||||||
if exc.code == "NoSuchKey":
|
if exc.code == "NoSuchKey":
|
||||||
self.logger.warning(
|
self.logger.warning(
|
||||||
f"Object '{object_name}' not found in bucket '{self._bucket_name}'"
|
"Object not found",
|
||||||
|
object_name=object_name,
|
||||||
|
bucket=self._bucket_name,
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
raise
|
raise
|
||||||
@@ -200,7 +195,7 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
object_name=object_name,
|
object_name=object_name,
|
||||||
)
|
)
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Deleted object '{object_name}' from bucket '{self._bucket_name}'"
|
"Deleted object", object_name=object_name, bucket=self._bucket_name
|
||||||
)
|
)
|
||||||
|
|
||||||
def list_objects(self, prefix: str = "") -> list[str]:
|
def list_objects(self, prefix: str = "") -> list[str]:
|
||||||
@@ -222,6 +217,9 @@ class MinioAdapter(ObjectRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
obj.object_name for obj in objects if obj.object_name is not None
|
obj.object_name for obj in objects if obj.object_name is not None
|
||||||
]
|
]
|
||||||
self.logger.debug(
|
self.logger.debug(
|
||||||
f"Listed {len(object_names)} object(s) in bucket '{self._bucket_name}' with prefix '{prefix}'"
|
"Listed objects",
|
||||||
|
count=len(object_names),
|
||||||
|
bucket=self._bucket_name,
|
||||||
|
prefix=prefix,
|
||||||
)
|
)
|
||||||
return object_names
|
return object_names
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from typing import cast
|
from typing import Any, cast
|
||||||
|
|
||||||
from python_repositories.adapters.connection_aware_adapter import (
|
from python_repositories.adapters.connection_aware_adapter import (
|
||||||
ConnectionAwareAdapter,
|
ConnectionAwareAdapter,
|
||||||
@@ -49,25 +49,20 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
def _is_client_ready(self) -> bool:
|
def _is_client_ready(self) -> bool:
|
||||||
return self._client is not None
|
return self._client is not None
|
||||||
|
|
||||||
def connect(self) -> None:
|
def _validate_injected_client(self) -> None:
|
||||||
"""Connect to the Redis server."""
|
if self._client is None:
|
||||||
if self._client_injected:
|
|
||||||
if self._client is not None:
|
|
||||||
try:
|
|
||||||
if not self._client.ping():
|
|
||||||
raise ConnectionError(
|
|
||||||
f"Could not connect to Redis at {self._config.uri}"
|
|
||||||
)
|
|
||||||
except (redis.ConnectionError, redis.TimeoutError) as exc:
|
|
||||||
raise ConnectionError(
|
|
||||||
f"Could not connect to Redis at {self._config.uri}"
|
|
||||||
) from exc
|
|
||||||
self._invalidate_health_cache()
|
|
||||||
return
|
return
|
||||||
if self._client is not None:
|
try:
|
||||||
self._client.close()
|
if not self._client.ping():
|
||||||
self._client = None
|
raise ConnectionError(
|
||||||
self._invalidate_health_cache()
|
f"Could not connect to Redis at {self._config.uri}"
|
||||||
|
)
|
||||||
|
except (redis.ConnectionError, redis.TimeoutError) as exc:
|
||||||
|
raise ConnectionError(
|
||||||
|
f"Could not connect to Redis at {self._config.uri}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def _establish_connection(self) -> None:
|
||||||
uri = self._config.uri
|
uri = self._config.uri
|
||||||
try:
|
try:
|
||||||
client = redis.Redis.from_url(
|
client = redis.Redis.from_url(
|
||||||
@@ -79,7 +74,6 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
except (redis.ConnectionError, redis.TimeoutError) as exc:
|
except (redis.ConnectionError, redis.TimeoutError) as exc:
|
||||||
raise ConnectionError(f"Could not connect to Redis at {uri}") from exc
|
raise ConnectionError(f"Could not connect to Redis at {uri}") from exc
|
||||||
self._client = client
|
self._client = client
|
||||||
self._invalidate_health_cache()
|
|
||||||
|
|
||||||
def disconnect(self) -> None:
|
def disconnect(self) -> None:
|
||||||
"""Disconnect from the Redis server."""
|
"""Disconnect from the Redis server."""
|
||||||
@@ -95,7 +89,7 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
except (redis.ConnectionError, redis.TimeoutError):
|
except (redis.ConnectionError, redis.TimeoutError):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def set(self, key: str, data: dict) -> None:
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
"""Set a JSON object in Redis."""
|
"""Set a JSON object in Redis."""
|
||||||
# Check input
|
# Check input
|
||||||
if not isinstance(key, str) or len(key) == 0:
|
if not isinstance(key, str) or len(key) == 0:
|
||||||
@@ -107,9 +101,9 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
assert self._client is not None
|
assert self._client is not None
|
||||||
# Set data
|
# Set data
|
||||||
self._client.json().set(key, self.path, data)
|
self._client.json().set(key, self.path, data)
|
||||||
self.logger.debug(f"Set {key} to {data}")
|
self.logger.debug("Set key", key=key, data_keys=list(data.keys()))
|
||||||
|
|
||||||
def get(self, key: str) -> dict | None:
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
"""Get a JSON object from Redis."""
|
"""Get a JSON object from Redis."""
|
||||||
# Check input
|
# Check input
|
||||||
if not isinstance(key, str) or len(key) == 0:
|
if not isinstance(key, str) or len(key) == 0:
|
||||||
@@ -119,10 +113,10 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
assert self._client is not None
|
assert self._client is not None
|
||||||
# Get data
|
# Get data
|
||||||
data = cast(
|
data = cast(
|
||||||
dict | None,
|
dict[str, Any] | None,
|
||||||
self._client.json().get(key),
|
self._client.json().get(key),
|
||||||
)
|
)
|
||||||
self.logger.debug(f"Got {data} from {key}")
|
self.logger.debug("Got value", key=key, found=data is not None)
|
||||||
return data
|
return data
|
||||||
|
|
||||||
def delete(self, key: str) -> None:
|
def delete(self, key: str) -> None:
|
||||||
@@ -135,7 +129,7 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
assert self._client is not None
|
assert self._client is not None
|
||||||
# Delete data
|
# Delete data
|
||||||
self._client.json().delete(key)
|
self._client.json().delete(key)
|
||||||
self.logger.debug(f"Deleted {key}")
|
self.logger.debug("Deleted key", key=key)
|
||||||
|
|
||||||
def _validate_pattern(self, pattern: str) -> None:
|
def _validate_pattern(self, pattern: str) -> None:
|
||||||
if not isinstance(pattern, str) or len(pattern) == 0:
|
if not isinstance(pattern, str) or len(pattern) == 0:
|
||||||
@@ -153,7 +147,7 @@ class RedisAdapter(JsonRepositoryInterface, ConnectionAwareAdapter):
|
|||||||
self._client.keys(pattern),
|
self._client.keys(pattern),
|
||||||
)
|
)
|
||||||
keys: list[str] = [key.decode(self.encoding) for key in keys_raw]
|
keys: list[str] = [key.decode(self.encoding) for key in keys_raw]
|
||||||
self.logger.debug(f"Got {keys} matching {pattern}")
|
self.logger.debug("Listed keys", pattern=pattern, count=len(keys))
|
||||||
return keys
|
return keys
|
||||||
|
|
||||||
def scan_keys(
|
def scan_keys(
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
"""Example domain repository backed by Redis JSON."""
|
"""Example domain repository backed by Redis JSON."""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from python_repositories.adapters.redis_adapter import RedisAdapter
|
from python_repositories.adapters.redis_adapter import RedisAdapter
|
||||||
|
|
||||||
|
|
||||||
@@ -9,10 +11,10 @@ class UserJsonRepository(RedisAdapter):
|
|||||||
def _key(self, user_id: str) -> str:
|
def _key(self, user_id: str) -> str:
|
||||||
return f"user:{user_id}"
|
return f"user:{user_id}"
|
||||||
|
|
||||||
def get_user(self, user_id: str) -> dict | None:
|
def get_user(self, user_id: str) -> dict[str, Any] | None:
|
||||||
return self.get(self._key(user_id))
|
return self.get(self._key(user_id))
|
||||||
|
|
||||||
def save_user(self, user_id: str, user: dict) -> None:
|
def save_user(self, user_id: str, user: dict[str, Any]) -> None:
|
||||||
self.set(self._key(user_id), user)
|
self.set(self._key(user_id), user)
|
||||||
|
|
||||||
def delete_user(self, user_id: str) -> None:
|
def delete_user(self, user_id: str) -> None:
|
||||||
|
|||||||
@@ -8,7 +8,11 @@ class ConnectionAwareInterface(ABC):
|
|||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def connect(self) -> None:
|
def connect(self) -> None:
|
||||||
"""Connect to resource."""
|
"""Connect to resource.
|
||||||
|
|
||||||
|
Implementations should be idempotent: calling connect while already
|
||||||
|
connected and healthy is a no-op.
|
||||||
|
"""
|
||||||
...
|
...
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
|
|||||||
@@ -2,18 +2,19 @@
|
|||||||
|
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
class JsonRepositoryInterface(ABC):
|
class JsonRepositoryInterface(ABC):
|
||||||
"""Interface that defines JSON document CRUD methods."""
|
"""Interface that defines JSON document CRUD methods."""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get(self, key: str) -> dict | None:
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
"""Get a JSON object by key."""
|
"""Get a JSON object by key."""
|
||||||
...
|
...
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def set(self, key: str, data: dict) -> None:
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
"""Set a JSON object by key."""
|
"""Set a JSON object by key."""
|
||||||
...
|
...
|
||||||
|
|
||||||
|
|||||||
Executable
+78
@@ -0,0 +1,78 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build and optionally push the Gitea Actions CI base image locally.
|
||||||
|
#
|
||||||
|
# Use when ci-image.yml is not yet on main (e.g. bootstrapping a PR branch) or
|
||||||
|
# when you need to rebuild without waiting for the nightly workflow.
|
||||||
|
#
|
||||||
|
# Build only (smoke test):
|
||||||
|
# export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||||
|
# bash scripts/ci/build-ci-image.sh --local
|
||||||
|
#
|
||||||
|
# Build and push to Gitea (unblocks python-repositories-ci jobs):
|
||||||
|
# export CI_RUNNER_TOKEN='ci-bot-personal-access-token'
|
||||||
|
# bash scripts/ci/build-ci-image.sh --push
|
||||||
|
#
|
||||||
|
# Optional overrides:
|
||||||
|
# REGISTRY=gitea.lille-vemmelund.dk
|
||||||
|
# REGISTRY_USER=ci-bot
|
||||||
|
# IMAGE_OWNER=lillevemmelund
|
||||||
|
# IMAGE_NAME=python-repositories-ci
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||||
|
|
||||||
|
REGISTRY="${REGISTRY:-gitea.lille-vemmelund.dk}"
|
||||||
|
REGISTRY_USER="${REGISTRY_USER:-ci-bot}"
|
||||||
|
IMAGE_OWNER="${IMAGE_OWNER:-lillevemmelund}"
|
||||||
|
IMAGE_NAME="${IMAGE_NAME:-python-repositories-ci}"
|
||||||
|
IMAGE="${IMAGE:-${REGISTRY}/${IMAGE_OWNER}/${IMAGE_NAME}}"
|
||||||
|
|
||||||
|
MODE="push"
|
||||||
|
if [[ "${1:-}" == "--local" ]]; then
|
||||||
|
MODE="local"
|
||||||
|
elif [[ "${1:-}" == "--push" || -z "${1:-}" ]]; then
|
||||||
|
MODE="push"
|
||||||
|
elif [[ -n "${1:-}" ]]; then
|
||||||
|
echo "Usage: build-ci-image.sh [--local | --push]" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
: "${CI_RUNNER_TOKEN:?CI_RUNNER_TOKEN is required (ci-bot token with read/write package access)}"
|
||||||
|
|
||||||
|
TOKEN_FILE="$(mktemp)"
|
||||||
|
cleanup() {
|
||||||
|
rm -f "$TOKEN_FILE"
|
||||||
|
}
|
||||||
|
trap cleanup EXIT
|
||||||
|
|
||||||
|
printf '%s' "$CI_RUNNER_TOKEN" >"$TOKEN_FILE"
|
||||||
|
|
||||||
|
cd "$REPO_ROOT"
|
||||||
|
|
||||||
|
if [[ "$MODE" == "local" ]]; then
|
||||||
|
echo "=== Building local CI image (no push): python-repositories-ci:local ==="
|
||||||
|
docker build --network=host -f docker/ci/Dockerfile \
|
||||||
|
--secret "id=uv_token,src=${TOKEN_FILE}" \
|
||||||
|
-t python-repositories-ci:local \
|
||||||
|
.
|
||||||
|
echo "Built python-repositories-ci:local"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "=== Logging in to ${REGISTRY} as ${REGISTRY_USER} ==="
|
||||||
|
echo "$CI_RUNNER_TOKEN" | docker login "$REGISTRY" -u "$REGISTRY_USER" --password-stdin
|
||||||
|
|
||||||
|
echo "=== Building ${IMAGE}:latest ==="
|
||||||
|
docker build --network=host -f docker/ci/Dockerfile \
|
||||||
|
--secret "id=uv_token,src=${TOKEN_FILE}" \
|
||||||
|
-t "${IMAGE}:latest" \
|
||||||
|
.
|
||||||
|
|
||||||
|
STAMP="$(date -u +%Y%m%d%H%M)"
|
||||||
|
echo "=== Pushing ${IMAGE}:latest and ${IMAGE}:${STAMP} ==="
|
||||||
|
docker tag "${IMAGE}:latest" "${IMAGE}:${STAMP}"
|
||||||
|
docker push "${IMAGE}:latest"
|
||||||
|
docker push "${IMAGE}:${STAMP}"
|
||||||
|
|
||||||
|
echo "Done. Re-run failing CI jobs — runners pull ${IMAGE}:latest"
|
||||||
@@ -102,7 +102,8 @@ def test_should_log_info_when_already_connected(
|
|||||||
"""Test that the MinioAdapter logs info when connect is called while already connected."""
|
"""Test that the MinioAdapter logs info when connect is called while already connected."""
|
||||||
with caplog.at_level(logging.INFO):
|
with caplog.at_level(logging.INFO):
|
||||||
minio_adapter.connect()
|
minio_adapter.connect()
|
||||||
assert "Already connected to Minio" in caplog.text
|
assert "Already connected" in caplog.text
|
||||||
|
assert "Minio" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_when_unable_to_connect() -> None:
|
def test_should_raise_connection_error_when_unable_to_connect() -> None:
|
||||||
@@ -150,7 +151,8 @@ def test_connect_creates_bucket_when_create_bucket_if_missing_enabled(
|
|||||||
with caplog.at_level(logging.INFO):
|
with caplog.at_level(logging.INFO):
|
||||||
adapter.connect()
|
adapter.connect()
|
||||||
|
|
||||||
assert f"Creating bucket '{bucket_name}'" in caplog.text
|
assert "Creating bucket" in caplog.text
|
||||||
|
assert bucket_name in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_should_log_error_on_exception_during_exit(
|
def test_should_log_error_on_exception_during_exit(
|
||||||
@@ -233,10 +235,11 @@ def test_should_log_warning_when_getting_nonexistent_object(
|
|||||||
with caplog.at_level("WARNING"):
|
with caplog.at_level("WARNING"):
|
||||||
result = adapter.get(object_name)
|
result = adapter.get(object_name)
|
||||||
assert result is None
|
assert result is None
|
||||||
assert (
|
assert "Object not found" in caplog.text
|
||||||
f"Object '{object_name}' not found in bucket '{adapter._bucket_name}'"
|
assert object_name in caplog.text
|
||||||
in caplog.text
|
bucket_name = adapter._bucket_name
|
||||||
)
|
assert bucket_name is not None
|
||||||
|
assert bucket_name in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_should_reraise_s3error_other_than_no_such_key() -> None:
|
def test_should_reraise_s3error_other_than_no_such_key() -> None:
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
"""Integration tests for the RedisAdapter."""
|
"""Integration tests for the RedisAdapter."""
|
||||||
|
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
|
import logging
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import redis
|
import redis
|
||||||
@@ -48,6 +49,17 @@ def clear_redis(raw_redis_client: redis.Redis) -> None:
|
|||||||
raw_redis_client.flushall()
|
raw_redis_client.flushall()
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_log_info_when_already_connected(
|
||||||
|
redis_adapter: RedisAdapter,
|
||||||
|
caplog: pytest.LogCaptureFixture,
|
||||||
|
) -> None:
|
||||||
|
"""Test that the RedisAdapter logs info when connect is called while already connected."""
|
||||||
|
with caplog.at_level(logging.INFO):
|
||||||
|
redis_adapter.connect()
|
||||||
|
assert "Already connected" in caplog.text
|
||||||
|
assert "Redis" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
def test_should_raise_connection_error_when_unable_to_connect() -> None:
|
def test_should_raise_connection_error_when_unable_to_connect() -> None:
|
||||||
"""Test that the RedisAdapter raises ConnectionError when unable to connect."""
|
"""Test that the RedisAdapter raises ConnectionError when unable to connect."""
|
||||||
adapter = RedisAdapter(config=RedisConfig(uri="redis://invalid:6379"))
|
adapter = RedisAdapter(config=RedisConfig(uri="redis://invalid:6379"))
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
"""Unit tests for JsonRepositoryInterface."""
|
"""Unit tests for JsonRepositoryInterface."""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from python_repositories.interfaces.json_repository_interface import (
|
from python_repositories.interfaces.json_repository_interface import (
|
||||||
JsonRepositoryInterface,
|
JsonRepositoryInterface,
|
||||||
@@ -12,7 +14,7 @@ def test_instantiation_fails_when_get_not_implemented() -> None:
|
|||||||
class Incomplete(JsonRepositoryInterface):
|
class Incomplete(JsonRepositoryInterface):
|
||||||
"""A class that does not implement get."""
|
"""A class that does not implement get."""
|
||||||
|
|
||||||
def set(self, key: str, data: dict) -> None:
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def delete(self, key: str) -> None:
|
def delete(self, key: str) -> None:
|
||||||
@@ -31,7 +33,7 @@ def test_instantiation_fails_when_set_not_implemented() -> None:
|
|||||||
class Incomplete(JsonRepositoryInterface):
|
class Incomplete(JsonRepositoryInterface):
|
||||||
"""A class that does not implement set."""
|
"""A class that does not implement set."""
|
||||||
|
|
||||||
def get(self, key: str) -> dict | None:
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def delete(self, key: str) -> None:
|
def delete(self, key: str) -> None:
|
||||||
@@ -50,10 +52,10 @@ def test_instantiation_fails_when_delete_not_implemented() -> None:
|
|||||||
class Incomplete(JsonRepositoryInterface):
|
class Incomplete(JsonRepositoryInterface):
|
||||||
"""A class that does not implement delete."""
|
"""A class that does not implement delete."""
|
||||||
|
|
||||||
def get(self, key: str) -> dict | None:
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def set(self, key: str, data: dict) -> None:
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def list_keys(self, pattern: str) -> list[str]:
|
def list_keys(self, pattern: str) -> list[str]:
|
||||||
@@ -69,10 +71,10 @@ def test_instantiation_fails_when_list_keys_not_implemented() -> None:
|
|||||||
class Incomplete(JsonRepositoryInterface):
|
class Incomplete(JsonRepositoryInterface):
|
||||||
"""A class that does not implement list_keys."""
|
"""A class that does not implement list_keys."""
|
||||||
|
|
||||||
def get(self, key: str) -> dict | None:
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def set(self, key: str, data: dict) -> None:
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def delete(self, key: str) -> None:
|
def delete(self, key: str) -> None:
|
||||||
@@ -86,10 +88,10 @@ def test_scan_keys_defaults_to_list_keys() -> None:
|
|||||||
"""Test that the default scan_keys implementation delegates to list_keys."""
|
"""Test that the default scan_keys implementation delegates to list_keys."""
|
||||||
|
|
||||||
class Complete(JsonRepositoryInterface):
|
class Complete(JsonRepositoryInterface):
|
||||||
def get(self, key: str) -> dict | None:
|
def get(self, key: str) -> dict[str, Any] | None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def set(self, key: str, data: dict) -> None:
|
def set(self, key: str, data: dict[str, Any]) -> None:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def delete(self, key: str) -> None:
|
def delete(self, key: str) -> None:
|
||||||
|
|||||||
@@ -75,6 +75,16 @@ def test_connect_with_injected_client_raises_on_failure() -> None:
|
|||||||
adapter.connect()
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_with_injected_client_skips_validation_when_client_cleared() -> None:
|
||||||
|
mock_client = MagicMock(spec=Minio)
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG, client=mock_client)
|
||||||
|
adapter.disconnect()
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.list_buckets.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
def test_connect_disconnects_before_reconnect(
|
def test_connect_disconnects_before_reconnect(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -95,6 +105,27 @@ def test_connect_disconnects_before_reconnect(
|
|||||||
new_client.list_buckets.assert_called_once()
|
new_client.list_buckets.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_skips_reconnect_when_already_connected(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
stale_client = MagicMock(spec=Minio)
|
||||||
|
stale_client.bucket_exists.return_value = True
|
||||||
|
adapter = MinioAdapter(config=TEST_MINIO_CONFIG)
|
||||||
|
adapter._client = stale_client
|
||||||
|
adapter._bucket_name = TEST_MINIO_CONFIG.bucket
|
||||||
|
|
||||||
|
minio_ctor = MagicMock()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"python_repositories.adapters.minio_adapter.minio.Minio",
|
||||||
|
minio_ctor,
|
||||||
|
)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
minio_ctor.assert_not_called()
|
||||||
|
assert adapter._client is stale_client
|
||||||
|
|
||||||
|
|
||||||
def test_connect_raises_when_bucket_missing(
|
def test_connect_raises_when_bucket_missing(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -99,10 +99,21 @@ def test_connect_with_injected_client_raises_on_redis_error() -> None:
|
|||||||
adapter.connect()
|
adapter.connect()
|
||||||
|
|
||||||
|
|
||||||
def test_connect_closes_existing_non_injected_client(
|
def test_connect_with_injected_client_skips_validation_when_client_cleared() -> None:
|
||||||
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG, client=mock_client)
|
||||||
|
adapter.disconnect()
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
mock_client.ping.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_reconnects_when_existing_client_unhealthy(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
stale_client = MagicMock(spec=redis.Redis)
|
stale_client = MagicMock(spec=redis.Redis)
|
||||||
|
stale_client.ping.side_effect = redis.ConnectionError("connection lost")
|
||||||
new_client = MagicMock(spec=redis.Redis)
|
new_client = MagicMock(spec=redis.Redis)
|
||||||
new_client.ping.return_value = True
|
new_client.ping.return_value = True
|
||||||
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
@@ -116,6 +127,24 @@ def test_connect_closes_existing_non_injected_client(
|
|||||||
assert adapter._client is new_client
|
assert adapter._client is new_client
|
||||||
|
|
||||||
|
|
||||||
|
def test_connect_skips_reconnect_when_already_connected(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
|
stale_client = MagicMock(spec=redis.Redis)
|
||||||
|
stale_client.ping.return_value = True
|
||||||
|
adapter = RedisAdapter(config=TEST_REDIS_CONFIG)
|
||||||
|
adapter._client = stale_client
|
||||||
|
|
||||||
|
from_url = MagicMock()
|
||||||
|
monkeypatch.setattr("redis.Redis.from_url", from_url)
|
||||||
|
|
||||||
|
adapter.connect()
|
||||||
|
|
||||||
|
stale_client.close.assert_not_called()
|
||||||
|
from_url.assert_not_called()
|
||||||
|
assert adapter._client is stale_client
|
||||||
|
|
||||||
|
|
||||||
def test_scan_keys_yields_decoded_keys() -> None:
|
def test_scan_keys_yields_decoded_keys() -> None:
|
||||||
mock_client = MagicMock(spec=redis.Redis)
|
mock_client = MagicMock(spec=redis.Redis)
|
||||||
mock_client.scan_iter.return_value = iter([b"key1", b"key2"])
|
mock_client.scan_iter.return_value = iter([b"key1", b"key2"])
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
version = 1
|
version = 1
|
||||||
revision = 3
|
revision = 2
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
resolution-markers = [
|
resolution-markers = [
|
||||||
"python_full_version >= '3.15'",
|
"python_full_version >= '3.15'",
|
||||||
@@ -1056,7 +1056,7 @@ wheels = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "python-repositories"
|
name = "python-repositories"
|
||||||
version = "2.0.0"
|
version = "2.0.3"
|
||||||
source = { editable = "." }
|
source = { editable = "." }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "python-dotenv" },
|
{ name = "python-dotenv" },
|
||||||
|
|||||||
Reference in New Issue
Block a user