added argocd manifests

Reviewed-on: http://192.168.1.2:3000/brian/bandwidth_probing/pulls/13

.gitea/workflows/default.yaml

@@ -0,0 +1,30 @@
+name: pipeline
+run-name: ${{ gitea.actor }} is running script
+runs-on: ubuntu-latest
+on:
+  push:
+    branches-ignore:
+      - "release"
+image: python:3.10
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: https://github.com/actions/setup-python@v3
+        with:
+          python-version: "3.10"
+          architecture: "x64"
+      - name: Install Packages
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 mypy pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: PEP8 check
+        run: flake8 ./code --benchmark
+      - name: Type check
+        run: |
+          python3 -m pip install types-python-dateutil
+          mypy ./code

.gitea/workflows/release.yaml

@@ -0,0 +1,54 @@
+name: Docker
+run-name: Build and Publish Service
+runs-on: ubuntu-latest
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: https://github.com/actions/setup-python@v3
+        with:
+          python-version: "3.10"
+          architecture: "x64"
+      - name: Install Packages
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 mypy pytest
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: PEP8 check
+        run: flake8 ./code --benchmark
+      - name: Type check
+        run: |
+          python3 -m pip install types-python-dateutil
+          mypy ./code
+  publish:
+    name: Publish
+    runs-on: ubuntu-latest
+    needs: [test]
+    steps:
+      - uses: actions/checkout@v3
+      -
+        name: Set Environment Variables
+        id: vars
+        env:
+          repository: ${{gitea.repository}}
+          ref_name: ${{gitea.ref_name}}
+        run: |
+          echo $repository
+          echo ::set-output name=OWNER::${repository#/*}
+          echo ::set-output name=PROJECT::${repository#*/}
+          echo $ref_name
+          echo ::set-output name=TAG::${ref_name#refs/*/}
+      -
+        name: Push Image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: 192.168.1.2:5000/${{gitea.repository}}:${{steps.vars.outputs.TAG}}

Dockerfile

@@ -0,0 +1,54 @@
+###########
+# BUILDER #
+###########
+
+# pull official base image
+FROM python:3.10-bookworm as BUILDER
+
+# install JDK 17
+ENV JAVA_HOME=/opt/java/openjdk
+COPY --from=eclipse-temurin:17-jre $JAVA_HOME $JAVA_HOME
+ENV PATH="${JAVA_HOME}/bin:${PATH}"
+
+# install Ookla's speedtest
+RUN apt update && \
+    apt install -y speedtest-cli
+
+# set environment variables
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# install dependencies
+WORKDIR /usr/src/app
+COPY requirements.txt .
+RUN pip install --upgrade pip && \
+    pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt
+
+#########
+# FINAL #
+#########
+
+FROM python:3.10-bookworm
+
+# create home direcroty and app user
+RUN mkdir -p /home/app && \
+    addgroup --system app && \
+    adduser --system --group app
+
+# install packages from builder
+COPY --from=BUILDER /usr/src/app/wheels /wheels
+COPY --from=BUILDER /usr/src/app/requirements.txt requirements.txt
+RUN pip install --no-cache /wheels/*
+
+# copy in files
+WORKDIR /home/app
+COPY ./code .
+
+# change ownership to app user
+RUN chown -R app:app /home/app
+
+# change to the app user
+USER app
+
+# start execution
+ENTRYPOINT [ "python", "main.py" ]

Jenkinsfile

@@ -0,0 +1,21 @@
+pipeline {
+    agent any
+
+    stages {
+        stage('Build') {
+            steps {
+                echo 'Building..'
+            }
+        }
+        stage('Test') {
+            steps {
+                echo 'Testing..'
+            }
+        }
+        stage('Deploy') {
+            steps {
+                echo 'Deploying..'
+            }
+        }
+    }
+}

README.md

@@ -1,3 +1,21 @@
 # bandwidth_probing
 
-Service that runs in a container and measures network bandwidth at a specified interval. The results are sent to a PostgreSQL database.
+Service that runs in a container and measures network bandwidth at a specified interval. The results are sent to a MongoDB database. Logs are sent to discord channel.
+
+## Environment Variables
+
+```python
+LOGGER_LEVEL: Literal['debug', 'info', 'warning', 'error', 'critical'] = 'debug'
+USER_ID: int = 0 # id number 0 is reserved for testing
+REPLICATES: int = 3 # number of replicates to be measured per event
+TRIGGER_INTERVAL_SECONDS: float = 3600 # seconds between events
+DATA_QUEUE_FLUSH_INTERVAL_SECONDS: float = 600 # seconds between attempts to push data stuck in data queue
+
+DB_IP_ADDRESS: str = '192.168.1.2'
+DB_NAME: str = 'bandwidth_probing'
+DB_COLLECTION_NAME: str = 'data'
+
+DISCORD_SERVICE_NAME: str = 'bandwidth_probing
+DISCORD_WEBHOOK_URL: str = 'https://discord.com/api/webhooks/1127970367047225354/wNkoRex4OncMw11OCJg6atR_xgHS2VhLrku4jYIUw81Kr4sutdD3Tt-XAltG0UQy14rZ'
+DISCORD_LOGGER_LEVEL: Literal['debug', 'info', 'warning', 'error', 'critical'] = 'warning'
+```

argo/configmap.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  namespace: bandwidth-probing
+data:
+  USER_ID: "1"
+  SCHEDULE: "05 * * * *"

argo/deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: bandwidth-probing
+  namespace: bandwidth-probing
+  labels:
+    environment: live
+    app: bandwidth-probing
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: bandwidth-probing
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: bandwidth-probing
+    spec:
+      containers:
+        - envFrom:
+            - configMapRef:
+                name: app-config
+          image: '192.168.1.2:5000/brian/bandwidth_probing:v1.3.2'
+          imagePullPolicy: Always
+          name: bandwidth-probing
+          resources:
+            limits:
+              cpu: 500m
+              memory: 256Mi
+            requests:
+              cpu: 50m
+              memory: 64Mi

argo/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: bandwidth-probing

code/bandwidth.py

@@ -0,0 +1,23 @@
+from speedtest import Speedtest
+from dotenv import load_dotenv
+import logging
+
+
+def measure():
+    """
+    Measure bandwidth usiing Ookla/speedtest.
+    """
+    s = Speedtest()
+    s.get_servers(servers=[])
+    s.get_best_server()
+    s.download(threads=None)
+    s.upload(threads=None)
+    s.results.share()
+    logging.debug('finished')
+    return s.results.dict()
+
+
+if __name__ == '__main__':
+    load_dotenv()
+    res = measure()
+    print(res)

code/cron.py

@@ -0,0 +1,53 @@
+from cron_converter import Cron
+from configparser import ConfigParser
+from pathlib import Path
+from datetime import datetime
+import dateutil
+import time
+import functools
+import logging
+
+# load default values
+config = ConfigParser()
+config.read(Path(__file__).parent / 'defaults.ini')
+SCHEDULE = config.get('cron', 'schedule')
+TZ = config.get('cron', 'tz')
+
+
+def cronjob(
+    schedule: str = SCHEDULE,
+    tz: str = TZ
+):
+    """
+    decorator for running function as a cronjob.
+    """
+    logging.info(f'started cronjob with schedule {schedule} and tz {tz}')
+
+    def decorator_cronjob(func):
+        @functools.wraps(func)
+        def wrapper_cronjob(*args, **kwargs):
+            scheduler = Cron(schedule).schedule(timezone_str=tz)
+            trigger_time = scheduler.next()
+            while True:
+                while trigger_time > datetime.now(tz=dateutil.tz.gettz(tz)):
+                    # waiting loop
+                    time.sleep(1)
+                # actual execution
+                try:
+                    func(*args, **kwargs)
+                except Exception as e:
+                    logging.warning(f'failed executing {func}: {e}')
+                    raise
+                else:
+                    # update trigger time
+                    trigger_time = scheduler.next()
+        return wrapper_cronjob
+    return decorator_cronjob
+
+
+if __name__ == '__main__':
+    @cronjob(schedule='* * * * *', tz=TZ)
+    def test_loop(msg: str):
+        print(msg)
+
+    test_loop('hello world')

code/database.py

@@ -0,0 +1,50 @@
+from pymongo import MongoClient
+from dotenv import load_dotenv
+from configparser import ConfigParser
+from pathlib import Path
+import logging
+import os
+
+
+# load default values
+config = ConfigParser()
+config.read(Path(__file__).parent / 'defaults.ini')
+DB_IP_ADDRESS = config.get('database', 'db_ip_address')
+DB_NAME = config.get('database', 'db_name')
+DB_COLLECTION_NAME = config.get('database', 'db_collection_name')
+
+
+def connect(
+    ip_addr: str = DB_IP_ADDRESS,
+    db_name: str = DB_NAME,
+    collection_name: str = DB_COLLECTION_NAME
+):
+    """
+    Connect to MongoDB database and return collection.
+    """
+    # load environment variables
+    load_dotenv()
+    ip_addr = os.getenv(
+        'DB_IP_ADDRESS',
+        default=DB_IP_ADDRESS
+    )
+    db_name = os.getenv(
+        'DB_NAME',
+        default=DB_NAME
+    )
+    collection_name = os.getenv(
+        'DB_COLLECTION_NAME',
+        default=DB_COLLECTION_NAME
+    )
+    # connect to database
+    client: MongoClient = MongoClient(ip_addr)
+    db = client[db_name]
+    collection = db[collection_name]
+    logging.debug('finished')
+    return collection
+
+
+if __name__ == '__main__':
+    load_dotenv()
+    collection = connect()
+    print(collection)

code/defaults.ini

@@ -0,0 +1,18 @@
+[main]
+logger_level=debug
+user_id=0
+replicates=3
+
+[database]
+db_ip_address=192.168.1.2
+db_name=bandwidth_probing
+db_collection_name=data
+
+[discord]
+webhook_url=https://discord.com/api/webhooks/1127970367047225354/wNkoRex4OncMw11OCJg6atR_xgHS2VhLrku4jYIUw81Kr4sutdD3Tt-XAltG0UQy14rZ
+service_name=bandwidth_probing
+logger_level=warning
+
+[cron]
+schedule=05 * * * *
+tz=Europe/Paris

code/initialise_app.py

@@ -0,0 +1,71 @@
+from discord_logging.handler import DiscordHandler
+from dotenv import load_dotenv
+from configparser import ConfigParser
+from pathlib import Path
+import logging
+import os
+
+
+# load default values
+config = ConfigParser()
+config.read(Path(__file__).parent / 'defaults.ini')
+LOGGER_LEVEL = config.get('main', 'logger_level')
+DISCORD_SERVICE_NAME = config.get('discord', 'service_name')
+DISCORD_WEBHOOK_URL = config.get('discord', 'webhook_url')
+DISCORD_LOGGER_LEVEL = config.get('discord', 'logger_level')
+
+
+def initialise_app(
+    logger_level: str = LOGGER_LEVEL,
+    discord_service_name: str = DISCORD_SERVICE_NAME,
+    discord_webhook_url: str = DISCORD_WEBHOOK_URL,
+    discord_logger_level: str = DISCORD_LOGGER_LEVEL
+) -> None:
+    """
+    Convienience function that ensures eveything is ready
+    before running the main loop.
+    """
+    # load environment variables
+    load_dotenv()
+    assert (
+        'USER_ID' in os.environ
+    ), (
+        'environment variable USER_ID must be specified'
+    )
+    logger_level = os.getenv(
+        'LOGGER_LEVEL',
+        default=LOGGER_LEVEL
+    )
+    discord_service_name = os.getenv(
+        'DISCORD_SERVICE_NAME',
+        default=DISCORD_SERVICE_NAME
+    )
+    discord_webhook_url = os.getenv(
+        'DISCORD_WEBHOOK_URL',
+        default=DISCORD_WEBHOOK_URL
+    )
+    discord_logger_level = os.getenv(
+        'DISCORD_LOGGER_LEVEL',
+        default=DISCORD_LOGGER_LEVEL
+    )
+    # setup logging
+    fmt = '|%(asctime)s| %(levelname)s %(filename)s:%(funcName)s:%(message)s'
+    datefmt = '%Y-%m-%d %H:%M:%S'
+    level = getattr(logging, logger_level.upper())
+    logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
+    # setup logging to discord
+    logger = logging.getLogger()
+    discord_handler = DiscordHandler(
+        service_name=discord_service_name,
+        webhook_url=discord_webhook_url,
+    )
+    discord_handler.setFormatter(logging.Formatter('%(message)s'))
+    level = getattr(logging, discord_logger_level.upper())
+    discord_handler.setLevel(level=level)
+    logger.addHandler(discord_handler)
+    logging.debug('finished')
+
+
+if __name__ == '__main__':
+    initialise_app()
+    logging.warning('test message')

code/main.py

@@ -0,0 +1,132 @@
+import time
+import logging
+import os
+import json
+from configparser import ConfigParser
+from pathlib import Path
+from queue import Queue
+from initialise_app import initialise_app
+from cron import cronjob
+from bandwidth import measure as measure_bandwidth
+from database import connect
+
+
+# load default values
+config = ConfigParser()
+config.read(Path(__file__).parent / 'defaults.ini')
+USER_ID = int(config.get('main', 'user_id'))
+REPLICATES = int(config.get('main', 'replicates'))
+SCHEDULE = config.get('cron', 'schedule')
+TZ = config.get('cron', 'tz')
+
+
+def flush_data(
+    data_queue: Queue,
+    user_id: int
+) -> None:
+    # connect to database
+    db = connect()
+    # stop early
+    queue_size = data_queue.qsize()
+    if queue_size == 0:
+        logging.debug('queue already empty')
+        logging.debug('finished')
+        return
+    # flush queue
+    for i in range(queue_size):
+        # get data
+        data = data_queue.get()
+        # prepare payload
+        payload_dict = {
+            'user_id': user_id,
+            'data': data
+        }
+        # send to database
+        try:
+            db_id = db.insert_one(payload_dict).inserted_id
+        except Exception as e:
+            data_queue.put(data)    # put data back in queue
+            logging.error('failed sending data to database')
+            logging.debug(
+                'failed pushing to database:\n'
+                f'{json.dumps(payload_dict, indent=4)}\n'
+                f'with error: {e}'
+            )
+            continue
+        else:
+            logging.debug(f'data sent to database received id: {db_id}')
+    logging.debug('finished')
+
+
+def event(
+    data_queue: Queue,
+    user_id: int,
+    replicates: int = REPLICATES
+) -> None:
+    # run event
+    for rep_num in range(replicates):
+        logging.debug(f'running replicate {rep_num+1} of {replicates}')
+        # do measurement
+        try:
+            data = measure_bandwidth()
+        except Exception as e:
+            logging.error(f'failed to measure bandwidth: {e}')
+            continue
+        # add data to queue
+        data_queue.put(data)
+        # polite pause
+        time.sleep(1)
+    # flush queue
+    flush_data(
+        data_queue=data_queue,
+        user_id=user_id
+    )
+    logging.debug('finished')
+
+
+if __name__ == '__main__':
+    # setup
+    initialise_app()
+    data_queue: Queue = Queue()
+    user_id = int(os.getenv('USER_ID', default=USER_ID))
+    replicates = int(os.getenv('REPLICATES', default=REPLICATES))
+    schedule = os.getenv(
+        key='SCHEDULE',
+        default=SCHEDULE
+    )
+    tz = os.getenv(
+        key='TZ',
+        default=TZ
+    )
+
+    @cronjob(schedule=schedule, tz=tz)
+    def main_loop(
+        data_queue: Queue,
+        user_id: int,
+        replicates: int = REPLICATES
+    ) -> None:
+        # run event
+        for rep_num in range(replicates):
+            logging.debug(f'running replicate {rep_num+1} of {replicates}')
+            # do measurement
+            try:
+                data = measure_bandwidth()
+            except Exception as e:
+                logging.error(f'failed to measure bandwidth: {e}')
+                continue
+            # add data to queue
+            data_queue.put(data)
+            # polite pause
+            time.sleep(1)
+        # flush queue
+        flush_data(
+            data_queue=data_queue,
+            user_id=user_id
+        )
+        logging.debug('finished')
+    # start main loop
+    main_loop(
+        data_queue=data_queue,
+        user_id=user_id,
+        replicates=replicates
+    )

docker-compose.yml

@@ -0,0 +1,8 @@
+version: "3.9"
+services:
+  bandwidth-probing:
+    build: .
+    container_name: bandwidth-probing
+    restart: unless-stopped
+    env_file:
+      - .env

mypy.ini

@@ -0,0 +1,10 @@
+[mypy]
+
+[mypy-discord_logging.*]
+ignore_missing_imports = True
+
+[mypy-speedtest.*]
+ignore_missing_imports = True
+
+[mypy-cron_converter.*]
+ignore_missing_imports = True

requirements.txt

@@ -0,0 +1,29 @@
+aiosignal==1.3.1
+async-timeout==4.0.2
+attrs==23.1.0
+certifi==2023.5.7
+charset-normalizer==3.2.0
+cron-converter==1.0.2
+discord-webhook==1.1.0
+dnspython==2.3.0
+flake8==6.0.0
+frozenlist==1.3.3
+idna==3.4
+mccabe==0.7.0
+multidict==6.0.4
+mypy==1.4.1
+mypy-extensions==1.0.0
+pycodestyle==2.10.0
+pyflakes==3.0.1
+pymongo==4.4.0
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-logging-discord-handler==0.1.4
+requests==2.31.0
+six==1.16.0
+speedtest-cli==2.1.3
+tomli==2.0.1
+types-python-dateutil==2.8.19.14
+typing_extensions==4.7.1
+urllib3==2.0.3
+yarl==1.9.2