deleted unused code

.gitignore

@@ -0,0 +1 @@
+__pycache__

Dockerfile

@@ -3,13 +3,7 @@
 ###########
 
 # pull official base image
-FROM python:alpine as builder
+FROM dvcorg/cml:0-dvc2-base1-gpu as builder
-
-# update system
-RUN apk update && \
-    apk add git && \
-    apk add rsync && \
-    apk add docker
 
 # set work directory
 WORKDIR /usr/src/app
@@ -20,7 +14,7 @@ ENV PYTHONUNBUFFERED 1
 
 # install dependencies
 RUN pip install --upgrade pip
-COPY requirements.txt .
+COPY req-prod.txt requirements.txt
 RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt
 
 #########
@@ -28,12 +22,15 @@ RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requir
 #########
 
 # pull official base image
-FROM python:alpine
+FROM dvcorg/cml:0-dvc2-base1-gpu
 
-# create app home
+# create home directory and app user
-ENV APP_HOME = /home/app
+RUN mkdir -p /home/app && \
-RUN mkdir -p $APP_HOME
+    addgroup --system app && \
-WORKDIR $APP_HOME
+    adduser --system --group app
+
+# add folder to store repo in
+RUN mkdir -p /home/app/repo
 
 # install packages from builder
 COPY --from=builder /usr/src/app/wheels /wheels
@@ -41,7 +38,14 @@ COPY --from=builder /usr/src/app/requirements.txt requirements.txt
 RUN pip install --no-cache /wheels/*
 
 # copy in files
+WORKDIR /home/app
 COPY ./code .
 
+# change ownership to app user
+RUN chown -R app:app /home/app
+
+# change to the app user
+USER app
+
 # start execution
 ENTRYPOINT [ "python", "main.py" ]

app.py

@@ -1,53 +0,0 @@
-# from waitress import serve
-# from flask import Flask, request, jsonify
-import uvicorn
-from fastapi import FastAPI, Request
-from dotenv import load_dotenv
-import logging
-from pydantic import BaseModel
-from typing import List
-
-from utils import clone_repository
-
-class GiteaRequest(BaseModel):
-    ref: str
-    before: str
-    after: str
-    compare_url: str
-    commits: List[dict]
-    total_commits: int
-    head_commit: dict
-    repository: dict
-    pusher: dict
-    sender: dict
-
-app = FastAPI()
-
-@app.get('/')
-async def root():
-    return '<h1>Welcome to Gitea Runner</h1>'
-
-@app.post('/test')
-async def test(request: Request):
-    """ Test connection to server. """
-    logging.debug('Content-Type: ', request.headers.get('content-type'))
-    return {'status': 'success'}
-
-@app.post('/event')
-def handle_event(request: GiteaRequest):
-    """ Handle event from webhook. """
-    # stop early if allowed
-    commit_msg = request.head_commit['message']
-    if commit_msg.lower().startswith('[skip ci]'):
-        return {'status': 'received task'}
-    # clone repository
-    repo_url = request.repository['clone_url']
-    local_repo_dir = clone_repository(repo_url)
-    print('local_repo_dir: ', local_repo_dir)
-    return {'status': 'success'}
-
-if __name__ == '__main__':
-    uvicorn.run(
-        app=app,
-        host='0.0.0.0'
-    )

code/app.py

@@ -1,38 +1,106 @@
-from flask import Flask, request, jsonify
+import uvicorn
-import os
+from fastapi import FastAPI, Request, HTTPException
+from fastapi.responses import HTMLResponse
 from dotenv import load_dotenv
-from ipaddress import ip_address, ip_network
 import logging
+from typing import List
+import os
 
-load_dotenv()
+from utils import clone_repository, load_runner_script
-app = Flask(__name__)
+from setup_logging import setup_logging
+from gitea_request import GiteaRequest
 
-@app.before_request
+app = FastAPI()
-def check_authorized():
-    """
-    Ensure request from allowed ip.
-    """
-    # prepare variables
-    assert 'ALLOWED_IP_RANGE' in os.environ
-    allowed_ip_range = ip_network(os.getenv['ALLOWED_IP_RANGE'])
-    requesting_ip = ip_address(request.remote_addr)
-    # check that requesting ip is allowed
-    if requesting_ip not in allowed_ip_range:
-        logging.info(f'received request from unauthorised ip: {request.remote_addr}')
-        return jsonify(status='forbidden'), 403
-    logging.info(f'received request from {request.remote_addr}')
 
-@app.before_request
+@app.get('/', response_class=HTMLResponse)
-def check_media_type():
+async def root():
+    content_str = """
+        <html>
+            <head>
+                <title>Gitea Runner</title>
+            </head>
+            <body>
+                <h1>Welcome to Gitea Runner</h1>
+                <p>see /docs for more information</p>
+            </body>
+        </html>
     """
-    Ensure correct Content-Type of request
+    return HTMLResponse(content=content_str, status_code=200)
-    """
-    if not request.headers.get('Content-Type').lower().startswith('application/json'):
-        logging.error(f'received request with wrong content type')
-        return jsonify(status='unsupported media type'), 415
-    
-@app.route('/test', methods=['POST'])
-def test():
-    logging.debug(request.get_json(force=True))
-    return jsonify(status='success', sender=request.remote_addr)
 
+@app.post('/test')
+async def test(request: Request):
+    """ Test connection to server. """
+    logging.debug('Content-Type: ', request.headers.get('content-type'))
+    raise HTTPException(
+        status_code=200,
+        detail='test completed'
+    )
+
+@app.post('/event')
+def handle_event(request: GiteaRequest):
+    """ Handle event from webhook. """
+    # extract information
+    commit_msg = request.head_commit['message']
+    repo_url = request.repository['clone_url']
+    branch_name = request.ref.split('/')[-1]
+    # stop early if asked
+    if commit_msg.lower().startswith('[skip ci]'):
+        raise HTTPException(
+            status_code=200,
+            detail='skipping ci'
+        )
+    # clone repository
+    local_repo_dir = clone_repository(repo_url)
+    # locate runner script
+    runner_script_path = local_repo_dir / 'runner_script.yml'
+    if not runner_script_path.exists():
+        raise HTTPException(
+            status_code=200,
+            detail="no runner_script.yml in repository root"
+        )
+    # load runner script
+    runner_script = load_runner_script(path=runner_script_path)
+    # determine pipeline to run
+    if branch_name not in runner_script.pipelines.keys():
+        # get default pipeline
+        if 'default' not in runner_script.pipelines.keys():
+            raise HTTPException(
+                status_code=404,
+                detail='no pipelines defined in runner_script.yml'
+            )
+        pipeline = runner_script.pipelines['default']
+    else:
+        # get custom pipeline
+        pipeline = runner_script.pipelines[branch_name]
+    logging.info(f'running pipeline {pipeline.name}')
+    # TODO: create virtualenv?
+    # install requirements
+    assert pipeline.requirements is not None
+    requirements_path = local_repo_dir / pipeline.requirements
+    os.system(f'pip install -r {requirements_path}')
+    for cmd in pipeline.script:
+        try:
+            os.system(cmd)
+        except Exception as e:
+            raise HTTPException(
+                status_code=503,
+                detail=f'failed running {cmd}: {e}'
+            )
+    # send response
+    raise HTTPException(
+        status_code=202,
+        detail='received task'
+    )
+# consider to include link to page that show progress
+
+if __name__ == '__main__':
+    load_dotenv('dev.env')
+    setup_logging()
+    listen_ip = os.getenv('LISTEN_IP', default='0.0.0.0')
+    listen_port = int(os.getenv('LISTEN_PORT', default=8000))
+    logging.info(f'starting FastAPI server ({listen_ip}:{listen_port})')
+    uvicorn.run(
+        app=app,
+        host=listen_ip,
+        port=listen_port
+    )

code/gitea_request.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel
+from typing import List
+
+class GiteaRequest(BaseModel):
+    ref: str
+    before: str
+    after: str
+    compare_url: str
+    commits: List[dict]
+    total_commits: int
+    head_commit: dict
+    repository: dict
+    pusher: dict
+    sender: dict

code/main.py

@@ -1,20 +1,18 @@
-"""
-Run tasks based on webhooks configured in Gitea.
-
-Command-line optinos:
-    --debug, -d Send more detailed log output to console.
-"""
-
 from dotenv import load_dotenv
-from waitress import serve
+import uvicorn
-
 from setup_logging import setup_logging
 from app import app
+import logging
+import os
 
 if __name__ == '__main__':
     # setup
-    load_dotenv()
     setup_logging()
-    # serve app
+    listen_ip = os.getenv('LISTEN_IP', default='0.0.0.0')
-    serve(app, host='0.0.0.0', port=1706)
+    listen_port = int(os.getenv('LISTEN_PORT', default=8000))
-    
+    logging.info(f'starting FastAPI server ({listen_ip}:{listen_port})')
+    uvicorn.run(
+        app=app,
+        host=listen_ip,
+        port=listen_port
+    )

code/runner_script.py

@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+from typing import List, Dict
+import yaml
+from pathlib import Path
+
+class Pipeline(BaseModel):
+    name: str
+    requirements: str
+    script: List[str]
+
+class RunnerScript(BaseModel):
+    pipelines: Dict[str, Pipeline]
+
+if __name__ == '__main__':
+    path = Path(__file__).parent.parent / 'runner_script.yml'
+    with open(path, 'r') as fh:
+        script = yaml.safe_load(fh)
+    runner_script = RunnerScript.parse_obj(script)
+    print(runner_script)
+    for k, v in runner_script.pipelines.items():
+        print(k)
+        print(v)

code/setup_logging.py

@@ -5,10 +5,7 @@ import os
 def setup_logging():
     fmt = '%(asctime)s | %(levelname)s | %(filename)s | %(funcName)s | %(message)s'
     datefmt = '%Y-%m-%d %H:%M:%S'
-    if not 'LOGGER_LEVEL' in os.environ:
+    logger_level = os.getenv('LOGGER_LEVEL', default='info')
-        logger_level = 'info'
-    else:
-        logger_level = os.getenv('LOGGER_LEVEL')
     level = getattr(logging, logger_level.upper())
     logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
     logging.debug('finished')

code/utils.py

@@ -0,0 +1,33 @@
+from pydantic import AnyHttpUrl
+from typing import List
+from pathlib import Path
+from git import Repo
+import logging
+import shutil
+import os
+import yaml
+from runner_script import RunnerScript
+
+def clone_repository(repo_url: AnyHttpUrl) -> Path:
+    # extract repo name
+    repo_name = repo_url.split('/')[-1].replace('.git', '')
+    repo_dir = os.getenv('REPO_DIR', default=None)
+    assert repo_dir is not None
+    dst_dir = Path(repo_dir) / repo_name
+    # prepare repo download dir
+    if dst_dir.exists():
+        shutil.rmtree(dst_dir)
+    dst_dir.mkdir()
+    # git clone repo
+    Repo.clone_from(url=repo_url, to_path=dst_dir)
+    logging.debug('finished')
+    return dst_dir
+
+def load_runner_script(path: Path) -> RunnerScript:
+    assert path.exists()
+    with open(path, 'r') as fh:
+        script = yaml.safe_load(fh)
+    runner_script = RunnerScript.parse_obj(script)
+    logging.debug('finished')
+    return runner_script
+    

dev.env

@@ -1,3 +1,4 @@
 LOGGER_LEVEL = debug
 LISTEN_IP = 0.0.0.0
-LISTEN_PORT = 1706
+LISTEN_PORT = 8000
+REPO_DIR = /usr/src/app/repo/

docker-compose.yml

@@ -0,0 +1,20 @@
+version: "3.9"
+services:
+  gitea-runner:
+    build: .
+    container_name: gitea-runner
+    restart: unless-stopped
+    ports:
+      - "8000:8000"
+    environment:
+      - LOGGER_LEVEL=debug
+      - LISTEN_IP=0.0.0.0
+      - LISTEN_PORT=8000
+      - REPO_DIR=/home/app/repo/
+    # deploy:
+    #   resources:
+    #     reservations:
+    #       devices:
+    #         - driver: nvidia
+    #           device_ids: ['0']
+    #           capabilities: [gpu]

req-dev.txt

@@ -37,6 +37,7 @@ pydantic==1.10.11
 Pygments==2.15.1
 python-dateutil==2.8.2
 python-dotenv==1.0.0
+PyYAML==6.0
 pyzmq==25.1.0
 six==1.16.0
 smmap==5.0.0

req-prod.txt

@@ -0,0 +1,5 @@
+uvicorn==0.16.0
+fastapi==0.99.1
+GitPython==3.1.31
+python-dotenv==1.0.0
+PyYAML==6.0

runner_script.yml

@@ -0,0 +1,11 @@
+definitions:
+  steps:
+    step: &step-test
+      name: test
+      requirements: req-prod.txt
+      script:
+        - nvidia-smi
+
+pipelines:
+  default:
+    <<: *step-test

utils.py

@@ -1,13 +0,0 @@
-from pydantic import AnyHttpUrl
-from pathlib import Path
-from git import Repo
-
-def clone_repository(repo_url: AnyHttpUrl) -> Path:
-    # extract repo name
-    repo_name = repo_url.split('/')[-1].replace('.git', '')
-    dst_dir = Path(__file__).parent / repo_name
-    # prepare repo download dir
-    dst_dir.mkdir(exist_ok=True)
-    # git clone repo
-    Repo.clone_from(url=repo_url, to_path=dst_dir)
-    return dst_dir