deleted unused code

changed function to execute commands from runner script
added and commented out code for passing through gpu
2023-09-04 14:43:19 +02:00 · 2023-07-07 09:46:12 +02:00 · 2023-07-07 09:45:45 +02:00 · 2023-07-06 17:34:08 +02:00 · 2023-07-06 17:32:41 +02:00 · 2023-07-06 17:30:40 +02:00
10 changed files with 163 additions and 36 deletions
@@ -24,10 +24,13 @@ RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requir
 # pull official base image
 FROM dvcorg/cml:0-dvc2-base1-gpu

-# create app home
-ENV APP_HOME = /home/app
-RUN mkdir -p $APP_HOME
-WORKDIR $APP_HOME
+# create home directory and app user
+RUN mkdir -p /home/app && \
+    addgroup --system app && \
+    adduser --system --group app
+
+# add folder to store repo in
+RUN mkdir -p /home/app/repo

 # install packages from builder
 COPY --from=builder /usr/src/app/wheels /wheels
@@ -35,7 +38,14 @@ COPY --from=builder /usr/src/app/requirements.txt requirements.txt
 RUN pip install --no-cache /wheels/*

 # copy in files
+WORKDIR /home/app
 COPY ./code .

+# change ownership to app user
+RUN chown -R app:app /home/app
+
+# change to the app user
+USER app
+
 # start execution
 ENTRYPOINT [ "python", "main.py" ]
@@ -1,50 +1,97 @@
 import uvicorn
-from fastapi import FastAPI, Request
+from fastapi import FastAPI, Request, HTTPException
+from fastapi.responses import HTMLResponse
 from dotenv import load_dotenv
 import logging
-from pydantic import BaseModel
 from typing import List
 import os

-from utils import clone_repository
+from utils import clone_repository, load_runner_script
 from setup_logging import setup_logging
-
-class GiteaRequest(BaseModel):
-    ref: str
-    before: str
-    after: str
-    compare_url: str
-    commits: List[dict]
-    total_commits: int
-    head_commit: dict
-    repository: dict
-    pusher: dict
-    sender: dict
+from gitea_request import GiteaRequest

 app = FastAPI()

-@app.get('/')
+@app.get('/', response_class=HTMLResponse)
 async def root():
-    return '<h1>Welcome to Gitea Runsner</h1>'
+    content_str = """
+        <html>
+            <head>
+                <title>Gitea Runner</title>
+            </head>
+            <body>
+                <h1>Welcome to Gitea Runner</h1>
+                <p>see /docs for more information</p>
+            </body>
+        </html>
+    """
+    return HTMLResponse(content=content_str, status_code=200)

@app.post('/test')
 async def test(request: Request):
    """ Test connection to server. """
    logging.debug('Content-Type: ', request.headers.get('content-type'))
-    return {'status': 'success'}
+    raise HTTPException(
+        status_code=200,
+        detail='test completed'
+    )

@app.post('/event')
 def handle_event(request: GiteaRequest):
    """ Handle event from webhook. """
-    # stop early if allowed
+    # extract information
    commit_msg = request.head_commit['message']
-    if commit_msg.lower().startswith('[skip ci]'):
-        return {'status': 'received task'}
-    # clone repository
    repo_url = request.repository['clone_url']
+    branch_name = request.ref.split('/')[-1]
+    # stop early if asked
+    if commit_msg.lower().startswith('[skip ci]'):
+        raise HTTPException(
+            status_code=200,
+            detail='skipping ci'
+        )
+    # clone repository
    local_repo_dir = clone_repository(repo_url)
-    print('local_repo_dir: ', local_repo_dir)
-    return {'status': 'success'}
+    # locate runner script
+    runner_script_path = local_repo_dir / 'runner_script.yml'
+    if not runner_script_path.exists():
+        raise HTTPException(
+            status_code=200,
+            detail="no runner_script.yml in repository root"
+        )
+    # load runner script
+    runner_script = load_runner_script(path=runner_script_path)
+    # determine pipeline to run
+    if branch_name not in runner_script.pipelines.keys():
+        # get default pipeline
+        if 'default' not in runner_script.pipelines.keys():
+            raise HTTPException(
+                status_code=404,
+                detail='no pipelines defined in runner_script.yml'
+            )
+        pipeline = runner_script.pipelines['default']
+    else:
+        # get custom pipeline
+        pipeline = runner_script.pipelines[branch_name]
+    logging.info(f'running pipeline {pipeline.name}')
+    # TODO: create virtualenv?
+    # install requirements
+    assert pipeline.requirements is not None
+    requirements_path = local_repo_dir / pipeline.requirements
+    os.system(f'pip install -r {requirements_path}')
+    for cmd in pipeline.script:
+        try:
+            os.system(cmd)
+        except Exception as e:
+            raise HTTPException(
+                status_code=503,
+                detail=f'failed running {cmd}: {e}'
+            )
+    # send response
+    raise HTTPException(
+        status_code=202,
+        detail='received task'
+    )
+# consider to include link to page that show progress

 if __name__ == '__main__':
    load_dotenv('dev.env')
@@ -0,0 +1,14 @@
+from pydantic import BaseModel
+from typing import List
+
+class GiteaRequest(BaseModel):
+    ref: str
+    before: str
+    after: str
+    compare_url: str
+    commits: List[dict]
+    total_commits: int
+    head_commit: dict
+    repository: dict
+    pusher: dict
+    sender: dict
@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+from typing import List, Dict
+import yaml
+from pathlib import Path
+
+class Pipeline(BaseModel):
+    name: str
+    requirements: str
+    script: List[str]
+
+class RunnerScript(BaseModel):
+    pipelines: Dict[str, Pipeline]
+
+if __name__ == '__main__':
+    path = Path(__file__).parent.parent / 'runner_script.yml'
+    with open(path, 'r') as fh:
+        script = yaml.safe_load(fh)
+    runner_script = RunnerScript.parse_obj(script)
+    print(runner_script)
+    for k, v in runner_script.pipelines.items():
+        print(k)
+        print(v)
@@ -1,8 +1,12 @@
 from pydantic import AnyHttpUrl
+from typing import List
 from pathlib import Path
 from git import Repo
+import logging
 import shutil
 import os
+import yaml
+from runner_script import RunnerScript

 def clone_repository(repo_url: AnyHttpUrl) -> Path:
    # extract repo name
@@ -16,4 +20,14 @@ def clone_repository(repo_url: AnyHttpUrl) -> Path:
    dst_dir.mkdir()
    # git clone repo
    Repo.clone_from(url=repo_url, to_path=dst_dir)
+    logging.debug('finished')
    return dst_dir
+
+def load_runner_script(path: Path) -> RunnerScript:
+    assert path.exists()
+    with open(path, 'r') as fh:
+        script = yaml.safe_load(fh)
+    runner_script = RunnerScript.parse_obj(script)
+    logging.debug('finished')
+    return runner_script
+    
@@ -4,6 +4,17 @@ services:
    build: .
    container_name: gitea-runner
    restart: unless-stopped
-    env_file: prod.env
    ports:
      - "8000:8000"
+    environment:
+      - LOGGER_LEVEL=debug
+      - LISTEN_IP=0.0.0.0
+      - LISTEN_PORT=8000
+      - REPO_DIR=/home/app/repo/
+    # deploy:
+    #   resources:
+    #     reservations:
+    #       devices:
+    #         - driver: nvidia
+    #           device_ids: ['0']
+    #           capabilities: [gpu]
@@ -1,4 +0,0 @@
-LOGGER_LEVEL = info
-LISTEN_IP = 0.0.0.0
-LISTEN_PORT = 8000
-REPO_DIR = /usr/src/app/repo/
@@ -37,6 +37,7 @@ pydantic==1.10.11
 Pygments==2.15.1
 python-dateutil==2.8.2
 python-dotenv==1.0.0
+PyYAML==6.0
 pyzmq==25.1.0
 six==1.16.0
 smmap==5.0.0
@@ -2,3 +2,4 @@ uvicorn==0.16.0
 fastapi==0.99.1
 GitPython==3.1.31
 python-dotenv==1.0.0
+PyYAML==6.0
@@ -0,0 +1,11 @@
+definitions:
+  steps:
+    step: &step-test
+      name: test
+      requirements: req-prod.txt
+      script:
+        - nvidia-smi
+
+pipelines:
+  default:
+    <<: *step-test
Author	SHA1	Message	Date
brb	c19fc58a25	deleted unused code	2023-09-04 14:43:19 +02:00
brb	bd5dc8fbc4	changed function to execute commands from runner script	2023-07-07 09:46:12 +02:00
brb	f4a81b4b9b	added and commented out code for passing through gpu	2023-07-07 09:45:45 +02:00
brb	477389d353	updated script	2023-07-06 17:34:08 +02:00
brb	90a44e8b6e	updated running method	2023-07-06 17:32:41 +02:00
brb	36d2233567	updated exception handling when running commands from runner script	2023-07-06 17:30:40 +02:00
brb	2fc979a845	updated to run as user instead of root	2023-07-06 17:28:38 +02:00
brb	7218004f2f	updated to run actual function	2023-07-06 17:22:21 +02:00
brb	75fd57391d	added pip install requirements	2023-07-06 17:22:09 +02:00
brb	a82e0a8f10	fixed logging bug	2023-07-06 16:44:49 +02:00
brb	d1864c7030	added logging	2023-07-06 16:44:37 +02:00
brb	43e4b9788a	updated to handle default and custom branch names	2023-07-06 16:42:54 +02:00
brb	84b409356b	added detection of branch name	2023-07-06 16:07:27 +02:00
brb	9513eb488b	adapted to new definition	2023-07-06 15:58:25 +02:00
brb	a51c49e8f4	adapted to new definition	2023-07-06 15:58:15 +02:00
brb	dba476e78a	updated definition	2023-07-06 15:58:03 +02:00
brb	87a738461c	defined runner script	2023-07-06 15:54:20 +02:00
brb	40c9b7442d	added printing of content from runner script	2023-07-06 15:53:34 +02:00
brb	c1aa6f3c41	added function to read runner script	2023-07-06 15:53:15 +02:00
brb	9102d03ef4	created runner script	2023-07-06 15:53:00 +02:00
brb	5c21aa767f	installed packages	2023-07-06 13:27:32 +02:00
brb	3187248fb9	[skip ci] fixed bug when accessing /test	2023-07-06 13:22:32 +02:00
brb	ed05a8eca5	update root page	2023-07-06 13:20:24 +02:00
brb	02ec54fd16	updated response text	2023-07-06 13:15:16 +02:00
brb	9fffa861e7	added checking for presence of runner script	2023-07-06 13:10:13 +02:00
brb	3a73c9cf6e	added more detailed responses to requests	2023-07-06 12:58:09 +02:00
brb	387e739a07	[skip ci] testing if ci is skipped correctly	2023-07-06 12:54:12 +02:00
brb	39d1ac1474	moved definition of class to separate file	2023-07-06 12:50:30 +02:00
brb	2aabc9ab37	added more logging	2023-07-06 12:20:02 +02:00
brb	2f964dabcc	added more logging	2023-07-06 12:19:49 +02:00
brb	47576392a9	updated logging level	2023-07-06 12:19:33 +02:00
brb	31e771eb9f	fixed bug when defining env vars	2023-07-06 11:59:21 +02:00
brb	935adf98ad	removed unused file	2023-07-06 11:58:36 +02:00
brb	39b2afaf1f	added creation of repo dir and moved env vars into docker compose	2023-07-06 11:58:21 +02:00