deleted unused code

changed function to execute commands from runner script
added and commented out code for passing through gpu
2023-09-04 14:43:19 +02:00 · 2023-07-07 09:46:12 +02:00 · 2023-07-07 09:45:45 +02:00 · 2023-07-06 17:34:08 +02:00 · 2023-07-06 17:32:41 +02:00 · 2023-07-06 17:30:40 +02:00
8 changed files with 120 additions and 17 deletions
@@ -24,10 +24,12 @@ RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requir
 # pull official base image
 FROM dvcorg/cml:0-dvc2-base1-gpu

-# create app home
-ENV APP_HOME = /home/app
-RUN mkdir -p $APP_HOME
-WORKDIR $APP_HOME
+# create home directory and app user
+RUN mkdir -p /home/app && \
+    addgroup --system app && \
+    adduser --system --group app
+
+# add folder to store repo in
 RUN mkdir -p /home/app/repo

 # install packages from builder
@@ -36,7 +38,14 @@ COPY --from=builder /usr/src/app/requirements.txt requirements.txt
 RUN pip install --no-cache /wheels/*

 # copy in files
+WORKDIR /home/app
 COPY ./code .

+# change ownership to app user
+RUN chown -R app:app /home/app
+
+# change to the app user
+USER app
+
 # start execution
 ENTRYPOINT [ "python", "main.py" ]
@@ -1,25 +1,37 @@
 import uvicorn
 from fastapi import FastAPI, Request, HTTPException
+from fastapi.responses import HTMLResponse
 from dotenv import load_dotenv
 import logging
 from typing import List
 import os

-from utils import clone_repository
+from utils import clone_repository, load_runner_script
 from setup_logging import setup_logging
 from gitea_request import GiteaRequest

 app = FastAPI()

-@app.get('/')
+@app.get('/', response_class=HTMLResponse)
 async def root():
-    return '<h1>Welcome to Gitea Runsner</h1>'
+    content_str = """
+        <html>
+            <head>
+                <title>Gitea Runner</title>
+            </head>
+            <body>
+                <h1>Welcome to Gitea Runner</h1>
+                <p>see /docs for more information</p>
+            </body>
+        </html>
+    """
+    return HTMLResponse(content=content_str, status_code=200)

@app.post('/test')
 async def test(request: Request):
    """ Test connection to server. """
    logging.debug('Content-Type: ', request.headers.get('content-type'))
-    HTTPException(
+    raise HTTPException(
        status_code=200,
        detail='test completed'
    )
@@ -27,31 +39,59 @@ async def test(request: Request):
@app.post('/event')
 def handle_event(request: GiteaRequest):
    """ Handle event from webhook. """
-    # stop early if asked
+    # extract information
    commit_msg = request.head_commit['message']
+    repo_url = request.repository['clone_url']
+    branch_name = request.ref.split('/')[-1]
+    # stop early if asked
    if commit_msg.lower().startswith('[skip ci]'):
        raise HTTPException(
            status_code=200,
            detail='skipping ci'
        )
    # clone repository
-    repo_url = request.repository['clone_url']
-    logging.info('cloning repository: {repo_url}')
    local_repo_dir = clone_repository(repo_url)
-    logging.info(f'local_repo_dir: {local_repo_dir}')
    # locate runner script
    runner_script_path = local_repo_dir / 'runner_script.yml'
    if not runner_script_path.exists():
        raise HTTPException(
            status_code=200,
-            detail='no "runner_script.yml" in repository root'
+            detail="no runner_script.yml in repository root"
        )
-    
+    # load runner script
+    runner_script = load_runner_script(path=runner_script_path)
+    # determine pipeline to run
+    if branch_name not in runner_script.pipelines.keys():
+        # get default pipeline
+        if 'default' not in runner_script.pipelines.keys():
+            raise HTTPException(
+                status_code=404,
+                detail='no pipelines defined in runner_script.yml'
+            )
+        pipeline = runner_script.pipelines['default']
+    else:
+        # get custom pipeline
+        pipeline = runner_script.pipelines[branch_name]
+    logging.info(f'running pipeline {pipeline.name}')
+    # TODO: create virtualenv?
+    # install requirements
+    assert pipeline.requirements is not None
+    requirements_path = local_repo_dir / pipeline.requirements
+    os.system(f'pip install -r {requirements_path}')
+    for cmd in pipeline.script:
+        try:
+            os.system(cmd)
+        except Exception as e:
+            raise HTTPException(
+                status_code=503,
+                detail=f'failed running {cmd}: {e}'
+            )
    # send response
    raise HTTPException(
        status_code=202,
        detail='received task'
    )
+# consider to include link to page that show progress

 if __name__ == '__main__':
    load_dotenv('dev.env')
@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+from typing import List, Dict
+import yaml
+from pathlib import Path
+
+class Pipeline(BaseModel):
+    name: str
+    requirements: str
+    script: List[str]
+
+class RunnerScript(BaseModel):
+    pipelines: Dict[str, Pipeline]
+
+if __name__ == '__main__':
+    path = Path(__file__).parent.parent / 'runner_script.yml'
+    with open(path, 'r') as fh:
+        script = yaml.safe_load(fh)
+    runner_script = RunnerScript.parse_obj(script)
+    print(runner_script)
+    for k, v in runner_script.pipelines.items():
+        print(k)
+        print(v)
@@ -1,9 +1,12 @@
 from pydantic import AnyHttpUrl
+from typing import List
 from pathlib import Path
 from git import Repo
 import logging
 import shutil
 import os
+import yaml
+from runner_script import RunnerScript

 def clone_repository(repo_url: AnyHttpUrl) -> Path:
    # extract repo name
@@ -18,4 +21,13 @@ def clone_repository(repo_url: AnyHttpUrl) -> Path:
    # git clone repo
    Repo.clone_from(url=repo_url, to_path=dst_dir)
    logging.debug('finished')
-    return dst_dir
+    return dst_dir
+
+def load_runner_script(path: Path) -> RunnerScript:
+    assert path.exists()
+    with open(path, 'r') as fh:
+        script = yaml.safe_load(fh)
+    runner_script = RunnerScript.parse_obj(script)
+    logging.debug('finished')
+    return runner_script
+    
@@ -10,4 +10,11 @@ services:
      - LOGGER_LEVEL=debug
      - LISTEN_IP=0.0.0.0
      - LISTEN_PORT=8000
-      - REPO_DIR=/home/app/repo/
+      - REPO_DIR=/home/app/repo/
+    # deploy:
+    #   resources:
+    #     reservations:
+    #       devices:
+    #         - driver: nvidia
+    #           device_ids: ['0']
+    #           capabilities: [gpu]
@@ -37,6 +37,7 @@ pydantic==1.10.11
 Pygments==2.15.1
 python-dateutil==2.8.2
 python-dotenv==1.0.0
+PyYAML==6.0
 pyzmq==25.1.0
 six==1.16.0
 smmap==5.0.0
@@ -1,4 +1,5 @@
 uvicorn==0.16.0
 fastapi==0.99.1
 GitPython==3.1.31
-python-dotenv==1.0.0
+python-dotenv==1.0.0
+PyYAML==6.0
@@ -0,0 +1,11 @@
+definitions:
+  steps:
+    step: &step-test
+      name: test
+      requirements: req-prod.txt
+      script:
+        - nvidia-smi
+
+pipelines:
+  default:
+    <<: *step-test
Author	SHA1	Message	Date
brb	c19fc58a25	deleted unused code	2023-09-04 14:43:19 +02:00
brb	bd5dc8fbc4	changed function to execute commands from runner script	2023-07-07 09:46:12 +02:00
brb	f4a81b4b9b	added and commented out code for passing through gpu	2023-07-07 09:45:45 +02:00
brb	477389d353	updated script	2023-07-06 17:34:08 +02:00
brb	90a44e8b6e	updated running method	2023-07-06 17:32:41 +02:00
brb	36d2233567	updated exception handling when running commands from runner script	2023-07-06 17:30:40 +02:00
brb	2fc979a845	updated to run as user instead of root	2023-07-06 17:28:38 +02:00
brb	7218004f2f	updated to run actual function	2023-07-06 17:22:21 +02:00
brb	75fd57391d	added pip install requirements	2023-07-06 17:22:09 +02:00
brb	a82e0a8f10	fixed logging bug	2023-07-06 16:44:49 +02:00
brb	d1864c7030	added logging	2023-07-06 16:44:37 +02:00
brb	43e4b9788a	updated to handle default and custom branch names	2023-07-06 16:42:54 +02:00
brb	84b409356b	added detection of branch name	2023-07-06 16:07:27 +02:00
brb	9513eb488b	adapted to new definition	2023-07-06 15:58:25 +02:00
brb	a51c49e8f4	adapted to new definition	2023-07-06 15:58:15 +02:00
brb	dba476e78a	updated definition	2023-07-06 15:58:03 +02:00
brb	87a738461c	defined runner script	2023-07-06 15:54:20 +02:00
brb	40c9b7442d	added printing of content from runner script	2023-07-06 15:53:34 +02:00
brb	c1aa6f3c41	added function to read runner script	2023-07-06 15:53:15 +02:00
brb	9102d03ef4	created runner script	2023-07-06 15:53:00 +02:00
brb	5c21aa767f	installed packages	2023-07-06 13:27:32 +02:00
brb	3187248fb9	[skip ci] fixed bug when accessing /test	2023-07-06 13:22:32 +02:00
brb	ed05a8eca5	update root page	2023-07-06 13:20:24 +02:00
brb	02ec54fd16	updated response text	2023-07-06 13:15:16 +02:00