brian/gitea_runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: brian/gitea_runner:9fffa861e7909fb1645837b3060fd71ef3adcfc4
Branches Tags
brian/gitea_runner:main brian/gitea_runner:test
..
compare: brian/gitea_runner:main
Branches Tags
brian/gitea_runner:main brian/gitea_runner:test
24 Commits
9fffa861e7 .. main
Author SHA1 Message Date
brb c19fc58a25 deleted unused code 2023-09-04 14:43:19 +02:00
brb bd5dc8fbc4 changed function to execute commands from runner script 2023-07-07 09:46:12 +02:00
brb f4a81b4b9b added and commented out code for passing through gpu 2023-07-07 09:45:45 +02:00
brb 477389d353 updated script 2023-07-06 17:34:08 +02:00
brb 90a44e8b6e updated running method 2023-07-06 17:32:41 +02:00
brb 36d2233567 updated exception handling when running commands from runner script 2023-07-06 17:30:40 +02:00
brb 2fc979a845 updated to run as user instead of root 2023-07-06 17:28:38 +02:00
brb 7218004f2f updated to run actual function 2023-07-06 17:22:21 +02:00
brb 75fd57391d added pip install requirements 2023-07-06 17:22:09 +02:00
brb a82e0a8f10 fixed logging bug 2023-07-06 16:44:49 +02:00
brb d1864c7030 added logging 2023-07-06 16:44:37 +02:00
brb 43e4b9788a updated to handle default and custom branch names 2023-07-06 16:42:54 +02:00
brb 84b409356b added detection of branch name 2023-07-06 16:07:27 +02:00
brb 9513eb488b adapted to new definition 2023-07-06 15:58:25 +02:00
brb a51c49e8f4 adapted to new definition 2023-07-06 15:58:15 +02:00
brb dba476e78a updated definition 2023-07-06 15:58:03 +02:00
brb 87a738461c defined runner script 2023-07-06 15:54:20 +02:00
brb 40c9b7442d added printing of content from runner script 2023-07-06 15:53:34 +02:00
brb c1aa6f3c41 added function to read runner script 2023-07-06 15:53:15 +02:00
brb 9102d03ef4 created runner script 2023-07-06 15:53:00 +02:00
brb 5c21aa767f installed packages 2023-07-06 13:27:32 +02:00
brb 3187248fb9 [skip ci] fixed bug when accessing /test 2023-07-06 13:22:32 +02:00
brb ed05a8eca5 update root page 2023-07-06 13:20:24 +02:00
brb 02ec54fd16 updated response text 2023-07-06 13:15:16 +02:00
8 changed files with 120 additions and 17 deletions
Expand all files Collapse all files
Dockerfile
+13 -4
View File
@@ -24,10 +24,12 @@ RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requir
# pull official base image # pull official base image
FROM dvcorg/cml:0-dvc2-base1-gpu FROM dvcorg/cml:0-dvc2-base1-gpu
# create app home # create home directory and app user
ENV APP_HOME = /home/app RUN mkdir -p /home/app && \
RUN mkdir -p $APP_HOME addgroup --system app && \
WORKDIR $APP_HOME adduser --system --group app
# add folder to store repo in
RUN mkdir -p /home/app/repo RUN mkdir -p /home/app/repo
# install packages from builder # install packages from builder
@@ -36,7 +38,14 @@ COPY --from=builder /usr/src/app/requirements.txt requirements.txt
RUN pip install --no-cache /wheels/* RUN pip install --no-cache /wheels/*
# copy in files # copy in files
WORKDIR /home/app
COPY ./code . COPY ./code .
# change ownership to app user
RUN chown -R app:app /home/app
# change to the app user
USER app
# start execution # start execution
ENTRYPOINT [ "python", "main.py" ] ENTRYPOINT [ "python", "main.py" ]
code/app.py
+50 -10
View File
@@ -1,25 +1,37 @@
import uvicorn import uvicorn
from fastapi import FastAPI, Request, HTTPException from fastapi import FastAPI, Request, HTTPException
from fastapi.responses import HTMLResponse
from dotenv import load_dotenv from dotenv import load_dotenv
import logging import logging
from typing import List from typing import List
import os import os
from utils import clone_repository from utils import clone_repository, load_runner_script
from setup_logging import setup_logging from setup_logging import setup_logging
from gitea_request import GiteaRequest from gitea_request import GiteaRequest
app = FastAPI() app = FastAPI()
@app.get('/') @app.get('/', response_class=HTMLResponse)
async def root(): async def root():
return '<h1>Welcome to Gitea Runsner</h1>' content_str = """
<html>
<head>
<title>Gitea Runner</title>
</head>
<body>
<h1>Welcome to Gitea Runner</h1>
<p>see /docs for more information</p>
</body>
</html>
"""
return HTMLResponse(content=content_str, status_code=200)
@app.post('/test') @app.post('/test')
async def test(request: Request): async def test(request: Request):
""" Test connection to server. """ """ Test connection to server. """
logging.debug('Content-Type: ', request.headers.get('content-type')) logging.debug('Content-Type: ', request.headers.get('content-type'))
HTTPException( raise HTTPException(
status_code=200, status_code=200,
detail='test completed' detail='test completed'
) )
@@ -27,31 +39,59 @@ async def test(request: Request):
@app.post('/event') @app.post('/event')
def handle_event(request: GiteaRequest): def handle_event(request: GiteaRequest):
""" Handle event from webhook. """ """ Handle event from webhook. """
# stop early if asked # extract information
commit_msg = request.head_commit['message'] commit_msg = request.head_commit['message']
repo_url = request.repository['clone_url']
branch_name = request.ref.split('/')[-1]
# stop early if asked
if commit_msg.lower().startswith('[skip ci]'): if commit_msg.lower().startswith('[skip ci]'):
raise HTTPException( raise HTTPException(
status_code=200, status_code=200,
detail='skipping ci' detail='skipping ci'
) )
# clone repository # clone repository
repo_url = request.repository['clone_url']
logging.info('cloning repository: {repo_url}')
local_repo_dir = clone_repository(repo_url) local_repo_dir = clone_repository(repo_url)
logging.info(f'local_repo_dir: {local_repo_dir}')
# locate runner script # locate runner script
runner_script_path = local_repo_dir / 'runner_script.yml' runner_script_path = local_repo_dir / 'runner_script.yml'
if not runner_script_path.exists(): if not runner_script_path.exists():
raise HTTPException( raise HTTPException(
status_code=200, status_code=200,
detail='no "runner_script.yml" in repository root' detail="no runner_script.yml in repository root"
) )
# load runner script
runner_script = load_runner_script(path=runner_script_path)
# determine pipeline to run
if branch_name not in runner_script.pipelines.keys():
# get default pipeline
if 'default' not in runner_script.pipelines.keys():
raise HTTPException(
status_code=404,
detail='no pipelines defined in runner_script.yml'
)
pipeline = runner_script.pipelines['default']
else:
# get custom pipeline
pipeline = runner_script.pipelines[branch_name]
logging.info(f'running pipeline {pipeline.name}')
# TODO: create virtualenv?
# install requirements
assert pipeline.requirements is not None
requirements_path = local_repo_dir / pipeline.requirements
os.system(f'pip install -r {requirements_path}')
for cmd in pipeline.script:
try:
os.system(cmd)
except Exception as e:
raise HTTPException(
status_code=503,
detail=f'failed running {cmd}: {e}'
)
# send response # send response
raise HTTPException( raise HTTPException(
status_code=202, status_code=202,
detail='received task' detail='received task'
) )
# consider to include link to page that show progress
if __name__ == '__main__': if __name__ == '__main__':
load_dotenv('dev.env') load_dotenv('dev.env')
code/runner_script.py
+22
View File
@@ -0,0 +1,22 @@
from pydantic import BaseModel
from typing import List, Dict
import yaml
from pathlib import Path
class Pipeline(BaseModel):
name: str
requirements: str
script: List[str]
class RunnerScript(BaseModel):
pipelines: Dict[str, Pipeline]
if __name__ == '__main__':
path = Path(__file__).parent.parent / 'runner_script.yml'
with open(path, 'r') as fh:
script = yaml.safe_load(fh)
runner_script = RunnerScript.parse_obj(script)
print(runner_script)
for k, v in runner_script.pipelines.items():
print(k)
print(v)
code/utils.py
+12
View File
@@ -1,9 +1,12 @@
from pydantic import AnyHttpUrl from pydantic import AnyHttpUrl
from typing import List
from pathlib import Path from pathlib import Path
from git import Repo from git import Repo
import logging import logging
import shutil import shutil
import os import os
import yaml
from runner_script import RunnerScript
def clone_repository(repo_url: AnyHttpUrl) -> Path: def clone_repository(repo_url: AnyHttpUrl) -> Path:
# extract repo name # extract repo name
@@ -19,3 +22,12 @@ def clone_repository(repo_url: AnyHttpUrl) -> Path:
Repo.clone_from(url=repo_url, to_path=dst_dir) Repo.clone_from(url=repo_url, to_path=dst_dir)
logging.debug('finished') logging.debug('finished')
return dst_dir return dst_dir
def load_runner_script(path: Path) -> RunnerScript:
assert path.exists()
with open(path, 'r') as fh:
script = yaml.safe_load(fh)
runner_script = RunnerScript.parse_obj(script)
logging.debug('finished')
return runner_script
docker-compose.yml
+7
View File
@@ -11,3 +11,10 @@ services:
- LISTEN_IP=0.0.0.0 - LISTEN_IP=0.0.0.0
- LISTEN_PORT=8000 - LISTEN_PORT=8000
- REPO_DIR=/home/app/repo/ - REPO_DIR=/home/app/repo/
# deploy:
# resources:
# reservations:
# devices:
# - driver: nvidia
# device_ids: ['0']
# capabilities: [gpu]
req-dev.txt
+1
View File
@@ -37,6 +37,7 @@ pydantic==1.10.11
Pygments==2.15.1 Pygments==2.15.1
python-dateutil==2.8.2 python-dateutil==2.8.2
python-dotenv==1.0.0 python-dotenv==1.0.0
PyYAML==6.0
pyzmq==25.1.0 pyzmq==25.1.0
six==1.16.0 six==1.16.0
smmap==5.0.0 smmap==5.0.0
req-prod.txt
+1
View File
@@ -2,3 +2,4 @@ uvicorn==0.16.0
fastapi==0.99.1 fastapi==0.99.1
GitPython==3.1.31 GitPython==3.1.31
python-dotenv==1.0.0 python-dotenv==1.0.0
PyYAML==6.0
runner_script.yml
+11
View File
@@ -0,0 +1,11 @@
definitions:
steps:
step: &step-test
name: test
requirements: req-prod.txt
script:
- nvidia-smi
pipelines:
default:
<<: *step-test