brian/gitea_runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: brian/gitea_runner:416e9de7ce2efa460f86c03297bf6e59878edf1f
Branches Tags
brian/gitea_runner:main brian/gitea_runner:test
...
compare: brian/gitea_runner:main
Branches Tags
brian/gitea_runner:main brian/gitea_runner:test
34 Commits
416e9de7ce ... main
Author SHA1 Message Date
brb c19fc58a25 deleted unused code 2023-09-04 14:43:19 +02:00
brb bd5dc8fbc4 changed function to execute commands from runner script 2023-07-07 09:46:12 +02:00
brb f4a81b4b9b added and commented out code for passing through gpu 2023-07-07 09:45:45 +02:00
brb 477389d353 updated script 2023-07-06 17:34:08 +02:00
brb 90a44e8b6e updated running method 2023-07-06 17:32:41 +02:00
brb 36d2233567 updated exception handling when running commands from runner script 2023-07-06 17:30:40 +02:00
brb 2fc979a845 updated to run as user instead of root 2023-07-06 17:28:38 +02:00
brb 7218004f2f updated to run actual function 2023-07-06 17:22:21 +02:00
brb 75fd57391d added pip install requirements 2023-07-06 17:22:09 +02:00
brb a82e0a8f10 fixed logging bug 2023-07-06 16:44:49 +02:00
brb d1864c7030 added logging 2023-07-06 16:44:37 +02:00
brb 43e4b9788a updated to handle default and custom branch names 2023-07-06 16:42:54 +02:00
brb 84b409356b added detection of branch name 2023-07-06 16:07:27 +02:00
brb 9513eb488b adapted to new definition 2023-07-06 15:58:25 +02:00
brb a51c49e8f4 adapted to new definition 2023-07-06 15:58:15 +02:00
brb dba476e78a updated definition 2023-07-06 15:58:03 +02:00
brb 87a738461c defined runner script 2023-07-06 15:54:20 +02:00
brb 40c9b7442d added printing of content from runner script 2023-07-06 15:53:34 +02:00
brb c1aa6f3c41 added function to read runner script 2023-07-06 15:53:15 +02:00
brb 9102d03ef4 created runner script 2023-07-06 15:53:00 +02:00
brb 5c21aa767f installed packages 2023-07-06 13:27:32 +02:00
brb 3187248fb9 [skip ci] fixed bug when accessing /test 2023-07-06 13:22:32 +02:00
brb ed05a8eca5 update root page 2023-07-06 13:20:24 +02:00
brb 02ec54fd16 updated response text 2023-07-06 13:15:16 +02:00
brb 9fffa861e7 added checking for presence of runner script 2023-07-06 13:10:13 +02:00
brb 3a73c9cf6e added more detailed responses to requests 2023-07-06 12:58:09 +02:00
brb 387e739a07 [skip ci] testing if ci is skipped correctly 2023-07-06 12:54:12 +02:00
brb 39d1ac1474 moved definition of class to separate file 2023-07-06 12:50:30 +02:00
brb 2aabc9ab37 added more logging 2023-07-06 12:20:02 +02:00
brb 2f964dabcc added more logging 2023-07-06 12:19:49 +02:00
brb 47576392a9 updated logging level 2023-07-06 12:19:33 +02:00
brb 31e771eb9f fixed bug when defining env vars 2023-07-06 11:59:21 +02:00
brb 935adf98ad removed unused file 2023-07-06 11:58:36 +02:00
brb 39b2afaf1f added creation of repo dir and moved env vars into docker compose 2023-07-06 11:58:21 +02:00
10 changed files with 163 additions and 36 deletions
Expand all files Collapse all files
Dockerfile
+14 -4
View File
@@ -24,10 +24,13 @@ RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requir
# pull official base image # pull official base image
FROM dvcorg/cml:0-dvc2-base1-gpu FROM dvcorg/cml:0-dvc2-base1-gpu
# create app home # create home directory and app user
ENV APP_HOME = /home/app RUN mkdir -p /home/app && \
RUN mkdir -p $APP_HOME addgroup --system app && \
WORKDIR $APP_HOME adduser --system --group app
# add folder to store repo in
RUN mkdir -p /home/app/repo
# install packages from builder # install packages from builder
COPY --from=builder /usr/src/app/wheels /wheels COPY --from=builder /usr/src/app/wheels /wheels
@@ -35,7 +38,14 @@ COPY --from=builder /usr/src/app/requirements.txt requirements.txt
RUN pip install --no-cache /wheels/* RUN pip install --no-cache /wheels/*
# copy in files # copy in files
WORKDIR /home/app
COPY ./code . COPY ./code .
# change ownership to app user
RUN chown -R app:app /home/app
# change to the app user
USER app
# start execution # start execution
ENTRYPOINT [ "python", "main.py" ] ENTRYPOINT [ "python", "main.py" ]
code/app.py
+71 -24
View File
@@ -1,50 +1,97 @@
import uvicorn import uvicorn
from fastapi import FastAPI, Request from fastapi import FastAPI, Request, HTTPException
from fastapi.responses import HTMLResponse
from dotenv import load_dotenv from dotenv import load_dotenv
import logging import logging
from pydantic import BaseModel
from typing import List from typing import List
import os import os
from utils import clone_repository from utils import clone_repository, load_runner_script
from setup_logging import setup_logging from setup_logging import setup_logging
from gitea_request import GiteaRequest
class GiteaRequest(BaseModel):
ref: str
before: str
after: str
compare_url: str
commits: List[dict]
total_commits: int
head_commit: dict
repository: dict
pusher: dict
sender: dict
app = FastAPI() app = FastAPI()
@app.get('/') @app.get('/', response_class=HTMLResponse)
async def root(): async def root():
return '<h1>Welcome to Gitea Runsner</h1>' content_str = """
<html>
<head>
<title>Gitea Runner</title>
</head>
<body>
<h1>Welcome to Gitea Runner</h1>
<p>see /docs for more information</p>
</body>
</html>
"""
return HTMLResponse(content=content_str, status_code=200)
@app.post('/test') @app.post('/test')
async def test(request: Request): async def test(request: Request):
""" Test connection to server. """ """ Test connection to server. """
logging.debug('Content-Type: ', request.headers.get('content-type')) logging.debug('Content-Type: ', request.headers.get('content-type'))
return {'status': 'success'} raise HTTPException(
status_code=200,
detail='test completed'
)
@app.post('/event') @app.post('/event')
def handle_event(request: GiteaRequest): def handle_event(request: GiteaRequest):
""" Handle event from webhook. """ """ Handle event from webhook. """
# stop early if allowed # extract information
commit_msg = request.head_commit['message'] commit_msg = request.head_commit['message']
if commit_msg.lower().startswith('[skip ci]'):
return {'status': 'received task'}
# clone repository
repo_url = request.repository['clone_url'] repo_url = request.repository['clone_url']
branch_name = request.ref.split('/')[-1]
# stop early if asked
if commit_msg.lower().startswith('[skip ci]'):
raise HTTPException(
status_code=200,
detail='skipping ci'
)
# clone repository
local_repo_dir = clone_repository(repo_url) local_repo_dir = clone_repository(repo_url)
print('local_repo_dir: ', local_repo_dir) # locate runner script
return {'status': 'success'} runner_script_path = local_repo_dir / 'runner_script.yml'
if not runner_script_path.exists():
raise HTTPException(
status_code=200,
detail="no runner_script.yml in repository root"
)
# load runner script
runner_script = load_runner_script(path=runner_script_path)
# determine pipeline to run
if branch_name not in runner_script.pipelines.keys():
# get default pipeline
if 'default' not in runner_script.pipelines.keys():
raise HTTPException(
status_code=404,
detail='no pipelines defined in runner_script.yml'
)
pipeline = runner_script.pipelines['default']
else:
# get custom pipeline
pipeline = runner_script.pipelines[branch_name]
logging.info(f'running pipeline {pipeline.name}')
# TODO: create virtualenv?
# install requirements
assert pipeline.requirements is not None
requirements_path = local_repo_dir / pipeline.requirements
os.system(f'pip install -r {requirements_path}')
for cmd in pipeline.script:
try:
os.system(cmd)
except Exception as e:
raise HTTPException(
status_code=503,
detail=f'failed running {cmd}: {e}'
)
# send response
raise HTTPException(
status_code=202,
detail='received task'
)
# consider to include link to page that show progress
if __name__ == '__main__': if __name__ == '__main__':
load_dotenv('dev.env') load_dotenv('dev.env')
code/gitea_request.py
+14
View File
@@ -0,0 +1,14 @@
from pydantic import BaseModel
from typing import List
class GiteaRequest(BaseModel):
ref: str
before: str
after: str
compare_url: str
commits: List[dict]
total_commits: int
head_commit: dict
repository: dict
pusher: dict
sender: dict
code/runner_script.py
+22
View File
@@ -0,0 +1,22 @@
from pydantic import BaseModel
from typing import List, Dict
import yaml
from pathlib import Path
class Pipeline(BaseModel):
name: str
requirements: str
script: List[str]
class RunnerScript(BaseModel):
pipelines: Dict[str, Pipeline]
if __name__ == '__main__':
path = Path(__file__).parent.parent / 'runner_script.yml'
with open(path, 'r') as fh:
script = yaml.safe_load(fh)
runner_script = RunnerScript.parse_obj(script)
print(runner_script)
for k, v in runner_script.pipelines.items():
print(k)
print(v)
code/utils.py
+14
View File
@@ -1,8 +1,12 @@
from pydantic import AnyHttpUrl from pydantic import AnyHttpUrl
from typing import List
from pathlib import Path from pathlib import Path
from git import Repo from git import Repo
import logging
import shutil import shutil
import os import os
import yaml
from runner_script import RunnerScript
def clone_repository(repo_url: AnyHttpUrl) -> Path: def clone_repository(repo_url: AnyHttpUrl) -> Path:
# extract repo name # extract repo name
@@ -16,4 +20,14 @@ def clone_repository(repo_url: AnyHttpUrl) -> Path:
dst_dir.mkdir() dst_dir.mkdir()
# git clone repo # git clone repo
Repo.clone_from(url=repo_url, to_path=dst_dir) Repo.clone_from(url=repo_url, to_path=dst_dir)
logging.debug('finished')
return dst_dir return dst_dir
def load_runner_script(path: Path) -> RunnerScript:
assert path.exists()
with open(path, 'r') as fh:
script = yaml.safe_load(fh)
runner_script = RunnerScript.parse_obj(script)
logging.debug('finished')
return runner_script
docker-compose.yml
+12 -1
View File
@@ -4,6 +4,17 @@ services:
build: . build: .
container_name: gitea-runner container_name: gitea-runner
restart: unless-stopped restart: unless-stopped
env_file: prod.env
ports: ports:
- "8000:8000" - "8000:8000"
environment:
- LOGGER_LEVEL=debug
- LISTEN_IP=0.0.0.0
- LISTEN_PORT=8000
- REPO_DIR=/home/app/repo/
# deploy:
# resources:
# reservations:
# devices:
# - driver: nvidia
# device_ids: ['0']
# capabilities: [gpu]
prod.env
-4
View File
@@ -1,4 +0,0 @@
LOGGER_LEVEL = info
LISTEN_IP = 0.0.0.0
LISTEN_PORT = 8000
REPO_DIR = /usr/src/app/repo/
req-dev.txt
+1
View File
@@ -37,6 +37,7 @@ pydantic==1.10.11
Pygments==2.15.1 Pygments==2.15.1
python-dateutil==2.8.2 python-dateutil==2.8.2
python-dotenv==1.0.0 python-dotenv==1.0.0
PyYAML==6.0
pyzmq==25.1.0 pyzmq==25.1.0
six==1.16.0 six==1.16.0
smmap==5.0.0 smmap==5.0.0
req-prod.txt
+1
View File
@@ -2,3 +2,4 @@ uvicorn==0.16.0
fastapi==0.99.1 fastapi==0.99.1
GitPython==3.1.31 GitPython==3.1.31
python-dotenv==1.0.0 python-dotenv==1.0.0
PyYAML==6.0
runner_script.yml
+11
View File
@@ -0,0 +1,11 @@
definitions:
steps:
step: &step-test
name: test
requirements: req-prod.txt
script:
- nvidia-smi
pipelines:
default:
<<: *step-test