brian/gitea_runner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: brian/gitea_runner:7eb8e5d2fd4df3c102eb74382c517285d28c0e82
Branches Tags
brian/gitea_runner:main brian/gitea_runner:test
...
compare: brian/gitea_runner:test
Branches Tags
brian/gitea_runner:main brian/gitea_runner:test
38 Commits
7eb8e5d2fd ... test
Author SHA1 Message Date
brb 9513eb488b adapted to new definition 2023-07-06 15:58:25 +02:00
brb a51c49e8f4 adapted to new definition 2023-07-06 15:58:15 +02:00
brb dba476e78a updated definition 2023-07-06 15:58:03 +02:00
brb 87a738461c defined runner script 2023-07-06 15:54:20 +02:00
brb 40c9b7442d added printing of content from runner script 2023-07-06 15:53:34 +02:00
brb c1aa6f3c41 added function to read runner script 2023-07-06 15:53:15 +02:00
brb 9102d03ef4 created runner script 2023-07-06 15:53:00 +02:00
brb 5c21aa767f installed packages 2023-07-06 13:27:32 +02:00
brb 3187248fb9 [skip ci] fixed bug when accessing /test 2023-07-06 13:22:32 +02:00
brb ed05a8eca5 update root page 2023-07-06 13:20:24 +02:00
brb 02ec54fd16 updated response text 2023-07-06 13:15:16 +02:00
brb 9fffa861e7 added checking for presence of runner script 2023-07-06 13:10:13 +02:00
brb 3a73c9cf6e added more detailed responses to requests 2023-07-06 12:58:09 +02:00
brb 387e739a07 [skip ci] testing if ci is skipped correctly 2023-07-06 12:54:12 +02:00
brb 39d1ac1474 moved definition of class to separate file 2023-07-06 12:50:30 +02:00
brb 2aabc9ab37 added more logging 2023-07-06 12:20:02 +02:00
brb 2f964dabcc added more logging 2023-07-06 12:19:49 +02:00
brb 47576392a9 updated logging level 2023-07-06 12:19:33 +02:00
brb 31e771eb9f fixed bug when defining env vars 2023-07-06 11:59:21 +02:00
brb 935adf98ad removed unused file 2023-07-06 11:58:36 +02:00
brb 39b2afaf1f added creation of repo dir and moved env vars into docker compose 2023-07-06 11:58:21 +02:00
brb 416e9de7ce added port mapping 2023-07-06 11:54:01 +02:00
brb 70a087ee7b ignored python run files 2023-07-06 11:52:46 +02:00
brb 60112f1484 containerised service 2023-07-06 11:52:08 +02:00
brb f0010d739b added dotenv 2023-07-06 11:51:48 +02:00
brb 155a93456c updated downloading of repository 2023-07-06 11:09:28 +02:00
brb fdd968e3a3 updated var name 2023-07-06 11:08:27 +02:00
brb 8c9880ecca updated running 2023-07-06 11:06:46 +02:00
brb 56d0429bab added production environment variables 2023-07-06 11:05:28 +02:00
brb 78a2201a81 added logging and os.environ reading 2023-07-06 11:04:42 +02:00
brb 29a3545da4 added default value when getting os environ 2023-07-06 11:04:13 +02:00
brb 08eaaa3b5e added repo download dir 2023-07-06 10:58:35 +02:00
brb 65d2e850b5 moved file 2023-07-06 10:57:05 +02:00
brb b2bb34250b added production environment requirements 2023-07-06 10:56:17 +02:00
brb 9c357f52a1 deleted unused file 2023-07-06 10:55:55 +02:00
brb 4454ac1b46 updated code to latest version 2023-07-06 10:55:36 +02:00
brb 344f29b77c removed unused imports 2023-07-06 10:55:21 +02:00
brb fe64f2d8a3 added automatic deletion of preexistent repo dir 2023-07-05 21:24:49 +02:00
15 changed files with 194 additions and 124 deletions
Expand all files Collapse all files
.gitignore
+1
View File
@@ -0,0 +1 @@
__pycache__
Dockerfile
+4 -9
View File
@@ -3,13 +3,7 @@
########### ###########
# pull official base image # pull official base image
FROM python:alpine as builder FROM dvcorg/cml:0-dvc2-base1-gpu as builder
# update system
RUN apk update && \
apk add git && \
apk add rsync && \
apk add docker
# set work directory # set work directory
WORKDIR /usr/src/app WORKDIR /usr/src/app
@@ -20,7 +14,7 @@ ENV PYTHONUNBUFFERED 1
# install dependencies # install dependencies
RUN pip install --upgrade pip RUN pip install --upgrade pip
COPY requirements.txt . COPY req-prod.txt requirements.txt
RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requirements.txt
######### #########
@@ -28,12 +22,13 @@ RUN pip wheel --no-cache-dir --no-deps --wheel-dir /usr/src/app/wheels -r requir
######### #########
# pull official base image # pull official base image
FROM python:alpine FROM dvcorg/cml:0-dvc2-base1-gpu
# create app home # create app home
ENV APP_HOME = /home/app ENV APP_HOME = /home/app
RUN mkdir -p $APP_HOME RUN mkdir -p $APP_HOME
WORKDIR $APP_HOME WORKDIR $APP_HOME
RUN mkdir -p /home/app/repo
# install packages from builder # install packages from builder
COPY --from=builder /usr/src/app/wheels /wheels COPY --from=builder /usr/src/app/wheels /wheels
app.py
-53
View File
@@ -1,53 +0,0 @@
# from waitress import serve
# from flask import Flask, request, jsonify
import uvicorn
from fastapi import FastAPI, Request
from dotenv import load_dotenv
import logging
from pydantic import BaseModel
from typing import List
from utils import clone_repository
class GiteaRequest(BaseModel):
ref: str
before: str
after: str
compare_url: str
commits: List[dict]
total_commits: int
head_commit: dict
repository: dict
pusher: dict
sender: dict
app = FastAPI()
@app.get('/')
async def root():
return '<h1>Welcome to Gitea Runner</h1>'
@app.post('/test')
async def test(request: Request):
""" Test connection to server. """
logging.debug('Content-Type: ', request.headers.get('content-type'))
return {'status': 'success'}
@app.post('/event')
def handle_event(request: GiteaRequest):
""" Handle event from webhook. """
# stop early if allowed
commit_msg = request.head_commit['message']
if commit_msg.lower().startswith('[skip ci]'):
return {'status': 'received task'}
# clone repository
repo_url = request.repository['clone_url']
local_repo_dir = clone_repository(repo_url)
print('local_repo_dir: ', local_repo_dir)
return {'status': 'success'}
if __name__ == '__main__':
uvicorn.run(
app=app,
host='0.0.0.0'
)
code/app.py
+75 -30
View File
@@ -1,38 +1,83 @@
from flask import Flask, request, jsonify import uvicorn
import os from fastapi import FastAPI, Request, HTTPException
from fastapi.responses import HTMLResponse
from dotenv import load_dotenv from dotenv import load_dotenv
from ipaddress import ip_address, ip_network
import logging import logging
from typing import List
import os
load_dotenv() from utils import clone_repository, load_runner_script
app = Flask(__name__) from setup_logging import setup_logging
from gitea_request import GiteaRequest
@app.before_request app = FastAPI()
def check_authorized():
@app.get('/', response_class=HTMLResponse)
async def root():
content_str = """
<html>
<head>
<title>Gitea Runner</title>
</head>
<body>
<h1>Welcome to Gitea Runner</h1>
<p>see /docs for more information</p>
</body>
</html>
""" """
Ensure request from allowed ip. return HTMLResponse(content=content_str, status_code=200)
"""
# prepare variables
assert 'ALLOWED_IP_RANGE' in os.environ
allowed_ip_range = ip_network(os.getenv['ALLOWED_IP_RANGE'])
requesting_ip = ip_address(request.remote_addr)
# check that requesting ip is allowed
if requesting_ip not in allowed_ip_range:
logging.info(f'received request from unauthorised ip: {request.remote_addr}')
return jsonify(status='forbidden'), 403
logging.info(f'received request from {request.remote_addr}')
@app.before_request @app.post('/test')
def check_media_type(): async def test(request: Request):
""" """ Test connection to server. """
Ensure correct Content-Type of request logging.debug('Content-Type: ', request.headers.get('content-type'))
""" raise HTTPException(
if not request.headers.get('Content-Type').lower().startswith('application/json'): status_code=200,
logging.error(f'received request with wrong content type') detail='test completed'
return jsonify(status='unsupported media type'), 415 )
@app.route('/test', methods=['POST']) @app.post('/event')
def test(): def handle_event(request: GiteaRequest):
logging.debug(request.get_json(force=True)) """ Handle event from webhook. """
return jsonify(status='success', sender=request.remote_addr) # stop early if asked
commit_msg = request.head_commit['message']
if commit_msg.lower().startswith('[skip ci]'):
raise HTTPException(
status_code=200,
detail='skipping ci'
)
# clone repository
repo_url = request.repository['clone_url']
logging.info('cloning repository: {repo_url}')
local_repo_dir = clone_repository(repo_url)
logging.info(f'local_repo_dir: {local_repo_dir}')
# locate runner script
runner_script_path = local_repo_dir / 'runner_script.yml'
if not runner_script_path.exists():
raise HTTPException(
status_code=200,
detail="no runner_script.yml in repository root"
)
# load runner commands
runner_script = load_runner_script(path=runner_script_path)
for pipeline_name, pipeline_step in runner_script.pipelines.items():
print(pipeline_name)
print(pipeline_step)
# send response
raise HTTPException(
status_code=202,
detail='received task'
)
if __name__ == '__main__':
load_dotenv('dev.env')
setup_logging()
listen_ip = os.getenv('LISTEN_IP', default='0.0.0.0')
listen_port = int(os.getenv('LISTEN_PORT', default=8000))
logging.info(f'starting FastAPI server ({listen_ip}:{listen_port})')
uvicorn.run(
app=app,
host=listen_ip,
port=listen_port
)
code/gitea_request.py
+14
View File
@@ -0,0 +1,14 @@
from pydantic import BaseModel
from typing import List
class GiteaRequest(BaseModel):
ref: str
before: str
after: str
compare_url: str
commits: List[dict]
total_commits: int
head_commit: dict
repository: dict
pusher: dict
sender: dict
code/main.py
+11 -13
View File
@@ -1,20 +1,18 @@
"""
Run tasks based on webhooks configured in Gitea.
Command-line optinos:
--debug, -d Send more detailed log output to console.
"""
from dotenv import load_dotenv from dotenv import load_dotenv
from waitress import serve import uvicorn
from setup_logging import setup_logging from setup_logging import setup_logging
from app import app from app import app
import logging
import os
if __name__ == '__main__': if __name__ == '__main__':
# setup # setup
load_dotenv()
setup_logging() setup_logging()
# serve app listen_ip = os.getenv('LISTEN_IP', default='0.0.0.0')
serve(app, host='0.0.0.0', port=1706) listen_port = int(os.getenv('LISTEN_PORT', default=8000))
logging.info(f'starting FastAPI server ({listen_ip}:{listen_port})')
uvicorn.run(
app=app,
host=listen_ip,
port=listen_port
)
code/runner_script.py
+22
View File
@@ -0,0 +1,22 @@
from pydantic import BaseModel
from typing import List, Dict
import yaml
from pathlib import Path
class Pipeline(BaseModel):
name: str
requirements: str
script: List[str]
class RunnerScript(BaseModel):
pipelines: Dict[str, Pipeline]
if __name__ == '__main__':
path = Path(__file__).parent.parent / 'runner_script.yml'
with open(path, 'r') as fh:
script = yaml.safe_load(fh)
runner_script = RunnerScript.parse_obj(script)
print(runner_script)
for k, v in runner_script.pipelines.items():
print(k)
print(v)
code/setup_logging.py
+1 -4
View File
@@ -5,10 +5,7 @@ import os
def setup_logging(): def setup_logging():
fmt = '%(asctime)s | %(levelname)s | %(filename)s | %(funcName)s | %(message)s' fmt = '%(asctime)s | %(levelname)s | %(filename)s | %(funcName)s | %(message)s'
datefmt = '%Y-%m-%d %H:%M:%S' datefmt = '%Y-%m-%d %H:%M:%S'
if not 'LOGGER_LEVEL' in os.environ: logger_level = os.getenv('LOGGER_LEVEL', default='info')
logger_level = 'info'
else:
logger_level = os.getenv('LOGGER_LEVEL')
level = getattr(logging, logger_level.upper()) level = getattr(logging, logger_level.upper())
logging.basicConfig(format=fmt, datefmt=datefmt, level=level) logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
logging.debug('finished') logging.debug('finished')
code/utils.py
+32
View File
@@ -0,0 +1,32 @@
from pydantic import AnyHttpUrl
from typing import List
from pathlib import Path
from git import Repo
import logging
import shutil
import os
import yaml
from runner_script import RunnerScript
def clone_repository(repo_url: AnyHttpUrl) -> Path:
# extract repo name
repo_name = repo_url.split('/')[-1].replace('.git', '')
repo_dir = os.getenv('REPO_DIR', default=None)
assert repo_dir is not None
dst_dir = Path(repo_dir) / repo_name
# prepare repo download dir
if dst_dir.exists():
shutil.rmtree(dst_dir)
dst_dir.mkdir()
# git clone repo
Repo.clone_from(url=repo_url, to_path=dst_dir)
logging.debug('finished')
return dst_dir
def load_runner_script(path: Path) -> RunnerScript:
assert path.exists()
with open(path, 'r') as fh:
script = yaml.safe_load(fh)
runner_script = RunnerScript.parse_obj(script)
return runner_script
dev.env
+2 -1
View File
@@ -1,3 +1,4 @@
LOGGER_LEVEL = debug LOGGER_LEVEL = debug
LISTEN_IP = 0.0.0.0 LISTEN_IP = 0.0.0.0
LISTEN_PORT = 1706 LISTEN_PORT = 8000
REPO_DIR = /usr/src/app/repo/
docker-compose.yml
+13
View File
@@ -0,0 +1,13 @@
version: "3.9"
services:
gitea-runner:
build: .
container_name: gitea-runner
restart: unless-stopped
ports:
- "8000:8000"
environment:
- LOGGER_LEVEL=debug
- LISTEN_IP=0.0.0.0
- LISTEN_PORT=8000
- REPO_DIR=/home/app/repo/
req-dev.txt
+1
View File
@@ -37,6 +37,7 @@ pydantic==1.10.11
Pygments==2.15.1 Pygments==2.15.1
python-dateutil==2.8.2 python-dateutil==2.8.2
python-dotenv==1.0.0 python-dotenv==1.0.0
PyYAML==6.0
pyzmq==25.1.0 pyzmq==25.1.0
six==1.16.0 six==1.16.0
smmap==5.0.0 smmap==5.0.0
req-prod.txt
+5
View File
@@ -0,0 +1,5 @@
uvicorn==0.16.0
fastapi==0.99.1
GitPython==3.1.31
python-dotenv==1.0.0
PyYAML==6.0
runner_script.yml
+12
View File
@@ -0,0 +1,12 @@
definitions:
steps:
step: &step-test
name: test
requirements: req-prod.txt
script:
- command arg1 arg2
- command2 arg1 arg2
pipelines:
default:
<<: *step-test
utils.py
-13
View File
@@ -1,13 +0,0 @@
from pydantic import AnyHttpUrl
from pathlib import Path
from git import Repo
def clone_repository(repo_url: AnyHttpUrl) -> Path:
# extract repo name
repo_name = repo_url.split('/')[-1].replace('.git', '')
dst_dir = Path(__file__).parent / repo_name
# prepare repo download dir
dst_dir.mkdir(exist_ok=True)
# git clone repo
Repo.clone_from(url=repo_url, to_path=dst_dir)
return dst_dir