updated ignored modules

flake8 compliant
added definition of external worker process
2023-07-14 14:56:04 +02:00 · 2023-07-14 14:55:42 +02:00 · 2023-07-14 14:54:13 +02:00 · 2023-07-14 14:53:46 +02:00 · 2023-07-14 14:53:20 +02:00 · 2023-07-14 14:52:49 +02:00
12 changed files with 380 additions and 0 deletions
@@ -0,0 +1,4 @@
 script: |
  echo "start of script"
  sleep 15
  echo "end of script"
@@ -9,3 +9,15 @@ Runner for one repository only. Does not try to run bash scripts.
 ## First Goal
 Get a training sessions running in the VM.
 ### Workflow
 - clone repo.
 - install dependencies.
 - run training script.
 - (add training evaluation)
 Needs: external storage for data and models.
@@ -0,0 +1,98 @@
 import uvicorn
 from fastapi import FastAPI, Request, HTTPException, BackgroundTasks
 from fastapi.responses import HTMLResponse
 from dotenv import load_dotenv
 import logging
 import os
 import httpx
 from utils import clone_repository, execute_script
 from setup_logging import setup_logging
 from gitea_request import GiteaRequest
 app = FastAPI()
 async def post_callback(url: str, payload: dict):
    async with httpx.AsyncClient() as client:
        await client.post(url, json=payload)
@app.get('/', response_class=HTMLResponse)
 async def root():
    content_str = """
        <html>
            <head>
                <title>Gitea Runner</title>
            </head>
            <body>
                <h1>Welcome to Gitea Runner</h1>
                <p>see /docs for more information</p>
            </body>
        </html>
    """
    return HTMLResponse(content=content_str, status_code=200)
@app.post('/test')
 async def test(request: Request):
    """ Test connection to server. """
    logging.debug('Content-Type: ', request.headers.get('content-type'))
    raise HTTPException(
        status_code=200,
        detail='test completed'
    )
@app.post('/')
 def handle_event(request: GiteaRequest, background_tasks: BackgroundTasks):
    """ Handle event from webhook. """
    # extract information
    commit_msg = request.head_commit['message']
    repo_url = request.repository['clone_url']
    # branch_name = request.ref.split('/')[-1]
    # stop early if asked
    if commit_msg.lower().startswith('[skip ci]'):
        raise HTTPException(
            status_code=200,
            detail='skipping ci'
        )
    # clone repository
    local_repo_dir = clone_repository(repo_url)
    # # locate runner script
    # runner_script_path = (
    #     local_repo_dir /
    #     '.gitea' /
    #     'gpu_runner' /
    #     'script.yml'
    # )
    # if not runner_script_path.exists():
    #     raise HTTPException(
    #         status_code=200,
    #         detail=f"{runner_script_path} does not exist"
    #     )
    # queue runner script
    background_tasks.add_task(
        execute_script,
        local_repo_dir
    )
    # send response
    raise HTTPException(
        status_code=202,
        detail='received task'
    )
 # consider to include link to page that show progress
 if __name__ == '__main__':
    load_dotenv('dev.env')
    setup_logging()
    listen_ip = os.getenv('LISTEN_IP', default='0.0.0.0')
    listen_port = int(os.getenv('LISTEN_PORT', default=8000))
    logging.info(f'starting FastAPI server ({listen_ip}:{listen_port})')
    uvicorn.run(
        app=app,
        host=listen_ip,
        port=listen_port,
        workers=1
    )
@@ -0,0 +1,7 @@
 [main]
 logger_level=debug
 repo_dir=repo
 [fastapi]
 listen_ip=0.0.0.0
 listen_port=8000
@@ -0,0 +1,15 @@
 from pydantic import BaseModel
 from typing import List
 class GiteaRequest(BaseModel):
    ref: str
    before: str
    after: str
    compare_url: str
    commits: List[dict]
    total_commits: int
    head_commit: dict
    repository: dict
    pusher: dict
    sender: dict
@@ -0,0 +1,40 @@
 import uvicorn
 from setup_logging import setup_logging
 from dotenv import load_dotenv
 from app import app
 import logging
 import os
 from configparser import ConfigParser
 from pathlib import Path
 import socket
 # from multiprocessing import Queue
 # from worker import Worker
 # load default values
 config = ConfigParser()
 config.read(Path(__file__).parent / 'defaults.ini')
 LISTEN_IP = config.get('fastapi', 'listen_ip')
 LISTEN_PORT = int(config.get('fastapi', 'listen_port'))
 if __name__ == '__main__':
    # setup
    load_dotenv()
    setup_logging()
    listen_ip = os.getenv('LISTEN_IP', default=LISTEN_IP)
    listen_port = int(os.getenv('LISTEN_PORT', default=LISTEN_PORT))
    own_ip = socket.gethostbyname(socket.gethostname())
    # # setup work queue
    # work_queue: Queue = Queue()
    # # setup worker process
    # worker = Worker(queue=work_queue)
    # start server
    logging.info(f'starting FastAPI server ({own_ip}:{listen_port})')
    # try:
    uvicorn.run(
        app=app,
        host=listen_ip,
        port=listen_port
    )
    # except KeyboardInterrupt:
    #     worker.stop()
@@ -0,0 +1,29 @@
 from pydantic import BaseModel
 from typing import List
 import yaml
 from pathlib import Path
 import logging
 class RunnerScript(BaseModel):
    script: List[str]
    @staticmethod
    def from_file(path: Path):
        assert path.exists()
        with open(path, 'r') as fh:
            content_dict = yaml.safe_load(fh)
        script_list = content_dict['script'].split('\n')
        logging.debug('finished')
        rs = RunnerScript.model_validate(  # type: ignore
            {
                'script': script_list
            }
        )
        return rs
 if __name__ == '__main__':
    path = Path('.gitea') / 'gpu_runner' / 'script.yml'
    rs = RunnerScript.from_file(path)
    print(rs)
@@ -0,0 +1,31 @@
 from dotenv import load_dotenv
 import logging
 from configparser import ConfigParser
 from pathlib import Path
 # load default values
 config = ConfigParser()
 config.read(Path(__file__).parent / 'defaults.ini')
 LOGGER_LEVEL = config.get('main', 'logger_level')
 def setup_logging(
    logger_level: str = LOGGER_LEVEL
 ):
    fmt = (
        '%(asctime)s | '
        '%(levelname)s | '
        '%(filename)s | '
        '%(funcName)s | '
        '%(message)s'
    )
    datefmt = '%Y-%m-%d %H:%M:%S'
    level = getattr(logging, logger_level.upper())
    logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
    logging.debug('finished')
 if __name__ == '__main__':
    load_dotenv('dev.env')
    setup_logging()
@@ -0,0 +1,70 @@
 from pydantic import AnyHttpUrl
 from pathlib import Path
 from git import Repo
 import logging
 import shutil
 import os
 from configparser import ConfigParser
 from runner_script import RunnerScript
 # load default values
 config = ConfigParser()
 config.read(Path(__file__).parent / 'defaults.ini')
 REPO_DIR = config.get('main', 'repo_dir')
 def clone_repository(
    repo_url: AnyHttpUrl,
    repo_dir: str = REPO_DIR
 ) -> Path:
    # extract repo name
    repo_name = repo_url.split('/')[-1].replace('.git', '')
    dst_dir = Path(repo_dir) / repo_name
    # prepare repo download dir
    if dst_dir.exists():
        shutil.rmtree(dst_dir)
    dst_dir.mkdir()
    # git clone repo
    Repo.clone_from(url=repo_url, to_path=dst_dir)
    logging.debug('finished')
    return dst_dir
 def execute_script(
    local_repo_dir: Path
 ) -> None:
    # locate the script
    script_path = (
        local_repo_dir /
        '.gitea' /
        'gpu_runner' /
        'script.yml'
    )
    # determine original working dir
    org_dir = os.getcwd()
    # initialise command list
    cmd_list = [
        f'cd {local_repo_dir.resolve()}',
        'virtualenv -p python3.10 venv',
        'source venv/bin/activate',
    ]
    # add in runner script
    rs = RunnerScript.from_file(path=script_path)
    for cmd in rs.script:
        cmd_list.append(cmd)
    # append cleanup commands
    cmd_list.append('deactivate')
    cmd_list.append(f'cd {org_dir}')
    cmd_list.append(f'rm -rf {local_repo_dir.resolve()}')
    # build command string
    cmd_str = ' && '.join(cmd_list)
    os.system(cmd_str)
    logging.debug('finished')
 if __name__ == '__main__':
    local_repo_dir = Path(__file__).parent.parent / 'repo' / 'gpu_runner'
    execute_script(
        local_repo_dir=local_repo_dir
    )
@@ -0,0 +1,40 @@
 from multiprocessing import Process, Queue
 import time
 from typing import List
 import os
 class Worker(object):
    def __init__(self, queue: Queue):
        self.run_flag = True
        self.queue = queue
        self.process = Process(
            target=self.handle_jobs,
            args=(self.queue,)
        )
        time.sleep(1)
        self.process.start()
    def stop(self):
        self.queue.put(['exit'])
        self.process.join()
    @staticmethod
    def handle_jobs(queue: Queue):
        while True:
            if queue.empty():
                time.sleep(1)
                continue
            cmd_list: List[str] = queue.get()
            print('received job:\n', cmd_list)
            if cmd_list[0].lower() == 'exit':
                break
            cmd_str = ' && '.join(cmd_list)
            os.system(cmd_str)
 if __name__ == '__main__':
    work_queue: Queue = Queue()
    worker = Worker(queue=work_queue)
    work_queue.put(['echo "hello world"', 'echo "hello world again"'])
    worker.stop()
@@ -0,0 +1,16 @@
 [mypy]
 [mypy-uvicorn.*]
 ignore_missing_imports = True
 [mypy-dotenv.*]
 ignore_missing_imports = True
 [mypy-git.*]
 ignore_missing_imports = True
 [mypy-fastapi.*]
 ignore_missing_imports = True
 [mypy-httpx.*]
 ignore_missing_imports = True
@@ -0,0 +1,18 @@
 annotated-types==0.5.0
 anyio==3.7.1
 click==8.1.4
 exceptiongroup==1.1.2
 fastapi==0.100.0
 gitdb==4.0.10
 GitPython==3.1.32
 h11==0.14.0
 idna==3.4
 pydantic==2.0.2
 pydantic_core==2.1.2
 python-dotenv==1.0.0
 PyYAML==6.0
 smmap==5.0.0
 sniffio==1.3.0
 starlette==0.27.0
 typing_extensions==4.7.1
 uvicorn==0.22.0
Author	SHA1	Message	Date
brb	cb2e45e5ae	updated ignored modules	2023-07-14 14:56:04 +02:00
brb	824215c37a	flake8 compliant	2023-07-14 14:55:42 +02:00
brb	0e39795b26	added definition of external worker process	2023-07-14 14:54:13 +02:00
brb	32e06d0d4e	removed usage of external worker process	2023-07-14 14:53:46 +02:00
brb	fd18d9450e	updated root post to start script execution in background task	2023-07-14 14:53:20 +02:00
brb	31ddd80fae	updated function to run directly via os.system and not in subprocess	2023-07-14 14:52:49 +02:00
brb	c6fccb9047	updated test script	2023-07-14 14:52:02 +02:00
brb	8841724740	updated execution of script	2023-07-13 13:12:11 +02:00
brb	db66de65b1	updated script	2023-07-13 13:10:45 +02:00
brb	6b165aa0ff	updated to find moved runner script	2023-07-13 12:54:32 +02:00
brb	aca204aa80	moved gpu_runner script	2023-07-13 12:53:05 +02:00
brb	1ef127d85f	updated execute runner script	2023-07-13 12:48:51 +02:00
brb	c6c56ef1cf	updated execute runner script	2023-07-13 12:46:53 +02:00
brb	a6a3464e7e	updated execute runner script	2023-07-13 12:46:24 +02:00
brb	86834b1d7c	updated execute runner script	2023-07-13 12:45:04 +02:00
brb	479674f807	updated repo_dir	2023-07-13 12:44:00 +02:00
brb	1e2e3e5e03	removed dev env vars	2023-07-13 12:42:03 +02:00
brb	70d78e8fa1	fixed bug with dev env vars	2023-07-13 12:39:35 +02:00
brb	ddb9e4f9b4	fixed bug	2023-07-13 12:37:32 +02:00
brb	e57f083b14	fixed bug	2023-07-13 12:35:52 +02:00
brb	b4f1eab752	added test printing of var	2023-07-13 12:35:11 +02:00
brb	320efedd50	added test printing of var	2023-07-13 12:33:55 +02:00
brb	176d09ca34	added test printing of var	2023-07-13 12:31:49 +02:00
brb	ca7061b0f6	added development env	2023-07-13 12:30:06 +02:00
brb	661baad416	updated values	2023-07-13 12:29:46 +02:00
brb	730717fc3b	changed app to accept calls directly to root	2023-07-13 12:24:23 +02:00
brb	cd636cce20	updated logging to show host ip and port	2023-07-13 12:21:24 +02:00
brb	6dd7d3f465	updated packages	2023-07-13 12:14:07 +02:00
brb	4a15e9fee0	flake8 compliant	2023-07-13 12:13:55 +02:00
brb	7a965e5dfc	mypy compliant	2023-07-13 12:11:49 +02:00
brb	2f69f49690	added mypy config	2023-07-13 12:11:43 +02:00
brb	1266204ecb	flake8 compliant	2023-07-13 11:53:14 +02:00
brb	6492b22219	defined default values	2023-07-13 11:48:25 +02:00
brb	8631722116	added loading of defaults	2023-07-13 11:48:10 +02:00
brb	2f014deb05	added loading of defaults	2023-07-13 11:47:50 +02:00
brb	3d8b240d9b	added logging	2023-07-13 11:38:20 +02:00
brb	5c2fc4022e	changed layout of file	2023-07-13 11:37:34 +02:00
brb	bf23ba3f38	added automatic parsing from file	2023-07-13 11:36:47 +02:00
brb	493dc874f0	added definition	2023-07-13 11:29:29 +02:00
brb	614a85c429	added definition	2023-07-13 11:29:14 +02:00
brb	9956d6a2c4	added definition	2023-07-13 11:23:52 +02:00
brb	25066d687e	added definition	2023-07-13 11:22:42 +02:00
brb	db8d478931	updated description	2023-07-13 11:18:57 +02:00