updated ignored modules

flake8 compliant
added definition of external worker process
2023-07-14 14:56:04 +02:00 · 2023-07-14 14:55:42 +02:00 · 2023-07-14 14:54:13 +02:00 · 2023-07-14 14:53:46 +02:00 · 2023-07-14 14:53:20 +02:00 · 2023-07-14 14:52:49 +02:00
12 changed files with 380 additions and 0 deletions
@@ -0,0 +1,4 @@
+script: |
+  echo "start of script"
+  sleep 15
+  echo "end of script"
@@ -9,3 +9,15 @@ Runner for one repository only. Does not try to run bash scripts.
 ## First Goal

 Get a training sessions running in the VM.
+
+### Workflow
+
+- clone repo.
+
+- install dependencies.
+
+- run training script.
+
+- (add training evaluation)
+
+Needs: external storage for data and models.
@@ -0,0 +1,98 @@
+import uvicorn
+from fastapi import FastAPI, Request, HTTPException, BackgroundTasks
+from fastapi.responses import HTMLResponse
+from dotenv import load_dotenv
+import logging
+import os
+import httpx
+
+from utils import clone_repository, execute_script
+from setup_logging import setup_logging
+from gitea_request import GiteaRequest
+
+app = FastAPI()
+
+
+async def post_callback(url: str, payload: dict):
+    async with httpx.AsyncClient() as client:
+        await client.post(url, json=payload)
+
+
+@app.get('/', response_class=HTMLResponse)
+async def root():
+    content_str = """
+        <html>
+            <head>
+                <title>Gitea Runner</title>
+            </head>
+            <body>
+                <h1>Welcome to Gitea Runner</h1>
+                <p>see /docs for more information</p>
+            </body>
+        </html>
+    """
+    return HTMLResponse(content=content_str, status_code=200)
+
+
+@app.post('/test')
+async def test(request: Request):
+    """ Test connection to server. """
+    logging.debug('Content-Type: ', request.headers.get('content-type'))
+    raise HTTPException(
+        status_code=200,
+        detail='test completed'
+    )
+
+
+@app.post('/')
+def handle_event(request: GiteaRequest, background_tasks: BackgroundTasks):
+    """ Handle event from webhook. """
+    # extract information
+    commit_msg = request.head_commit['message']
+    repo_url = request.repository['clone_url']
+    # branch_name = request.ref.split('/')[-1]
+    # stop early if asked
+    if commit_msg.lower().startswith('[skip ci]'):
+        raise HTTPException(
+            status_code=200,
+            detail='skipping ci'
+        )
+    # clone repository
+    local_repo_dir = clone_repository(repo_url)
+    # # locate runner script
+    # runner_script_path = (
+    #     local_repo_dir /
+    #     '.gitea' /
+    #     'gpu_runner' /
+    #     'script.yml'
+    # )
+    # if not runner_script_path.exists():
+    #     raise HTTPException(
+    #         status_code=200,
+    #         detail=f"{runner_script_path} does not exist"
+    #     )
+    # queue runner script
+    background_tasks.add_task(
+        execute_script,
+        local_repo_dir
+    )
+    # send response
+    raise HTTPException(
+        status_code=202,
+        detail='received task'
+    )
+# consider to include link to page that show progress
+
+
+if __name__ == '__main__':
+    load_dotenv('dev.env')
+    setup_logging()
+    listen_ip = os.getenv('LISTEN_IP', default='0.0.0.0')
+    listen_port = int(os.getenv('LISTEN_PORT', default=8000))
+    logging.info(f'starting FastAPI server ({listen_ip}:{listen_port})')
+    uvicorn.run(
+        app=app,
+        host=listen_ip,
+        port=listen_port,
+        workers=1
+    )
@@ -0,0 +1,7 @@
+[main]
+logger_level=debug
+repo_dir=repo
+
+[fastapi]
+listen_ip=0.0.0.0
+listen_port=8000
@@ -0,0 +1,15 @@
+from pydantic import BaseModel
+from typing import List
+
+
+class GiteaRequest(BaseModel):
+    ref: str
+    before: str
+    after: str
+    compare_url: str
+    commits: List[dict]
+    total_commits: int
+    head_commit: dict
+    repository: dict
+    pusher: dict
+    sender: dict
@@ -0,0 +1,40 @@
+import uvicorn
+from setup_logging import setup_logging
+from dotenv import load_dotenv
+from app import app
+import logging
+import os
+from configparser import ConfigParser
+from pathlib import Path
+import socket
+# from multiprocessing import Queue
+# from worker import Worker
+
+# load default values
+config = ConfigParser()
+config.read(Path(__file__).parent / 'defaults.ini')
+LISTEN_IP = config.get('fastapi', 'listen_ip')
+LISTEN_PORT = int(config.get('fastapi', 'listen_port'))
+
+
+if __name__ == '__main__':
+    # setup
+    load_dotenv()
+    setup_logging()
+    listen_ip = os.getenv('LISTEN_IP', default=LISTEN_IP)
+    listen_port = int(os.getenv('LISTEN_PORT', default=LISTEN_PORT))
+    own_ip = socket.gethostbyname(socket.gethostname())
+    # # setup work queue
+    # work_queue: Queue = Queue()
+    # # setup worker process
+    # worker = Worker(queue=work_queue)
+    # start server
+    logging.info(f'starting FastAPI server ({own_ip}:{listen_port})')
+    # try:
+    uvicorn.run(
+        app=app,
+        host=listen_ip,
+        port=listen_port
+    )
+    # except KeyboardInterrupt:
+    #     worker.stop()
@@ -0,0 +1,29 @@
+from pydantic import BaseModel
+from typing import List
+import yaml
+from pathlib import Path
+import logging
+
+
+class RunnerScript(BaseModel):
+    script: List[str]
+
+    @staticmethod
+    def from_file(path: Path):
+        assert path.exists()
+        with open(path, 'r') as fh:
+            content_dict = yaml.safe_load(fh)
+        script_list = content_dict['script'].split('\n')
+        logging.debug('finished')
+        rs = RunnerScript.model_validate(  # type: ignore
+            {
+                'script': script_list
+            }
+        )
+        return rs
+
+
+if __name__ == '__main__':
+    path = Path('.gitea') / 'gpu_runner' / 'script.yml'
+    rs = RunnerScript.from_file(path)
+    print(rs)
@@ -0,0 +1,31 @@
+from dotenv import load_dotenv
+import logging
+from configparser import ConfigParser
+from pathlib import Path
+
+
+# load default values
+config = ConfigParser()
+config.read(Path(__file__).parent / 'defaults.ini')
+LOGGER_LEVEL = config.get('main', 'logger_level')
+
+
+def setup_logging(
+    logger_level: str = LOGGER_LEVEL
+):
+    fmt = (
+        '%(asctime)s | '
+        '%(levelname)s | '
+        '%(filename)s | '
+        '%(funcName)s | '
+        '%(message)s'
+    )
+    datefmt = '%Y-%m-%d %H:%M:%S'
+    level = getattr(logging, logger_level.upper())
+    logging.basicConfig(format=fmt, datefmt=datefmt, level=level)
+    logging.debug('finished')
+
+
+if __name__ == '__main__':
+    load_dotenv('dev.env')
+    setup_logging()
@@ -0,0 +1,70 @@
+from pydantic import AnyHttpUrl
+from pathlib import Path
+from git import Repo
+import logging
+import shutil
+import os
+from configparser import ConfigParser
+from runner_script import RunnerScript
+
+
+# load default values
+config = ConfigParser()
+config.read(Path(__file__).parent / 'defaults.ini')
+REPO_DIR = config.get('main', 'repo_dir')
+
+
+def clone_repository(
+    repo_url: AnyHttpUrl,
+    repo_dir: str = REPO_DIR
+) -> Path:
+    # extract repo name
+    repo_name = repo_url.split('/')[-1].replace('.git', '')
+    dst_dir = Path(repo_dir) / repo_name
+    # prepare repo download dir
+    if dst_dir.exists():
+        shutil.rmtree(dst_dir)
+    dst_dir.mkdir()
+    # git clone repo
+    Repo.clone_from(url=repo_url, to_path=dst_dir)
+    logging.debug('finished')
+    return dst_dir
+
+
+def execute_script(
+    local_repo_dir: Path
+) -> None:
+    # locate the script
+    script_path = (
+        local_repo_dir /
+        '.gitea' /
+        'gpu_runner' /
+        'script.yml'
+    )
+    # determine original working dir
+    org_dir = os.getcwd()
+    # initialise command list
+    cmd_list = [
+        f'cd {local_repo_dir.resolve()}',
+        'virtualenv -p python3.10 venv',
+        'source venv/bin/activate',
+    ]
+    # add in runner script
+    rs = RunnerScript.from_file(path=script_path)
+    for cmd in rs.script:
+        cmd_list.append(cmd)
+    # append cleanup commands
+    cmd_list.append('deactivate')
+    cmd_list.append(f'cd {org_dir}')
+    cmd_list.append(f'rm -rf {local_repo_dir.resolve()}')
+    # build command string
+    cmd_str = ' && '.join(cmd_list)
+    os.system(cmd_str)
+    logging.debug('finished')
+
+
+if __name__ == '__main__':
+    local_repo_dir = Path(__file__).parent.parent / 'repo' / 'gpu_runner'
+    execute_script(
+        local_repo_dir=local_repo_dir
+    )
@@ -0,0 +1,40 @@
+from multiprocessing import Process, Queue
+import time
+from typing import List
+import os
+
+
+class Worker(object):
+    def __init__(self, queue: Queue):
+        self.run_flag = True
+        self.queue = queue
+        self.process = Process(
+            target=self.handle_jobs,
+            args=(self.queue,)
+        )
+        time.sleep(1)
+        self.process.start()
+
+    def stop(self):
+        self.queue.put(['exit'])
+        self.process.join()
+
+    @staticmethod
+    def handle_jobs(queue: Queue):
+        while True:
+            if queue.empty():
+                time.sleep(1)
+                continue
+            cmd_list: List[str] = queue.get()
+            print('received job:\n', cmd_list)
+            if cmd_list[0].lower() == 'exit':
+                break
+            cmd_str = ' && '.join(cmd_list)
+            os.system(cmd_str)
+
+
+if __name__ == '__main__':
+    work_queue: Queue = Queue()
+    worker = Worker(queue=work_queue)
+    work_queue.put(['echo "hello world"', 'echo "hello world again"'])
+    worker.stop()
@@ -0,0 +1,16 @@
+[mypy]
+
+[mypy-uvicorn.*]
+ignore_missing_imports = True
+
+[mypy-dotenv.*]
+ignore_missing_imports = True
+
+[mypy-git.*]
+ignore_missing_imports = True
+
+[mypy-fastapi.*]
+ignore_missing_imports = True
+
+[mypy-httpx.*]
+ignore_missing_imports = True
@@ -0,0 +1,18 @@
+annotated-types==0.5.0
+anyio==3.7.1
+click==8.1.4
+exceptiongroup==1.1.2
+fastapi==0.100.0
+gitdb==4.0.10
+GitPython==3.1.32
+h11==0.14.0
+idna==3.4
+pydantic==2.0.2
+pydantic_core==2.1.2
+python-dotenv==1.0.0
+PyYAML==6.0
+smmap==5.0.0
+sniffio==1.3.0
+starlette==0.27.0
+typing_extensions==4.7.1
+uvicorn==0.22.0
Author	SHA1	Message	Date
brb	cb2e45e5ae	updated ignored modules	2023-07-14 14:56:04 +02:00
brb	824215c37a	flake8 compliant	2023-07-14 14:55:42 +02:00
brb	0e39795b26	added definition of external worker process	2023-07-14 14:54:13 +02:00
brb	32e06d0d4e	removed usage of external worker process	2023-07-14 14:53:46 +02:00
brb	fd18d9450e	updated root post to start script execution in background task	2023-07-14 14:53:20 +02:00
brb	31ddd80fae	updated function to run directly via os.system and not in subprocess	2023-07-14 14:52:49 +02:00
brb	c6fccb9047	updated test script	2023-07-14 14:52:02 +02:00
brb	8841724740	updated execution of script	2023-07-13 13:12:11 +02:00
brb	db66de65b1	updated script	2023-07-13 13:10:45 +02:00
brb	6b165aa0ff	updated to find moved runner script	2023-07-13 12:54:32 +02:00
brb	aca204aa80	moved gpu_runner script	2023-07-13 12:53:05 +02:00
brb	1ef127d85f	updated execute runner script	2023-07-13 12:48:51 +02:00
brb	c6c56ef1cf	updated execute runner script	2023-07-13 12:46:53 +02:00
brb	a6a3464e7e	updated execute runner script	2023-07-13 12:46:24 +02:00
brb	86834b1d7c	updated execute runner script	2023-07-13 12:45:04 +02:00
brb	479674f807	updated repo_dir	2023-07-13 12:44:00 +02:00
brb	1e2e3e5e03	removed dev env vars	2023-07-13 12:42:03 +02:00
brb	70d78e8fa1	fixed bug with dev env vars	2023-07-13 12:39:35 +02:00
brb	ddb9e4f9b4	fixed bug	2023-07-13 12:37:32 +02:00
brb	e57f083b14	fixed bug	2023-07-13 12:35:52 +02:00
brb	b4f1eab752	added test printing of var	2023-07-13 12:35:11 +02:00
brb	320efedd50	added test printing of var	2023-07-13 12:33:55 +02:00
brb	176d09ca34	added test printing of var	2023-07-13 12:31:49 +02:00
brb	ca7061b0f6	added development env	2023-07-13 12:30:06 +02:00
brb	661baad416	updated values	2023-07-13 12:29:46 +02:00
brb	730717fc3b	changed app to accept calls directly to root	2023-07-13 12:24:23 +02:00
brb	cd636cce20	updated logging to show host ip and port	2023-07-13 12:21:24 +02:00
brb	6dd7d3f465	updated packages	2023-07-13 12:14:07 +02:00
brb	4a15e9fee0	flake8 compliant	2023-07-13 12:13:55 +02:00
brb	7a965e5dfc	mypy compliant	2023-07-13 12:11:49 +02:00
brb	2f69f49690	added mypy config	2023-07-13 12:11:43 +02:00
brb	1266204ecb	flake8 compliant	2023-07-13 11:53:14 +02:00
brb	6492b22219	defined default values	2023-07-13 11:48:25 +02:00
brb	8631722116	added loading of defaults	2023-07-13 11:48:10 +02:00
brb	2f014deb05	added loading of defaults	2023-07-13 11:47:50 +02:00
brb	3d8b240d9b	added logging	2023-07-13 11:38:20 +02:00
brb	5c2fc4022e	changed layout of file	2023-07-13 11:37:34 +02:00
brb	bf23ba3f38	added automatic parsing from file	2023-07-13 11:36:47 +02:00
brb	493dc874f0	added definition	2023-07-13 11:29:29 +02:00
brb	614a85c429	added definition	2023-07-13 11:29:14 +02:00
brb	9956d6a2c4	added definition	2023-07-13 11:23:52 +02:00
brb	25066d687e	added definition	2023-07-13 11:22:42 +02:00
brb	db8d478931	updated description	2023-07-13 11:18:57 +02:00