updated ignored modules

.gitea/gpu_runner/script.yml

@@ -0,0 +1,4 @@
+script: |
+  echo "start of script"
+  sleep 15
+  echo "end of script"

code/app.py

@@ -1,17 +1,23 @@
 import uvicorn
-from fastapi import FastAPI, Request, HTTPException
+from fastapi import FastAPI, Request, HTTPException, BackgroundTasks
 from fastapi.responses import HTMLResponse
 from dotenv import load_dotenv
 import logging
 import os
+import httpx
 
-from utils import clone_repository, execute_runner_script
+from utils import clone_repository, execute_script
 from setup_logging import setup_logging
 from gitea_request import GiteaRequest
 
 app = FastAPI()
 
 
+async def post_callback(url: str, payload: dict):
+    async with httpx.AsyncClient() as client:
+        await client.post(url, json=payload)
+
+
 @app.get('/', response_class=HTMLResponse)
 async def root():
     content_str = """
@@ -39,7 +45,7 @@ async def test(request: Request):
 
 
 @app.post('/')
-def handle_event(request: GiteaRequest):
+def handle_event(request: GiteaRequest, background_tasks: BackgroundTasks):
     """ Handle event from webhook. """
     # extract information
     commit_msg = request.head_commit['message']
@@ -53,30 +59,23 @@ def handle_event(request: GiteaRequest):
         )
     # clone repository
     local_repo_dir = clone_repository(repo_url)
-    # locate runner script
+    # # locate runner script
-    runner_script_path = local_repo_dir / 'runner_script.yml'
+    # runner_script_path = (
-    if not runner_script_path.exists():
+    #     local_repo_dir /
-        raise HTTPException(
+    #     '.gitea' /
-            status_code=200,
+    #     'gpu_runner' /
-            detail="no runner_script.yml in repository root"
+    #     'script.yml'
-        )
+    # )
-    # execute runner script
+    # if not runner_script_path.exists():
-    logging.info('rexecuting runner script')
+    #     raise HTTPException(
-    execute_runner_script(path=runner_script_path)
+    #         status_code=200,
-    # # TODO: create virtualenv?
+    #         detail=f"{runner_script_path} does not exist"
-    # # install requirements
+    #     )
-    # assert pipeline.requirements is not None
+    # queue runner script
-    # requirements_path = local_repo_dir / pipeline.requirements
+    background_tasks.add_task(
-    # os.system(f'pip install -r {requirements_path}')
+        execute_script,
-    # for cmd in pipeline.script:
+        local_repo_dir
-    #     try:
+    )
-    #         os.system(cmd)
-    #     except Exception as e:
-    #         raise HTTPException(
-    #             status_code=503,
-    #             detail=f'failed running {cmd}: {e}'
-    #         )
-
     # send response
     raise HTTPException(
         status_code=202,
@@ -94,5 +93,6 @@ if __name__ == '__main__':
     uvicorn.run(
         app=app,
         host=listen_ip,
-        port=listen_port
+        port=listen_port,
+        workers=1
     )

code/defaults.ini

@@ -1,6 +1,6 @@
 [main]
 logger_level=debug
-repo_dir=/usr/src/app/repo
+repo_dir=repo
 
 [fastapi]
 listen_ip=0.0.0.0

code/main.py

@@ -7,7 +7,8 @@ import os
 from configparser import ConfigParser
 from pathlib import Path
 import socket
-
+# from multiprocessing import Queue
+# from worker import Worker
 
 # load default values
 config = ConfigParser()
@@ -23,9 +24,17 @@ if __name__ == '__main__':
     listen_ip = os.getenv('LISTEN_IP', default=LISTEN_IP)
     listen_port = int(os.getenv('LISTEN_PORT', default=LISTEN_PORT))
     own_ip = socket.gethostbyname(socket.gethostname())
+    # # setup work queue
+    # work_queue: Queue = Queue()
+    # # setup worker process
+    # worker = Worker(queue=work_queue)
+    # start server
     logging.info(f'starting FastAPI server ({own_ip}:{listen_port})')
+    # try:
     uvicorn.run(
         app=app,
         host=listen_ip,
         port=listen_port
     )
+    # except KeyboardInterrupt:
+    #     worker.stop()

code/runner_script.py

@@ -24,6 +24,6 @@ class RunnerScript(BaseModel):
 
 
 if __name__ == '__main__':
-    path = Path(__file__).parent.parent / 'runner_script.yml'
+    path = Path('.gitea') / 'gpu_runner' / 'script.yml'
     rs = RunnerScript.from_file(path)
     print(rs)

code/utils.py

@@ -4,14 +4,22 @@ from git import Repo
 import logging
 import shutil
 import os
+from configparser import ConfigParser
 from runner_script import RunnerScript
 
 
-def clone_repository(repo_url: AnyHttpUrl) -> Path:
+# load default values
+config = ConfigParser()
+config.read(Path(__file__).parent / 'defaults.ini')
+REPO_DIR = config.get('main', 'repo_dir')
+
+
+def clone_repository(
+    repo_url: AnyHttpUrl,
+    repo_dir: str = REPO_DIR
+) -> Path:
     # extract repo name
     repo_name = repo_url.split('/')[-1].replace('.git', '')
-    repo_dir = os.getenv('REPO_DIR', default=None)
-    assert repo_dir is not None
     dst_dir = Path(repo_dir) / repo_name
     # prepare repo download dir
     if dst_dir.exists():
@@ -23,8 +31,40 @@ def clone_repository(repo_url: AnyHttpUrl) -> Path:
     return dst_dir
 
 
-def execute_runner_script(path: Path) -> None:
+def execute_script(
-    script_list = RunnerScript.from_file(path)
+    local_repo_dir: Path
-    for cmd in script_list:
+) -> None:
-        os.system(cmd)
+    # locate the script
+    script_path = (
+        local_repo_dir /
+        '.gitea' /
+        'gpu_runner' /
+        'script.yml'
+    )
+    # determine original working dir
+    org_dir = os.getcwd()
+    # initialise command list
+    cmd_list = [
+        f'cd {local_repo_dir.resolve()}',
+        'virtualenv -p python3.10 venv',
+        'source venv/bin/activate',
+    ]
+    # add in runner script
+    rs = RunnerScript.from_file(path=script_path)
+    for cmd in rs.script:
+        cmd_list.append(cmd)
+    # append cleanup commands
+    cmd_list.append('deactivate')
+    cmd_list.append(f'cd {org_dir}')
+    cmd_list.append(f'rm -rf {local_repo_dir.resolve()}')
+    # build command string
+    cmd_str = ' && '.join(cmd_list)
+    os.system(cmd_str)
     logging.debug('finished')
+
+
+if __name__ == '__main__':
+    local_repo_dir = Path(__file__).parent.parent / 'repo' / 'gpu_runner'
+    execute_script(
+        local_repo_dir=local_repo_dir
+    )

code/worker.py

@@ -0,0 +1,40 @@
+from multiprocessing import Process, Queue
+import time
+from typing import List
+import os
+
+
+class Worker(object):
+    def __init__(self, queue: Queue):
+        self.run_flag = True
+        self.queue = queue
+        self.process = Process(
+            target=self.handle_jobs,
+            args=(self.queue,)
+        )
+        time.sleep(1)
+        self.process.start()
+
+    def stop(self):
+        self.queue.put(['exit'])
+        self.process.join()
+
+    @staticmethod
+    def handle_jobs(queue: Queue):
+        while True:
+            if queue.empty():
+                time.sleep(1)
+                continue
+            cmd_list: List[str] = queue.get()
+            print('received job:\n', cmd_list)
+            if cmd_list[0].lower() == 'exit':
+                break
+            cmd_str = ' && '.join(cmd_list)
+            os.system(cmd_str)
+
+
+if __name__ == '__main__':
+    work_queue: Queue = Queue()
+    worker = Worker(queue=work_queue)
+    work_queue.put(['echo "hello world"', 'echo "hello world again"'])
+    worker.stop()

dev.env

@@ -1,2 +0,0 @@
-REPO_DIR = /repo
-ENV = dev

mypy.ini

@@ -11,3 +11,6 @@ ignore_missing_imports = True
 
 [mypy-fastapi.*]
 ignore_missing_imports = True
+
+[mypy-httpx.*]
+ignore_missing_imports = True

runner_script.yml

@@ -1,3 +0,0 @@
-script: |
-  echo "hello world"
-  echo "hello back"