Merge pull request #1 from kpei/amogus

Use o!rdr to render videos

.github/workflows/test.yml

@@ -34,5 +34,6 @@ jobs:
           S3_BUCKET=${{ secrets.S3_BUCKET }}
           STREAMABLE_USERNAME=${{ secrets.STREAMABLE_USERNAME }}
           STREAMABLE_PASSWORD=${{ secrets.STREAMABLE_PASSWORD }}
+          ORDR_API_KEY=${{ secrets.ORDR_API_KEY }}
           EOF
       - run: ./bin/test.sh docker

Dockerfile

@@ -1,31 +1,10 @@
 FROM python:3.8-slim
-ENV APT_PKGS build-essential git libavcodec-dev libavformat-dev libfreetype6-dev libjpeg-dev libswscale-dev unzip zlib1g-dev
-ENV OSU_SKIN_PATH /home/bot/skin
-ENV OSR2MP4_REV fb2c4ccd6236eab8c891d2c2d36eaf08f0005082
 ENV PYTHONPATH /home/bot
-RUN \
-  useradd --create-home --uid 1000 bot && \
-  apt-get update && \
-  apt-get --yes install ${APT_PKGS} ffmpeg && \
-  rm --recursive --force /var/lib/apt/lists && \
-  pip install pillow-simd && \
-  git clone https://github.com/uyitroa/osr2mp4-core /tmp/osr2mp4-core && \
-  cd /tmp/osr2mp4-core/osr2mp4 && \
-  git checkout $OSR2MP4_REV && \
-  python install.py && \
-  cd ImageProcess/Curves/libcurves && \
-  python setup.py build_ext --inplace && \
-  cd ../../../VideoProcess/FFmpegWriter && \
-  python setup.py build_ext --inplace && \
-  mv /tmp/osr2mp4-core/osr2mp4 /home/bot/osr2mp4 && \
-  chown --recursive bot /home/bot/osr2mp4
-COPY assets/skin.osk /tmp/skin.osk
+RUN useradd --create-home --uid 1000 bot
 COPY requirements.txt /tmp/requirements.txt
 RUN \
-  unzip /tmp/skin.osk -d /home/bot/skin && \
   pip install --requirement /tmp/requirements.txt && \
-  apt-get --yes remove ${APT_PKGS} && \
-  apt-get --yes autoremove && \
   rm --recursive --force /tmp/*
 USER bot
 COPY src/ /home/bot/src
+CMD [ "tail", "-f", "/dev/null" ]

README.md

@@ -18,12 +18,11 @@ Environment variables that need to be set in `.env`:
 - `OSU_API_KEY`
 - `OSU_USERNAME`
 - `OSU_PASSWORD`
-- `STREAMABLE_USERNAME`
-- `STREAMABLE_PASSWORD`
+- `ORDR_API_KEY`
 - `SERVER_ADDR`
 
 ```sh
 docker-compose build
-docker-compose run -u root worker sh -c 'chown 1000 $LOG_DIR $VIDEO_DIR'
+docker-compose run -u root worker sh -c 'chown 1000 $LOG_DIR'
 docker-compose up -d --scale worker=4
 ```

bin/test.sh

@@ -11,22 +11,18 @@ if [[ "$1" == "docker" ]]; then
     echo "Saving Redis AOF backup to $backup"
     cp data/appendonly.aof "$backup"
   fi
-  mapset="$(mktemp --directory)"
-  unzip -d "$mapset" assets/mapset.osz
   docker-compose up --build --detach worker
-  docker cp "$mapset" osr2mp4-bot-test_worker_1:/tmp/mapset
-  rm --recursive --force "$mapset"
-  for file in assets/replay.osr setup.cfg test; do
+  for file in setup.cfg test; do
     docker cp "$file" "osr2mp4-bot-test_worker_1:/tmp/$(basename $file)"
   done
   docker-compose exec -T redis redis-cli FLUSHALL
-  docker-compose exec -T -u root worker sh -c 'chown 1000 $LOG_DIR $VIDEO_DIR /tmp/mapset'
+  docker-compose exec -T -u root worker sh -c 'chown 1000 $LOG_DIR $VIDEO_DIR'
   set +e
   docker-compose exec -T worker bash <<EOF
     pip install pytest-cov
     mkdir \$HOME/testenv
     cd \$HOME/testenv
-    cp --recursive \$HOME/src /tmp/mapset /tmp/replay.osr /tmp/test /tmp/setup.cfg .
+    cp --recursive \$HOME/src /tmp/test /tmp/setup.cfg .
     python -m pytest --cov src $@
 EOF
   exit="$?"

docker-compose.yml

@@ -8,6 +8,21 @@ services:
     depends_on:
       - redis
     command: python -m src
+    volumes:
+      - ./src:/home/bot/src:delegated
+    environment:
+      JOB_TIMEOUT: 3600
+      REDIS_HOST: redis
+  websocket:
+    # Receives render updates from o!rdr
+    build: .
+    restart: unless-stopped
+    env_file: .env
+    depends_on:
+      - redis
+    command: python -m src.websocket
+    volumes:
+      - ./src:/home/bot/src:delegated
     environment:
       JOB_TIMEOUT: 3600
       REDIS_HOST: redis
@@ -17,34 +32,19 @@ services:
     restart: unless-stopped
     env_file: .env
     depends_on:
-      - files
       - redis
     command: rq worker --with-scheduler --quiet --url redis://redis
     environment:
-      JOB_TIMEOUT: 3600
+      JOB_TIMEOUT: 1800
       LOG_DIR: /tmp/logs
       REDIS_HOST: redis
-      VIDEO_DIR: /tmp/videos
     volumes:
       - ${PWD}/logs:/tmp/logs
-      - videos:/tmp/videos
-  files:
-    # Publically accessible file server where Streamable can read videos from.
-    image: halverneus/static-file-server:v1.8.1
-    restart: unless-stopped
-    environment:
-      FOLDER: /data
-      PORT: 8080
-    volumes:
-      - videos:/data
-    ports:
-      - ${SERVER_PORT:-80}:8080
+      - ./src:/home/bot/src:delegated
   redis:
     # Job queue backend and persistent storage service.
     image: redis:6
     restart: unless-stopped
     command: --appendonly yes --loglevel warning
     volumes:
       - ${PWD}/data:/data
-volumes:
-  videos:

requirements.txt

@@ -3,5 +3,6 @@ osuapi==0.0.42
 praw==7.1.0
 redis==3.5.3
 requests==2.24
-rq==1.6.1
+rq==1.8.1
 sentry-sdk==0.19.1
+python-socketio[client]==5.11.3

setup.cfg

@@ -20,5 +20,8 @@ ignore_missing_imports = True
 [mypy-rq]
 ignore_missing_imports = True
 
+[mypy-socketio]
+ignore_missing_imports = True
+
 [tool:pytest]
 filterwarnings = ignore::DeprecationWarning

src/websocket/__main__.py

@@ -0,0 +1,3 @@
+from .ws import main
+
+main()

src/websocket/ws.py

@@ -0,0 +1,44 @@
+from typing import Any
+import socketio
+import logging
+
+from src.worker.cache import is_render_active, set_render_id
+
+fmt = "%(asctime)s %(levelname)s: %(message)s"
+logging.basicConfig(level=logging.INFO, format=fmt)
+
+sio = socketio.Client(reconnection=True)
+
+@sio.event
+def connect() -> None:
+    logging.info("o!rdr websocket connected.")
+
+@sio.event
+def disconnect() -> None:
+    logging.error("o!rdr websocket disconnected.")
+
+@sio.on("render_done_json")
+def on_render_done(data: Any) -> None:
+    try:
+        if is_render_active(data["renderID"]):
+            set_render_id(data["renderID"], data["videoUrl"])
+            logging.info(f"Got video url (id:{data['renderID']}) - {data['videoUrl']}")
+    except Exception:
+        logging.exception(
+            f"o!rdr websocket: error handling render finished - data: {data}"
+        )
+
+@sio.on("render_failed_json")
+def on_render_failed(data: Any) -> None:
+    try:
+        if is_render_active(data["renderID"]):
+            set_render_id(data["renderID"], "failed")
+            logging.info(f"render {data['renderID']} failed - {data['errorMessage']}")
+    except Exception:
+        logging.exception(
+            f"o!rdr websocket: error handling render failed - data: {data}"
+        )
+
+def main() -> None:
+    sio.connect("https://apis.issou.best", socketio_path="/ordr/ws")
+    sio.wait()

src/worker/__init__.py

@@ -10,11 +10,14 @@ def __init__(self, msg: str) -> None:
         self.msg = msg
 
 
-from .cache import get_video, set_video, set_video_progress  # noqa: E402
-from .osu import download_mapset, download_replay  # noqa: E402
-from .recorder import record  # noqa: E402
+from .cache import get_video, set_active_render, set_video_progress  # noqa: E402
+from .osu import download_replay  # noqa: E402
 from .reddit import failure, finished, parse_comment, reply, success  # noqa: E402
-from .streamable import upload  # noqa: E402
+from src.worker.ordr import (  # noqa: E402,E501
+    delete_replay,
+    wait_and_set_video_url,
+    submit_replay,
+)
 
 
 def job(comment: Comment) -> None:
@@ -29,23 +32,22 @@ def job(comment: Comment) -> None:
         video_url = get_video(score)
         if video_url:
             logging.info(f"Found video in cache ({video_url})")
+            success(comment, video_url)
         else:
             set_video_progress(score, True)
-            logging.info(f"mapset={mapset}, score={score}")
-            logging.info("Downloading mapset...")
-            mapset_path = download_mapset(mapset)
-            logging.info(f"Mapset downloaded to {mapset_path}")
             logging.info("Downloading replay...")
             replay_path = download_replay(score)
             logging.info(f"Replay downloaded to {replay_path}")
-            logging.info("Recording...")
-            video_path = record(mapset_path, replay_path)
-            logging.info(f"Video recorded to {video_path}")
-            logging.info("Uploading...")
-            video_url = upload(video_path, title)
-            logging.info(f"Video uploaded to {video_url}")
-            set_video(score, video_url)
-        success(comment, video_url)
+            logging.info("Submitting Replay to o!rdr...")
+            render_id = submit_replay(replay_path)
+            if not render_id:
+                raise ReplyWith("o!rdr replay rendering failed, no render id")
+            set_active_render(render_id)
+            logging.info(
+                f"Replay submitted to o!rdr ({render_id}) - waiting for video url"
+            )
+            wait_and_set_video_url(score, render_id, comment)
+            delete_replay(replay_path)
     except ReplyWith as e:
         if is_osubot_comment(comment):
             logging.warning(e.msg)
@@ -55,9 +57,4 @@ def job(comment: Comment) -> None:
         logging.exception("Something failed...")
         failure(comment)
     finally:
-        try:
-            set_video_progress(score, False)
-        except NameError:
-            # If something failed in parse_comment, there's no score ID.
-            pass
         finished(comment)

src/worker/cache.py

@@ -9,6 +9,7 @@
 JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT", "3600"))
 PREFIX = "video"
 PROGRESS = "progress"
+RENDER = "render"
 REDIS = Redis(os.getenv("REDIS_HOST", "localhost"))
 
 
@@ -25,6 +26,27 @@ def set_video(score: int, url: str) -> None:
     set_video_progress(score, False)
 
 
+def set_active_render(render_id: str) -> None:
+    """Mark an o!rdr as being currently or no longer in progress."""
+    REDIS.set(f"{RENDER}:{render_id}", "true", ex=JOB_TIMEOUT)
+
+
+def is_render_active(render_id: str) -> bool:
+    """Check if the o!rdr render is actually part of the import"""
+    if not REDIS.get(f"{RENDER}:{render_id}"):
+        return False
+    return True
+
+
+def set_render_id(render_id: str, url: str) -> None:
+    REDIS.set(f"{PREFIX}:{RENDER}:{render_id}", url, ex=JOB_TIMEOUT)
+
+
+def get_render_id(render_id: str) -> Optional[str]:
+    url = REDIS.get(f"{PREFIX}:{RENDER}:{render_id}")
+    return None if url is None else url.decode()
+
+
 def set_video_progress(score: int, status: bool) -> None:
     """Mark `score` as being currently or no longer in progress."""
     key = f"{PREFIX}:{PROGRESS}:{score}"