From b2079446e513947c1073dcfbd786369a647c0226 Mon Sep 17 00:00:00 2001 From: wheelz Date: Wed, 24 Jun 2026 19:38:42 +0000 Subject: [PATCH] feat: add single-folder RomM import and arr helpers --- romm-automation-import-worker.py | 111 +++++++++++++++++++++++----- romm-automation-truenas-compose.yml | 64 ++++++++++++++++ romm-automation-truenas.env.example | 16 +++- 3 files changed, 168 insertions(+), 23 deletions(-) diff --git a/romm-automation-import-worker.py b/romm-automation-import-worker.py index dc8465c..dfa55a6 100644 --- a/romm-automation-import-worker.py +++ b/romm-automation-import-worker.py @@ -1,16 +1,15 @@ #!/usr/bin/env python3 """Simple RomM staging import worker. -Watches /staging/incoming// for files, moves them through +Watches a single landing folder, /staging/incoming, for ROM files, infers the +RomM platform slug when it can do so safely, moves files through processing/verified, copies them into /romm/library/roms//, and then triggers RomM's library scan API. -This intentionally requires a platform subfolder under incoming. Example: - /staging/incoming/ngc/Paper Mario.iso - /staging/incoming/nes/Homebrew.nes - -Files dropped directly into /staging/incoming are quarantined so the worker does -not guess the wrong RomM platform. +The worker intentionally quarantines files when the platform cannot be inferred +with reasonable confidence. Add mappings with IMPORT_PLATFORM_MAP_JSON if needed. +Example: + IMPORT_PLATFORM_MAP_JSON={".iso":"ngc",".rvz":"ngc"} """ from __future__ import annotations @@ -18,6 +17,7 @@ from __future__ import annotations import json import os import shutil +import struct import sys import time import traceback @@ -43,6 +43,41 @@ ROMM_PASSWORD = os.environ.get("ROMM_PASSWORD", "") IGNORE_SUFFIXES = {".part", ".partial", ".tmp", ".crdownload"} +# Safe extension mappings that are normally unambiguous for RomM platform slugs. +EXTENSION_PLATFORM_MAP: dict[str, str] = { + ".nes": "nes", + ".fds": "fds", + ".sfc": "snes", + ".smc": "snes", + ".gb": "gb", + ".gbc": "gbc", + ".gba": "gba", + ".n64": "n64", + ".z64": "n64", + ".v64": "n64", + ".ndS".lower(): "nds", + ".3ds": "3ds", + ".cia": "3ds", + ".gcm": "ngc", + ".rvz": "ngc", + ".wbfs": "wii", + ".wad": "wii", +} + +# Extensions that require header detection or an explicit override because they +# can belong to many systems. +AMBIGUOUS_EXTENSIONS = {".iso", ".bin", ".cue", ".chd", ".zip", ".7z", ".rar"} + +try: + user_map = json.loads(os.environ.get("IMPORT_PLATFORM_MAP_JSON", "{}") or "{}") + if isinstance(user_map, dict): + for ext, platform in user_map.items(): + if isinstance(ext, str) and isinstance(platform, str) and ext and platform: + normalized_ext = ext.lower() if ext.startswith(".") else f".{ext.lower()}" + EXTENSION_PLATFORM_MAP[normalized_ext] = platform.strip() +except json.JSONDecodeError as exc: + print(f"WARNING: failed to parse IMPORT_PLATFORM_MAP_JSON: {exc}", flush=True) + def log(message: str) -> None: ts = datetime.now(timezone.utc).isoformat(timespec="seconds") @@ -73,17 +108,51 @@ def is_ready_file(path: Path) -> bool: def iter_incoming_files() -> list[Path]: if not INCOMING.exists(): return [] - return sorted([p for p in INCOMING.rglob("*") if is_ready_file(p)]) + # Single landing folder mode: process files dropped directly in incoming. + # Ignore files already inside subdirectories so processing/verified layouts do + # not accidentally get re-imported if someone creates folders in incoming. + return sorted([p for p in INCOMING.iterdir() if is_ready_file(p)]) -def platform_for(path: Path) -> str | None: - rel = path.relative_to(INCOMING) - if len(rel.parts) < 2: +def detect_gamecube_or_wii(path: Path) -> str | None: + """Return ngc/wii for disc images with Nintendo magic, otherwise None.""" + try: + with path.open("rb") as fh: + fh.seek(0x18) + raw = fh.read(8) + if len(raw) < 8: + return None + # GameCube magic at 0x1C: 0xC2339F3D. Wii magic at 0x18: 0x5D1C9EA3. + wii_magic = struct.unpack(">I", raw[0:4])[0] + gc_magic = struct.unpack(">I", raw[4:8])[0] + if gc_magic == 0xC2339F3D: + return "ngc" + if wii_magic == 0x5D1C9EA3: + return "wii" + except OSError: return None - platform = rel.parts[0].strip() - if platform in {"", ".", ".."} or "/" in platform: - return None - return platform + return None + + +def platform_for(path: Path) -> tuple[str | None, str]: + suffix = path.suffix.lower() + + detected = None + if suffix in {".iso", ".gcm", ".rvz", ".wbfs"}: + detected = detect_gamecube_or_wii(path) + if detected: + return detected, f"Nintendo disc image header detected as {detected}" + + if suffix in EXTENSION_PLATFORM_MAP: + platform = EXTENSION_PLATFORM_MAP[suffix] + if suffix in AMBIGUOUS_EXTENSIONS: + return platform, f"explicit/import env extension override {suffix}->{platform}" + return platform, f"extension mapping {suffix}->{platform}" + + if suffix in AMBIGUOUS_EXTENSIONS: + return None, f"ambiguous extension {suffix}; add IMPORT_PLATFORM_MAP_JSON override or use a less ambiguous file format" + + return None, f"unknown extension {suffix or ''}; add IMPORT_PLATFORM_MAP_JSON override if this is expected" def unique_dest(dest: Path) -> Path: @@ -101,7 +170,8 @@ def unique_dest(dest: Path) -> Path: def write_report(name: str, data: dict) -> None: REPORTS.mkdir(parents=True, exist_ok=True) - path = REPORTS / name + safe_name = "".join(c if c.isalnum() or c in "-. _" else "_" for c in name) + path = REPORTS / safe_name path.write_text(json.dumps(data, indent=2, sort_keys=True), encoding="utf-8") @@ -157,9 +227,9 @@ def trigger_romm_scan() -> None: def import_file(path: Path) -> bool: - platform = platform_for(path) + platform, platform_reason = platform_for(path) if not platform: - quarantine(path, "File was dropped directly into incoming. Put ROMs under incoming// so the worker does not guess the platform.") + quarantine(path, f"Could not infer RomM platform from single-folder incoming mode: {platform_reason}") return False processing_dir = PROCESSING / platform @@ -170,7 +240,7 @@ def import_file(path: Path) -> bool: processing_path = unique_dest(processing_dir / path.name) shutil.move(str(path), str(processing_path)) - log(f"processing {processing_path} as platform={platform}") + log(f"processing {processing_path} as platform={platform} ({platform_reason})") verified_path = unique_dest(verified_dir / processing_path.name) shutil.copy2(str(processing_path), str(verified_path)) @@ -189,6 +259,7 @@ def import_file(path: Path) -> bool: report = { "time_utc": datetime.now(timezone.utc).isoformat(), "platform": platform, + "platform_reason": platform_reason, "incoming": str(path), "verified": str(verified_path), "romm_path": str(romm_path), @@ -204,7 +275,7 @@ def import_file(path: Path) -> bool: def main() -> int: ensure_dirs() - log(f"RomM import worker started; incoming={INCOMING}; roms={ROMM_ROMS}; poll={POLL_SECONDS}s; scan_after_import={SCAN_AFTER_IMPORT}") + log(f"RomM import worker started; single-folder incoming={INCOMING}; roms={ROMM_ROMS}; poll={POLL_SECONDS}s; scan_after_import={SCAN_AFTER_IMPORT}") while True: try: files = iter_incoming_files() diff --git a/romm-automation-truenas-compose.yml b/romm-automation-truenas-compose.yml index d45b94e..b67bf1d 100644 --- a/romm-automation-truenas-compose.yml +++ b/romm-automation-truenas-compose.yml @@ -182,6 +182,7 @@ services: ROMM_SCAN_AFTER_IMPORT: ${ROMM_SCAN_AFTER_IMPORT:-true} IMPORT_POLL_SECONDS: ${IMPORT_POLL_SECONDS:-60} IMPORT_QUIET_IDLE: ${IMPORT_QUIET_IDLE:-true} + IMPORT_PLATFORM_MAP_JSON: ${IMPORT_PLATFORM_MAP_JSON:-{}} volumes: - ./romm-automation-import-worker.py:/work/romm-automation-import-worker.py:ro - ${TRUENAS_DOCKER_ROOT:-/mnt/HomeStorage02/Docker}/RomAutomation/staging:/staging @@ -191,6 +192,69 @@ services: networks: - romm-automation + prowlarr: + image: lscr.io/linuxserver/prowlarr:latest + container_name: romm-automation-prowlarr + restart: unless-stopped + environment: + - PUID=${PUID:-568} + - PGID=${PGID:-568} + - TZ=${TZ:-America/Chicago} + volumes: + - ${TRUENAS_DOCKER_ROOT:-/mnt/HomeStorage02/Docker}/RomAutomation/prowlarr/:/config + - ${TRUENAS_DOCKER_ROOT:-/mnt/HomeStorage02/Docker}/RomAutomation/staging:/media/ + ports: + - "${PROWLARR_PORT:-9896}:9696" + networks: + - romm-automation + + qbittorrent: + container_name: romm-automation-qbittorrent + image: ghcr.io/hotio/qbittorrent:release-5.1.2 + restart: unless-stopped + ports: + - "${QBITTORRENT_PORT:-8380}:8080" + environment: + - PUID=${PUID:-568} + - PGID=${PGID:-568} + - UMASK=002 + - TZ=${TZ:-America/Chicago} + - WEBUI_PORTS=8080/tcp,8080/udp + - VPN_ENABLED=true + - VPN_CONF=wg0 + - VPN_PROVIDER=proton + - VPN_LAN_NETWORK=192.168.0.0/16 + - VPN_AUTO_PORT_FORWARD=true + - VPN_FIREWALL_TYPE=auto + - VPN_HEALTHCHECK_ENABLED=false + - VPN_NAMESERVERS=wg + - PRIVOXY_ENABLED=false + cap_add: + - NET_ADMIN + sysctls: + - net.ipv4.conf.all.src_valid_mark=1 + - net.ipv6.conf.all.disable_ipv6=1 + volumes: + - ${TRUENAS_DOCKER_ROOT:-/mnt/HomeStorage02/Docker}/RomAutomation/qbittorrent/:/config + - ${TRUENAS_DOCKER_ROOT:-/mnt/HomeStorage02/Docker}/RomAutomation/staging:/media + networks: + - romm-automation + + homarr: + container_name: romm-automation-homarr + image: ghcr.io/homarr-labs/homarr:latest + restart: unless-stopped + volumes: + - ${TRUENAS_DOCKER_ROOT:-/mnt/HomeStorage02/Docker}/RomAutomation/homarr:/appdata # Persistent config/data storage + environment: + - SECRET_ENCRYPTION_KEY=${HOMARR_SECRET_KEY} # Required: paste 64-char hex string here + - TZ=${TZ:-America/Chicago} # Set your timezone + ports: + - "${HOMARR_PORT:-7775}:7575" + networks: + - romm-automation + + networks: romm-automation: name: romm-automation diff --git a/romm-automation-truenas.env.example b/romm-automation-truenas.env.example index 8f133e0..b8517ea 100644 --- a/romm-automation-truenas.env.example +++ b/romm-automation-truenas.env.example @@ -17,18 +17,25 @@ ROMM_LIBRARY_PATH=/mnt/HomeStorage02/Docker/RomM/library # Published ports on the TrueNAS Docker host GGREQUESTZ_PORT=3003 ROMARR_PORT=9797 +PROWLARR_PORT=9896 +QBITTORRENT_PORT=8380 +HOMARR_PORT=7775 # Optional Compose profiles. # Start blank or validation-only first. Add romarr only after ROMARR_IMAGE is confirmed. # Valid values: romarr,discord,validation or comma-separated combinations. COMPOSE_PROFILES=validation -# Import worker: watches staging/incoming//, imports into RomM, -# then triggers RomM's scan_library task. Keep immediate scan enabled while testing; -# later we can debounce/batch scans if needed. +# Import worker: watches a single landing folder at staging/incoming, infers the +# RomM platform when safe, imports into RomM, then triggers RomM's scan_library +# task. Keep immediate scan enabled while testing; later we can debounce/batch +# scans if needed. ROMM_SCAN_AFTER_IMPORT=true IMPORT_POLL_SECONDS=60 IMPORT_QUIET_IDLE=true +# Optional JSON extension/platform overrides for ambiguous file types. +# Example: {".iso":"ngc",".chd":"psx"} +IMPORT_PLATFORM_MAP_JSON={} # GG Requestz support services GGREQUESTZ_POSTGRES_DB=ggrequestz @@ -64,6 +71,9 @@ PROWLARR_URL=http://your-prowlarr-host:9696 QBITTORRENT_URL=http://your-vpn-qbit-host:8080 QBITTORRENT_CATEGORY=romarr +# Homarr. Generate with: openssl rand -hex 32 +HOMARR_SECRET_KEY=change_me_generate_64_hex_chars + # RomM-ComM / Discord bot. Only needed if starting the discord profile. ROMMCOMM_DISCORD_TOKEN=fill_me ROMMCOMM_GUILD_ID=fill_me