commit 7758ebdebbfb7beceb92f2a402c2d6e6662750d9 Author: Your Name Date: Wed Aug 12 23:55:33 2026 +0700 first commit diff --git a/ctfcup2025-school-final b/ctfcup2025-school-final new file mode 160000 index 0000000..61bb5d9 --- /dev/null +++ b/ctfcup2025-school-final @@ -0,0 +1 @@ +Subproject commit 61bb5d9a9d5ee62156dbd0ce0fee5acaa0e953c8 diff --git a/infra/forcad-local/README.md b/infra/forcad-local/README.md new file mode 100644 index 0000000..ca40267 --- /dev/null +++ b/infra/forcad-local/README.md @@ -0,0 +1,145 @@ +# forcad-local + +Автоматизированный локальный оркестратор для запуска ForcAD + игровых сервисов + чекеров в контейнерах. + +Инструмент делает: +- auto-discovery сервисов в переданной папке; +- preflight-проверки структуры сервисов и чекеров; +- containerized сборку checker-артефактов (например `make` для `archive`) без хостовых dev-пакетов; +- auto-cleanup хвостов прошлых запусков перед новым `up`; +- prewarm k3d/k3s образов перед деплоем k3d-сервисов; +- retry для k3d-деплоя и rollback при ошибке (настраивается); +- генерацию runtime-конфига ForcAD без правок исходников борды/сервисов; +- запуск `light` smoke-check `check` для каждого сервиса без полного ForcAD стека; +- опциональный `full` режим с полным ForcAD стеком; +- команды `up`, `down`, `status`, `validate`. + +## Требования + +- `docker` + `docker compose` +- Python 3.10+ +- Для Helm/k3d-профилей сервисов: `helm`, `kubectl`, `k3d` (если требуется конкретному сервису) + +## Базовый запуск + +```bash +python3 infra/forcad-local/forcad_local.py validate \ + --services-root /path/to/services \ + --checkers-root /path/to/checkers +``` + +```bash +python3 infra/forcad-local/forcad_local.py up \ + --services-root /path/to/services \ + --checkers-root /path/to/checkers \ + --mode light +``` + +```bash +python3 infra/forcad-local/forcad_local.py status +``` + +```bash +python3 infra/forcad-local/forcad_local.py down +``` + +После `up` борда доступна по `http://127.0.0.1:8080/` (или через `--board-port`). +Это относится только к `--mode full`. В `--mode light` борда не запускается. + +## Какие структуры поддерживаются + +Для сервиса в `services-root//`: + +1. **Compose-профиль** + - должен быть `docker-compose.yml` +2. **Script/Helm-профиль** + - `deploy.sh`, либо + - `redeploy.sh`, либо + - `deploy/chart/values.yaml` (с возможным `build-docker.sh`) + +Для чекера в `checkers-root//`: +- должен быть `checker.py` **или** `*.checker.py` +- если есть `requirements.txt` (или `requirments.txt`) — зависимости автоматически объединяются +- если есть `Makefile` — сборка выполняется внутри builder-контейнера (`cpp-make` профиль по умолчанию) + +## Автопроверки (preflight) + +Команда `validate` проверяет: +- обнаружены ли сервисы; +- есть ли deploy-профиль; +- есть ли checker entrypoint; +- определён ли host-порт для сервиса; +- нет ли конфликтов портов между сервисами и портом борды; +- наличие системных команд (`docker`, а для helm/k3d-профиля — предупреждения про `helm`, `kubectl`, `k3d`). +- валидность конфигурации containerized checker-build профиля. + +## Поведение `up` по умолчанию + +- сервисы сортируются в детерминированном порядке (k3d-сервисы сначала); +- перед запуском выполняется auto-cleanup по обнаруженным сервисам; +- для k3d-сервисов заранее выполняется pull базовых образов; +- если k3d-деплой падает, делается retry (`--k3d-retries`); +- в `strict` режиме падение одного сервиса прерывает запуск и откатывает уже поднятые; +- в `best-effort` режимe проблемные сервисы пропускаются. + +## Runtime-файлы + +Все временные файлы создаются в: +- `infra/forcad-local/.runtime/` + +Кэш сборки checker-артефактов: +- `infra/forcad-local/.cache/checker-builds/` + +Там же сохраняется `state.json`, который используется командами `status` и `down`. + +## Переопределения через manifest + +Если auto-discovery не хватает, в `services-root` можно положить файл `forcad-local.json`: + +```json +{ + "services": { + "warehouse": { + "checker_dir": "warehouse", + "port": 31443, + "checker_timeout": 40, + "checker_type": "pfr", + "places": 2, + "profile": "helm_chart", + "start_cmd": "bash redeploy.sh", + "stop_cmd": "helm uninstall warehouse -n warehouse", + "checker_build": { + "enabled": false + } + }, + "archive": { + "checker_build": { + "enabled": true, + "profile": "cpp-make", + "command": "make", + "artifacts": ["libgen"] + } + } + } +} +``` + +Поля опциональны. Используй их, если сервис нестандартный или порт не удаётся вывести автоматически. + +## Полезные параметры + +- `--service ` — выбрать конкретные сервисы (можно повторять) +- `--board-port 8080` — порт борды +- `--mode light|full` — режим работы (`light` по умолчанию) +- `--failure-policy strict|best-effort` — остановиться при первой ошибке или пропускать упавшие сервисы +- `--rollback-on-failure / --no-rollback-on-failure` — откат уже поднятых сервисов при ошибке в strict режиме +- `--auto-cleanup / --no-auto-cleanup` — очистка прошлых запусков перед новым `up` +- `--prewarm-k3d / --no-prewarm-k3d` — предварительный pull k3d/k3s образов +- `--k3d-retries` — число повторов деплоя для k3d-сервисов +- `--k3d-retry-delay` — задержка между retry для k3d +- `--image-pull-retries` — число дополнительных retry на pull docker-образов +- `--image-pull-retry-delay` — задержка между retry pull +- `--team-count 2` — число команд в runtime-конфиге +- `--workers 2` — число celery worker'ов +- `--cache-dir` — путь к кэшу сборки checker-артефактов +- `--no-fast` — не использовать `docker-compose-fast.yml` diff --git a/infra/forcad-local/__pycache__/forcad_local.cpython-312.pyc b/infra/forcad-local/__pycache__/forcad_local.cpython-312.pyc new file mode 100644 index 0000000..6a43326 Binary files /dev/null and b/infra/forcad-local/__pycache__/forcad_local.cpython-312.pyc differ diff --git a/infra/forcad-local/builders/cpp-make.Dockerfile b/infra/forcad-local/builders/cpp-make.Dockerfile new file mode 100644 index 0000000..51753b7 --- /dev/null +++ b/infra/forcad-local/builders/cpp-make.Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + bash \ + build-essential \ + ca-certificates \ + file \ + libboost-serialization-dev \ + libsqlite3-dev \ + make \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /work + +CMD ["bash"] diff --git a/infra/forcad-local/forcad_local.py b/infra/forcad-local/forcad_local.py new file mode 100644 index 0000000..d3e3bcc --- /dev/null +++ b/infra/forcad-local/forcad_local.py @@ -0,0 +1,1609 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import re +import shlex +import shutil +import socket +import subprocess +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Sequence, Tuple + + +LOCAL_HOST_ALIASES = {"127.0.0.1", "localhost", "host.docker.internal", "local", "test-service"} +DEFAULT_ADMIN_USER = "forcad" +DEFAULT_ADMIN_PASSWORD = "forcad_local" + +BUILDER_PROFILES = { + "cpp-make": { + "dockerfile": "builders/cpp-make.Dockerfile", + "image": "forcad-local-builder-cpp-make:1", + }, +} + +K3D_PREWARM_IMAGES = [ + "ghcr.io/k3d-io/k3d-tools:5.9.0", + "rancher/k3s:v1.31.5-k3s1", +] + + +@dataclass +class ServiceSpec: + name: str + service_dir: Path + checker_dir: Path + profile: str + checker_entry: Path + checker_type: str + places: int + checker_timeout: int + protocol: str + host_port: Optional[int] + requirements: List[str] = field(default_factory=list) + checker_build_enabled: bool = False + checker_build_profile: Optional[str] = None + checker_build_command: Optional[str] = None + checker_build_artifacts: List[str] = field(default_factory=list) + start_cmd: Optional[str] = None + stop_cmd: Optional[str] = None + + +@dataclass +class ValidationReport: + errors: List[str] = field(default_factory=list) + warnings: List[str] = field(default_factory=list) + + def ok(self) -> bool: + return not self.errors + + +def info(message: str) -> None: + print(f"[INFO] {message}") + + +def warn(message: str) -> None: + print(f"[WARN] {message}") + + +def err(message: str) -> None: + print(f"[ERROR] {message}", file=sys.stderr) + + +def run_command(command: Sequence[str], cwd: Optional[Path] = None, check: bool = True) -> subprocess.CompletedProcess: + printable = " ".join(command) + location = f" (cwd={cwd})" if cwd else "" + info(f"Running: {printable}{location}") + result = subprocess.run(command, cwd=cwd, text=True) + if check and result.returncode != 0: + raise RuntimeError(f"Command failed with exit code {result.returncode}: {printable}") + return result + + +def run_shell(command: str, cwd: Optional[Path] = None, check: bool = True) -> subprocess.CompletedProcess: + location = f" (cwd={cwd})" if cwd else "" + info(f"Running shell: {command}{location}") + result = subprocess.run(["bash", "-lc", command], cwd=cwd, text=True) + if check and result.returncode != 0: + raise RuntimeError(f"Command failed with exit code {result.returncode}: {command}") + return result + + +def run_command_with_retries( + command: Sequence[str], + *, + cwd: Optional[Path] = None, + retries: int = 0, + retry_delay: int = 5, + purpose: str = "command", +) -> subprocess.CompletedProcess: + attempts = 1 + max(0, retries) + last_exception: Optional[Exception] = None + for attempt in range(1, attempts + 1): + try: + return run_command(command, cwd=cwd, check=True) + except Exception as exc: # noqa: BLE001 + last_exception = exc + if attempt >= attempts: + raise + warn( + f"{purpose} failed ({attempt}/{attempts}): {exc}. " + f"Retrying in {retry_delay}s..." + ) + time.sleep(retry_delay) + if last_exception: + raise last_exception + raise RuntimeError(f"{purpose} failed for unknown reason") + + +def require_command(command: str, report: ValidationReport) -> None: + if shutil.which(command) is None: + report.errors.append(f"Не найдена системная команда `{command}`.") + + +def parse_manifest(services_root: Path) -> Dict[str, dict]: + manifest_path = services_root / "forcad-local.json" + if not manifest_path.exists(): + return {} + try: + data = json.loads(manifest_path.read_text()) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Некорректный JSON в {manifest_path}: {exc}") from exc + services = data.get("services", {}) + if not isinstance(services, dict): + raise RuntimeError(f"`services` в {manifest_path} должен быть объектом.") + return services + + +def detect_profile(service_dir: Path, overrides: dict) -> Optional[str]: + if "profile" in overrides: + return str(overrides["profile"]) + if (service_dir / "docker-compose.yml").exists(): + return "compose" + if (service_dir / "deploy.sh").exists(): + return "deploy_script" + if (service_dir / "redeploy.sh").exists() or (service_dir / "deploy" / "chart" / "values.yaml").exists(): + return "helm_chart" + return None + + +def detect_checker_entry(checker_dir: Path) -> Optional[Path]: + checker_py = checker_dir / "checker.py" + if checker_py.exists(): + return Path("checker.py") + + dotted = sorted(checker_dir.glob("*.checker.py")) + if len(dotted) == 1: + return Path(dotted[0].name) + if len(dotted) > 1: + return Path(dotted[0].name) + + checker_named = sorted(p for p in checker_dir.glob("*.py") if "checker" in p.name.lower()) + if len(checker_named) == 1: + return Path(checker_named[0].name) + return None + + +def checker_protocol(entry_content: str) -> str: + if "https://" in entry_content: + return "https" + if "http://" in entry_content: + return "http" + if ".split(':')" in entry_content or "split(\":\")" in entry_content: + return "tcp" + return "unknown" + + +def checker_type(entry_content: str) -> str: + return "pfr" if "OK_WITH_FLAG_ID" in entry_content else "hackerdom" + + +def checker_places(entry_content: str) -> int: + count = len(re.findall(r"define_vuln", entry_content)) + return max(1, count) + + +def infer_make_artifacts(makefile_path: Path) -> List[str]: + makefile_text = makefile_path.read_text() + artifacts = re.findall(r"-o\s+([A-Za-z0-9_.\-\/]+)", makefile_text) + return sorted(set(artifacts)) + + +def infer_checker_artifacts(checker_dir: Path) -> List[str]: + artifacts: List[str] = [] + for py_file in checker_dir.glob("*.py"): + try: + text = py_file.read_text() + except UnicodeDecodeError: + continue + if "./libgen" in text or "libgen" in text: + artifacts.append("libgen") + return sorted(set(artifacts)) + + +def detect_checker_build(checker_dir: Path, service_override: dict) -> Tuple[bool, Optional[str], Optional[str], List[str]]: + raw_cfg = service_override.get("checker_build") + if isinstance(raw_cfg, dict): + enabled = bool(raw_cfg.get("enabled", True)) + if not enabled: + return False, None, None, [] + profile = str(raw_cfg.get("profile", "cpp-make")) + command = str(raw_cfg.get("command", "make")) + raw_artifacts = raw_cfg.get("artifacts", []) + artifacts = [str(x) for x in raw_artifacts] if isinstance(raw_artifacts, list) else [] + return True, profile, command, artifacts + + makefile = checker_dir / "Makefile" + if makefile.exists(): + artifacts = infer_make_artifacts(makefile) or infer_checker_artifacts(checker_dir) + return True, "cpp-make", "make", artifacts + + return False, None, None, [] + + +def load_requirements(checker_dir: Path) -> List[str]: + requirements: List[str] = [] + candidates = [checker_dir / "requirements.txt", checker_dir / "requirments.txt"] + for candidate in candidates: + if not candidate.exists(): + continue + for line in candidate.read_text().splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + requirements.append(stripped) + return requirements + + +def discover_host_port(service_dir: Path, profile: str, protocol: str, overrides: dict) -> Optional[int]: + if "port" in overrides: + try: + return int(overrides["port"]) + except (TypeError, ValueError): + return None + + if profile == "compose": + compose_path = service_dir / "docker-compose.yml" + if not compose_path.exists(): + return None + text = compose_path.read_text() + matches = re.findall(r"(\d{2,5})\s*:\s*(\d{2,5})(?:/\w+)?", text) + if not matches: + return None + return int(matches[0][0]) + + values_files = list(service_dir.glob("**/values.yaml")) + node_ports: List[int] = [] + for path in values_files: + text = path.read_text() + for match in re.findall(r"nodePort:\s*(\d{2,5})", text): + node_ports.append(int(match)) + # Some charts (e.g. ingress-nginx values) keep NodePorts as: + # nodePorts: + # http: 31080 + # https: 31443 + if not node_ports: + for match in re.findall(r":\s*(\d{2,5})\b", text): + value = int(match) + if value >= 30000: + node_ports.append(value) + + if node_ports: + if protocol == "http": + return min(node_ports) + return max(node_ports) + + deploy_script = service_dir / "deploy.sh" + if deploy_script.exists(): + script_text = deploy_script.read_text() + direct = re.search(r"localhost:(\d{2,5})", script_text) + if direct: + return int(direct.group(1)) + + return None + + +def discover_services( + services_root: Path, + checkers_root: Path, + selected_services: Optional[set[str]] = None, +) -> Tuple[List[ServiceSpec], ValidationReport]: + report = ValidationReport() + overrides = parse_manifest(services_root) + + if not services_root.exists(): + report.errors.append(f"Папка сервисов не найдена: {services_root}") + return [], report + if not checkers_root.exists(): + report.errors.append(f"Папка чекеров не найдена: {checkers_root}") + return [], report + + specs: List[ServiceSpec] = [] + service_dirs = sorted(p for p in services_root.iterdir() if p.is_dir() and not p.name.startswith(".")) + + for service_dir in service_dirs: + name = service_dir.name + if selected_services and name not in selected_services: + continue + + service_override = overrides.get(name, {}) + profile = detect_profile(service_dir, service_override) + if profile is None: + report.warnings.append(f"Пропускаю `{name}`: не найден поддерживаемый профиль деплоя.") + continue + + checker_name = str(service_override.get("checker_dir", name)) + checker_dir = checkers_root / checker_name + if not checker_dir.exists(): + report.errors.append(f"`{name}`: не найдена папка чекера `{checker_dir}`.") + continue + + entry_override = service_override.get("checker_entry") + if entry_override: + checker_entry = Path(str(entry_override)) + if not (checker_dir / checker_entry).exists(): + report.errors.append(f"`{name}`: не найден checker entry `{checker_entry}` в `{checker_dir}`.") + continue + else: + detected_entry = detect_checker_entry(checker_dir) + if detected_entry is None: + report.errors.append(f"`{name}`: не найден исполняемый checker (`checker.py` или `*.checker.py`).") + continue + checker_entry = detected_entry + + entry_text = (checker_dir / checker_entry).read_text() + protocol = checker_protocol(entry_text) + detected_type = checker_type(entry_text) + detected_places = checker_places(entry_text) + + task_type = str(service_override.get("checker_type", detected_type)) + places = int(service_override.get("places", detected_places)) + timeout = int(service_override.get("checker_timeout", 30)) + host_port = discover_host_port(service_dir, profile, protocol, service_override) + requirements = load_requirements(checker_dir) + build_enabled, build_profile, build_command, build_artifacts = detect_checker_build(checker_dir, service_override) + + spec = ServiceSpec( + name=name, + service_dir=service_dir, + checker_dir=checker_dir, + profile=profile, + checker_entry=checker_entry, + checker_type=task_type, + places=places, + checker_timeout=timeout, + protocol=protocol, + host_port=host_port, + requirements=requirements, + checker_build_enabled=build_enabled, + checker_build_profile=build_profile, + checker_build_command=build_command, + checker_build_artifacts=build_artifacts, + start_cmd=service_override.get("start_cmd"), + stop_cmd=service_override.get("stop_cmd"), + ) + specs.append(spec) + + if not specs: + report.errors.append("Не найдено ни одного валидного сервиса для запуска.") + return specs, report + + +def validate_preflight(specs: List[ServiceSpec], board_port: int) -> ValidationReport: + report = ValidationReport() + require_command("docker", report) + seen_ports: Dict[int, str] = {} + + for spec in specs: + if spec.host_port is None: + report.errors.append(f"`{spec.name}`: не удалось определить host-порт (укажи `port` в `forcad-local.json`).") + else: + if spec.host_port == board_port: + report.errors.append( + f"`{spec.name}`: host-порт {spec.host_port} конфликтует с портом борды {board_port}." + ) + if spec.host_port in seen_ports: + report.errors.append( + f"Конфликт портов: `{spec.name}` и `{seen_ports[spec.host_port]}` используют {spec.host_port}." + ) + seen_ports[spec.host_port] = spec.name + + if spec.profile == "compose" and not (spec.service_dir / "docker-compose.yml").exists(): + report.errors.append(f"`{spec.name}`: профиль compose выбран, но `docker-compose.yml` отсутствует.") + + if spec.profile in {"deploy_script", "helm_chart"}: + for dep in ("helm", "kubectl"): + if shutil.which(dep) is None: + report.warnings.append( + f"`{spec.name}`: команда `{dep}` не найдена; сервис может не подняться." + ) + if (spec.service_dir / "k3d-config.yaml").exists() and shutil.which("k3d") is None: + report.warnings.append(f"`{spec.name}`: найден `k3d-config.yaml`, но команда `k3d` отсутствует.") + + if spec.checker_build_enabled: + if not spec.checker_build_profile: + report.errors.append(f"`{spec.name}`: включена сборка чекера, но не указан checker_build_profile.") + elif spec.checker_build_profile not in BUILDER_PROFILES: + report.errors.append( + f"`{spec.name}`: неизвестный checker build profile `{spec.checker_build_profile}`." + ) + if not spec.checker_build_command: + report.errors.append(f"`{spec.name}`: включена сборка чекера, но не задан checker_build_command.") + if not spec.checker_build_artifacts: + report.warnings.append( + f"`{spec.name}`: не заданы checker build artifacts, кэш сборки для него отключён." + ) + + return report + + +def merge_reports(*reports: ValidationReport) -> ValidationReport: + result = ValidationReport() + for report in reports: + result.errors.extend(report.errors) + result.warnings.extend(report.warnings) + return result + + +def print_report(report: ValidationReport) -> None: + for warning in report.warnings: + warn(warning) + for error_message in report.errors: + err(error_message) + + +def resolve_checkers_root(services_root: Path, checkers_root_arg: Optional[Path]) -> Path: + if checkers_root_arg: + return checkers_root_arg.resolve() + + sibling = services_root.parent / "checkers" + if sibling.exists(): + return sibling.resolve() + + inside = services_root / "checkers" + if inside.exists(): + return inside.resolve() + + raise RuntimeError( + "Не удалось определить папку чекеров автоматически. Укажи `--checkers-root` явно." + ) + + +def safe_service_var(service_name: str) -> str: + normalized = re.sub(r"[^A-Za-z0-9]+", "_", service_name).upper() + return f"AD_TARGET_{normalized}" + + +def ensure_clean_dir(path: Path) -> None: + if path.exists(): + shutil.rmtree(path) + path.mkdir(parents=True, exist_ok=True) + + +def resolve_builder_image(orchestrator_root: Path, profile: str) -> str: + profile_config = BUILDER_PROFILES.get(profile) + if not profile_config: + raise RuntimeError(f"Unknown builder profile: {profile}") + + image = profile_config["image"] + inspect = run_command(["docker", "image", "inspect", image], check=False) + if inspect.returncode == 0: + return image + + dockerfile = orchestrator_root / profile_config["dockerfile"] + if not dockerfile.exists(): + raise RuntimeError(f"Builder dockerfile not found for profile `{profile}`: {dockerfile}") + + info(f"Building checker builder image `{image}` from `{dockerfile}`.") + run_command( + [ + "docker", + "build", + "-f", + str(dockerfile), + "-t", + image, + str(orchestrator_root), + ] + ) + return image + + +def hash_checker_inputs(checker_dir: Path, build_profile: str, build_command: str) -> str: + digest = hashlib.sha256() + digest.update(build_profile.encode()) + digest.update(b"\0") + digest.update(build_command.encode()) + digest.update(b"\0") + + for item in sorted(checker_dir.rglob("*")): + if not item.is_file(): + continue + rel = item.relative_to(checker_dir).as_posix() + digest.update(rel.encode()) + digest.update(b"\0") + digest.update(item.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def restore_build_cache( + service: ServiceSpec, + checker_runtime_dir: Path, + cache_root: Path, + cache_key: str, +) -> bool: + if not service.checker_build_artifacts: + return False + + entry = cache_root / service.name / cache_key + if not entry.exists(): + return False + + for artifact in service.checker_build_artifacts: + src = entry / artifact + dst = checker_runtime_dir / artifact + if not src.exists(): + return False + if src.is_dir(): + if dst.exists(): + shutil.rmtree(dst) + shutil.copytree(src, dst) + else: + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + + info(f"Checker build cache hit for `{service.name}`.") + return True + + +def store_build_cache( + service: ServiceSpec, + checker_runtime_dir: Path, + cache_root: Path, + cache_key: str, +) -> None: + if not service.checker_build_artifacts: + return + + entry = cache_root / service.name / cache_key + if entry.exists(): + shutil.rmtree(entry) + entry.mkdir(parents=True, exist_ok=True) + + for artifact in service.checker_build_artifacts: + src = checker_runtime_dir / artifact + if not src.exists(): + continue + dst = entry / artifact + if src.is_dir(): + shutil.copytree(src, dst) + else: + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + + +def create_wrapper(service: ServiceSpec, destination: Path) -> None: + env_var = safe_service_var(service.name) + default_target = f"host.docker.internal:{service.host_port}" + wrapper = f"""#!/usr/bin/env python3 +import os +import subprocess +import sys +from pathlib import Path + +LOCAL_ALIASES = {sorted(LOCAL_HOST_ALIASES)!r} +DEFAULT_TARGET = os.environ.get("{env_var}", "{default_target}") + +def patch_hostname(argv): + if len(argv) < 3: + return argv + action = argv[1].lower() + if action not in {{"check", "put", "get", "test"}}: + return argv + host = argv[2] + if host in LOCAL_ALIASES: + argv[2] = DEFAULT_TARGET + elif ":" not in host and ":" in DEFAULT_TARGET: + argv[2] = f"{{host}}:{{DEFAULT_TARGET.rsplit(':', 1)[1]}}" + return argv + +def main(): + checker_path = Path(__file__).with_name("{service.checker_entry.as_posix()}") + argv = patch_hostname(sys.argv[:]) + completed = subprocess.run([sys.executable, str(checker_path), *argv[1:]], cwd=Path(__file__).parent) + raise SystemExit(completed.returncode) + +if __name__ == "__main__": + main() +""" + destination.write_text(wrapper) + destination.chmod(0o755) + + +def generate_runtime( + runtime_dir: Path, + specs: List[ServiceSpec], + team_count: int, + board_port: int, + round_time: int, + flag_lifetime: int, + start_delay_seconds: int, +) -> dict: + ensure_clean_dir(runtime_dir) + checkers_dir = runtime_dir / "checkers" + env_dir = runtime_dir / "env" + checkers_dir.mkdir(parents=True, exist_ok=True) + env_dir.mkdir(parents=True, exist_ok=True) + + requirements = {"gornilo", "checklib==0.7.0"} + task_chunks: List[str] = [] + + for spec in specs: + target_dir = checkers_dir / spec.name + shutil.copytree(spec.checker_dir, target_dir, dirs_exist_ok=True) + create_wrapper(spec, target_dir / "forcad_wrapper.py") + + for requirement in spec.requirements: + requirements.add(requirement) + + task_chunks.append( + "\n".join( + [ + f" - name: {spec.name}", + f" checker: {spec.name}/forcad_wrapper.py", + f" checker_timeout: {spec.checker_timeout}", + f" checker_type: {spec.checker_type}", + " gets: 1", + " puts: 1", + f" places: {max(1, spec.places)}", + ] + ) + ) + + (checkers_dir / "requirements.txt").write_text("\n".join(sorted(requirements)) + "\n") + + start_time = dt.datetime.now(dt.UTC) + dt.timedelta(seconds=start_delay_seconds) + start_time_text = start_time.strftime("%Y-%m-%d %H:%M:%S") + teams = "\n".join( + [ + "\n".join( + [ + " - ip: host.docker.internal", + f" name: Team {idx + 1}", + f" highlighted: {'true' if idx == 0 else 'false'}", + ] + ) + for idx in range(team_count) + ] + ) + + config_text = "\n".join( + [ + "game:", + " mode: classic", + f" round_time: {round_time}", + f" start_time: '{start_time_text}'", + " timezone: UTC", + " default_score: 2500", + f" flag_lifetime: {flag_lifetime}", + " game_hardness: 10.0", + " inflation: true", + " checkers_path: /checkers/", + " env_path: /checkers/bin/", + "admin:", + f" username: {DEFAULT_ADMIN_USER}", + f" password: {DEFAULT_ADMIN_PASSWORD}", + "tasks:", + *task_chunks, + "teams:", + teams, + "storages:", + " db:", + f" user: {DEFAULT_ADMIN_USER}", + f" password: {DEFAULT_ADMIN_PASSWORD}", + " dbname: forcad", + " host: postgres", + " port: 5432", + " redis:", + f" password: {DEFAULT_ADMIN_PASSWORD}", + " db: 0", + " host: redis", + " port: 6379", + " rabbitmq:", + f" user: {DEFAULT_ADMIN_USER}", + f" password: {DEFAULT_ADMIN_PASSWORD}", + " host: rabbitmq", + " port: 5672", + " vhost: forcad", + "", + ] + ) + config_path = runtime_dir / "config.yml" + config_path.write_text(config_text) + + postgres_env = "\n".join( + [ + "POSTGRES_HOST=postgres", + "POSTGRES_PORT=5432", + f"POSTGRES_USER={DEFAULT_ADMIN_USER}", + f"POSTGRES_PASSWORD={DEFAULT_ADMIN_PASSWORD}", + "POSTGRES_DB=forcad", + "", + ] + ) + redis_env = "\n".join( + [ + "REDIS_HOST=redis", + "REDIS_PORT=6379", + f"REDIS_PASSWORD={DEFAULT_ADMIN_PASSWORD}", + "", + ] + ) + rabbit_env = "\n".join( + [ + "RABBITMQ_HOST=rabbitmq", + "RABBITMQ_PORT=5672", + f"RABBITMQ_DEFAULT_USER={DEFAULT_ADMIN_USER}", + f"RABBITMQ_DEFAULT_PASS={DEFAULT_ADMIN_PASSWORD}", + "RABBITMQ_DEFAULT_VHOST=forcad", + f"BROKER_API_URL=http://{DEFAULT_ADMIN_USER}:{DEFAULT_ADMIN_PASSWORD}@rabbitmq:15672/api/", + "", + ] + ) + admin_env = "\n".join( + [ + f"ADMIN_USERNAME={DEFAULT_ADMIN_USER}", + f"ADMIN_PASSWORD={DEFAULT_ADMIN_PASSWORD}", + "", + ] + ) + + postgres_path = env_dir / "postgres_environment.env" + redis_path = env_dir / "redis_environment.env" + rabbit_path = env_dir / "rabbitmq_environment.env" + admin_path = env_dir / "admin.env" + postgres_path.write_text(postgres_env) + redis_path.write_text(redis_env) + rabbit_path.write_text(rabbit_env) + admin_path.write_text(admin_env) + + target_env_lines = [f" {safe_service_var(spec.name)}: host.docker.internal:{spec.host_port}" for spec in specs] + target_env = "\n".join(target_env_lines) + + forcad_override = "\n".join( + [ + "version: '3.8'", + "services:", + " celery:", + " user: root", + " volumes:", + f" - \"{checkers_dir}:/checkers/\"", + " env_file:", + f" - \"{postgres_path}\"", + f" - \"{redis_path}\"", + f" - \"{rabbit_path}\"", + " environment:", + " TEST: \"\"", + " SERVICE: worker", + target_env, + " extra_hosts:", + " - \"host.docker.internal:host-gateway\"", + " command: bash -lc \"pip install --no-cache-dir -r /checkers/requirements.txt && /entrypoint.sh\"", + "", + " flower:", + " user: root", + " volumes:", + f" - \"{checkers_dir}:/checkers/\"", + " env_file:", + f" - \"{admin_path}\"", + f" - \"{postgres_path}\"", + f" - \"{redis_path}\"", + f" - \"{rabbit_path}\"", + " environment:", + " TEST: \"\"", + " SERVICE: flower", + target_env, + " extra_hosts:", + " - \"host.docker.internal:host-gateway\"", + " command: bash -lc \"pip install --no-cache-dir -r /checkers/requirements.txt && /entrypoint.sh\"", + "", + " initializer:", + " env_file:", + f" - \"{postgres_path}\"", + f" - \"{redis_path}\"", + f" - \"{rabbit_path}\"", + " volumes:", + f" - \"{config_path}:/config.yml:ro\"", + " environment:", + " TEST: \"\"", + " CONFIG_PATH: /config.yml", + "", + " ticker:", + " env_file:", + f" - \"{postgres_path}\"", + f" - \"{redis_path}\"", + f" - \"{rabbit_path}\"", + "", + " client-api:", + " env_file:", + f" - \"{postgres_path}\"", + f" - \"{redis_path}\"", + f" - \"{rabbit_path}\"", + "", + " admin-api:", + " env_file:", + f" - \"{admin_path}\"", + f" - \"{postgres_path}\"", + f" - \"{redis_path}\"", + f" - \"{rabbit_path}\"", + "", + " events:", + " env_file:", + f" - \"{postgres_path}\"", + f" - \"{redis_path}\"", + f" - \"{rabbit_path}\"", + "", + " http-receiver:", + " env_file:", + f" - \"{postgres_path}\"", + f" - \"{redis_path}\"", + f" - \"{rabbit_path}\"", + "", + " redis:", + " env_file:", + f" - \"{redis_path}\"", + "", + " rabbitmq:", + " env_file:", + f" - \"{rabbit_path}\"", + "", + " postgres:", + " env_file:", + f" - \"{postgres_path}\"", + "", + " nginx:", + " ports:", + f" - \"{board_port}:80\"", + "", + ] + ) + + override_path = runtime_dir / "forcad.override.yml" + override_path.write_text(forcad_override) + + return { + "runtime_dir": str(runtime_dir), + "config_path": str(config_path), + "checkers_dir": str(checkers_dir), + "override_path": str(override_path), + } + + +def build_checker_artifacts( + runtime_dir: Path, + specs: List[ServiceSpec], + cache_dir: Path, + failure_policy: str = "strict", +) -> Tuple[List[ServiceSpec], Dict[str, str]]: + orchestrator_root = Path(__file__).resolve().parent + cache_root = cache_dir / "checker-builds" + cache_root.mkdir(parents=True, exist_ok=True) + builder_images: Dict[str, str] = {} + ready_specs: List[ServiceSpec] = [] + failed: Dict[str, str] = {} + + for spec in specs: + try: + if not spec.checker_build_enabled: + ready_specs.append(spec) + continue + if not spec.checker_build_profile or not spec.checker_build_command: + raise RuntimeError(f"Invalid checker build config for `{spec.name}`.") + + checker_runtime_dir = runtime_dir / "checkers" / spec.name + cache_key = hash_checker_inputs( + checker_dir=checker_runtime_dir, + build_profile=spec.checker_build_profile, + build_command=spec.checker_build_command, + ) + if restore_build_cache(spec, checker_runtime_dir, cache_root, cache_key): + ready_specs.append(spec) + continue + + if spec.checker_build_profile not in builder_images: + builder_images[spec.checker_build_profile] = resolve_builder_image( + orchestrator_root=orchestrator_root, + profile=spec.checker_build_profile, + ) + image = builder_images[spec.checker_build_profile] + + info(f"Containerized checker build for `{spec.name}` ({spec.checker_build_profile}).") + run_command( + [ + "docker", + "run", + "--rm", + "--add-host", + "host.docker.internal:host-gateway", + "-v", + f"{checker_runtime_dir}:/work", + "-w", + "/work", + image, + "bash", + "-lc", + spec.checker_build_command, + ] + ) + store_build_cache(spec, checker_runtime_dir, cache_root, cache_key) + ready_specs.append(spec) + except Exception as exc: # noqa: BLE001 + message = str(exc) + failed[spec.name] = message + if failure_policy == "strict": + raise + warn(f"Skipping `{spec.name}` due checker build failure: {message}") + + return ready_specs, failed + + +def is_port_open(port: int) -> bool: + with socket.socket() as sock: + sock.settimeout(0.5) + return sock.connect_ex(("127.0.0.1", int(port))) == 0 + + +def ensure_service_port_available(spec: ServiceSpec) -> None: + if spec.host_port is None: + return + if not is_port_open(spec.host_port): + return + warn(f"Port {spec.host_port} is busy before deploying `{spec.name}`, trying cleanup.") + stop_service(spec, include_legacy_compose=True) + time.sleep(1) + if is_port_open(spec.host_port): + raise RuntimeError( + f"Port {spec.host_port} is still busy before deploying `{spec.name}`. " + "Run cleanup and retry." + ) + + +def prewarm_k3d_images( + specs: List[ServiceSpec], + strict: bool, + image_pull_retries: int, + image_pull_retry_delay: int, +) -> None: + if not any(is_k3d_service(spec) for spec in specs): + return + for image in K3D_PREWARM_IMAGES: + try: + run_command_with_retries( + ["docker", "pull", image], + retries=image_pull_retries, + retry_delay=image_pull_retry_delay, + purpose=f"prewarm pull `{image}`", + ) + except Exception as exc: # noqa: BLE001 + message = f"Failed to prewarm image `{image}`: {exc}" + if strict: + raise RuntimeError(message) from exc + warn(message) + + +def auto_cleanup_services(specs: List[ServiceSpec]) -> None: + info("Auto-cleanup previous service runs.") + for spec in reversed(specs): + stop_service(spec, include_legacy_compose=True) + + +def rollback_services(specs: List[ServiceSpec]) -> None: + if not specs: + return + warn("Rolling back already deployed services.") + for spec in reversed(specs): + stop_service(spec, include_legacy_compose=True) + + +def deploy_service_with_retry( + spec: ServiceSpec, + k3d_retries: int, + retry_delay: int, + image_pull_retries: int, + image_pull_retry_delay: int, +) -> None: + attempts = 1 + (k3d_retries if is_k3d_service(spec) else 0) + for attempt in range(1, attempts + 1): + try: + ensure_service_port_available(spec) + deploy_service( + spec, + image_pull_retries=image_pull_retries, + image_pull_retry_delay=image_pull_retry_delay, + ) + return + except Exception as exc: # noqa: BLE001 + if attempt >= attempts: + raise + warn(f"Deploy `{spec.name}` attempt {attempt}/{attempts} failed: {exc}") + stop_service(spec, include_legacy_compose=True) + time.sleep(retry_delay) + + +def compose_project_names(spec: ServiceSpec) -> List[str]: + names: List[str] = [f"forcadsvc_{spec.name}"] + legacy = spec.service_dir.name + if legacy not in names: + names.append(legacy) + return names + + +def is_k3d_service(spec: ServiceSpec) -> bool: + if (spec.service_dir / "k3d-config.yaml").exists(): + return True + deploy_script = spec.service_dir / "deploy.sh" + if deploy_script.exists(): + return "k3d cluster create" in deploy_script.read_text() + return False + + +def service_priority(spec: ServiceSpec) -> Tuple[int, str]: + # Start k3d-heavy services first while Docker daemon is still idle. + if is_k3d_service(spec): + return (0, spec.name) + if spec.profile == "compose": + return (1, spec.name) + return (2, spec.name) + + +def order_services_for_deploy(specs: List[ServiceSpec]) -> List[ServiceSpec]: + return sorted(specs, key=service_priority) + + +def deploy_service( + spec: ServiceSpec, + *, + image_pull_retries: int = 0, + image_pull_retry_delay: int = 5, +) -> None: + if spec.start_cmd: + run_shell(spec.start_cmd, cwd=spec.service_dir) + return + + if spec.profile == "compose": + compose_path = spec.service_dir / "docker-compose.yml" + project = compose_project_names(spec)[0] + # Pull externally referenced images with retries before "up". + run_command_with_retries( + [ + "docker", + "compose", + "-f", + str(compose_path), + "-p", + project, + "pull", + ], + retries=image_pull_retries, + retry_delay=image_pull_retry_delay, + purpose=f"image pull for `{spec.name}`", + ) + run_command( + [ + "docker", + "compose", + "-f", + str(compose_path), + "-p", + project, + "up", + "-d", + "--build", + "--remove-orphans", + ] + ) + return + + deploy_script = spec.service_dir / "deploy.sh" + if deploy_script.exists(): + run_command(["bash", str(deploy_script)], cwd=spec.service_dir) + return + + redeploy_script = spec.service_dir / "redeploy.sh" + if redeploy_script.exists(): + run_command(["bash", str(redeploy_script)], cwd=spec.service_dir) + return + + build_script = spec.service_dir / "build-docker.sh" + chart_dir = spec.service_dir / "deploy" / "chart" + if build_script.exists(): + run_command(["bash", str(build_script)], cwd=spec.service_dir) + if chart_dir.exists(): + run_command( + [ + "helm", + "upgrade", + "--install", + spec.name, + str(chart_dir), + "--create-namespace", + "--namespace", + spec.name, + "--wait", + "--timeout", + "10m", + ], + cwd=spec.service_dir, + ) + return + + raise RuntimeError(f"Не знаю, как поднять сервис `{spec.name}` (profile={spec.profile}).") + + +def stop_service(spec: ServiceSpec, include_legacy_compose: bool = True) -> None: + if spec.stop_cmd: + run_shell(spec.stop_cmd, cwd=spec.service_dir, check=False) + return + + if spec.profile == "compose": + compose_path = spec.service_dir / "docker-compose.yml" + projects = compose_project_names(spec) + if not include_legacy_compose: + projects = projects[:1] + for project in projects: + run_command( + ["docker", "compose", "-f", str(compose_path), "-p", project, "down", "-v", "--remove-orphans"], + check=False, + ) + return + + if (spec.service_dir / "k3d-config.yaml").exists() and shutil.which("k3d"): + config_text = (spec.service_dir / "k3d-config.yaml").read_text() + match = re.search(r"\bname:\s*([A-Za-z0-9_.-]+)", config_text) + cluster_name = match.group(1) if match else f"{spec.name}-cluster" + run_command(["k3d", "cluster", "delete", cluster_name], check=False) + + if shutil.which("helm"): + run_command(["helm", "uninstall", spec.name, "-n", spec.name], check=False) + + +def wait_for_service(spec: ServiceSpec, timeout_seconds: int = 180) -> None: + if spec.host_port is None: + raise RuntimeError(f"У `{spec.name}` не определён host_port.") + info(f"Ожидаю доступность `{spec.name}` на порту {spec.host_port} ...") + deadline = time.time() + timeout_seconds + while time.time() < deadline: + with socket.socket() as sock: + sock.settimeout(2.0) + try: + sock.connect(("127.0.0.1", int(spec.host_port))) + info(f"`{spec.name}` доступен на 127.0.0.1:{spec.host_port}") + return + except OSError: + time.sleep(2) + raise RuntimeError(f"`{spec.name}` не стал доступен на порту {spec.host_port} за {timeout_seconds} сек.") + + +def forcad_compose_files(forcad_root: Path, override_path: Path, fast: bool) -> List[str]: + files = [str(forcad_root / "docker-compose.yml")] + if fast: + files.append(str(forcad_root / "docker-compose-fast.yml")) + files.append(str(override_path)) + return files + + +def run_forcad_up(forcad_root: Path, override_path: Path, fast: bool, workers: int) -> None: + compose_files = forcad_compose_files(forcad_root, override_path, fast) + command = ["docker", "compose"] + for path in compose_files: + command += ["-f", path] + command += ["up", "-d", "--build", "--scale", f"celery={workers}"] + run_command(command, cwd=forcad_root) + + +def run_forcad_down(forcad_root: Path, override_path: Path, fast: bool) -> None: + compose_files = forcad_compose_files(forcad_root, override_path, fast) + command = ["docker", "compose"] + for path in compose_files: + command += ["-f", path] + command += ["down", "-v", "--remove-orphans"] + run_command(command, cwd=forcad_root, check=False) + + +def run_forcad_status(forcad_root: Path, override_path: Path, fast: bool) -> None: + compose_files = forcad_compose_files(forcad_root, override_path, fast) + command = ["docker", "compose"] + for path in compose_files: + command += ["-f", path] + command += ["ps"] + run_command(command, cwd=forcad_root, check=False) + + +def run_smoke_checks(forcad_root: Path, override_path: Path, fast: bool, specs: List[ServiceSpec]) -> None: + compose_files = forcad_compose_files(forcad_root, override_path, fast) + for spec in specs: + command = ["docker", "compose"] + for path in compose_files: + command += ["-f", path] + command += [ + "exec", + "-T", + "celery", + f"/checkers/{spec.name}/forcad_wrapper.py", + "check", + "host.docker.internal", + ] + info(f"Smoke-check `{spec.name}` ...") + result = run_command(command, cwd=forcad_root, check=False) + if result.returncode == 0: + info(f"Smoke-check `{spec.name}`: OK") + else: + warn(f"Smoke-check `{spec.name}` завершился с кодом {result.returncode}.") + + +def run_light_smoke_checks(runtime_dir: Path, specs: List[ServiceSpec]) -> None: + checkers_dir = runtime_dir / "checkers" + quoted_services = " ".join(shlex.quote(spec.name) for spec in specs) + command = ( + "python3 -m pip install --no-cache-dir -r /checkers/requirements.txt >/tmp/pip.log 2>&1 && " + "failed=0; " + f"for svc in {quoted_services}; do " + "echo \"[LIGHT] checker check $svc\"; " + "python3 /checkers/$svc/forcad_wrapper.py check host.docker.internal || failed=1; " + "done; " + "exit $failed" + ) + result = run_command( + [ + "docker", + "run", + "--rm", + "--add-host", + "host.docker.internal:host-gateway", + "-v", + f"{checkers_dir}:/checkers", + "python:3.11-slim", + "bash", + "-lc", + command, + ], + check=False, + ) + if result.returncode == 0: + info("Light smoke-checks completed successfully.") + else: + warn(f"Light smoke-checks completed with non-zero status: {result.returncode}") + + +def save_state(runtime_dir: Path, payload: dict) -> None: + state_path = runtime_dir / "state.json" + state_path.write_text(json.dumps(payload, indent=2)) + + +def load_state(runtime_dir: Path) -> Optional[dict]: + state_path = runtime_dir / "state.json" + if not state_path.exists(): + return None + return json.loads(state_path.read_text()) + + +def load_context_from_args(args: argparse.Namespace, allow_state: bool = False) -> Tuple[Path, Path, Path]: + forcad_root = args.forcad_root.resolve() + runtime_dir = args.runtime_dir.resolve() + + services_root = args.services_root.resolve() if args.services_root else None + checkers_root = args.checkers_root.resolve() if args.checkers_root else None + + if services_root is None and allow_state: + state = load_state(runtime_dir) + if state: + services_root = Path(state["services_root"]) + checkers_root = Path(state["checkers_root"]) + forcad_root = Path(state["forcad_root"]) + + if services_root is None: + raise RuntimeError("Нужно указать `--services-root` или иметь сохранённый state от `up`.") + if checkers_root is None: + checkers_root = resolve_checkers_root(services_root, None) + + return forcad_root, services_root, checkers_root + + +def cmd_validate(args: argparse.Namespace) -> int: + forcad_root, services_root, checkers_root = load_context_from_args(args, allow_state=False) + if not forcad_root.exists(): + err(f"Не найден ForcAD root: {forcad_root}") + return 2 + + selected = set(args.service) if args.service else None + specs, discovery_report = discover_services(services_root, checkers_root, selected) + validation_report = validate_preflight(specs, args.board_port) + report = merge_reports(discovery_report, validation_report) + + print_report(report) + if not report.ok(): + return 2 + + info("Preflight-проверка пройдена.") + for spec in specs: + info( + f"service={spec.name} profile={spec.profile} checker={spec.checker_entry} " + f"type={spec.checker_type} places={spec.places} port={spec.host_port}" + ) + return 0 + + +def cmd_up(args: argparse.Namespace) -> int: + forcad_root, services_root, checkers_root = load_context_from_args(args, allow_state=False) + selected = set(args.service) if args.service else None + specs, discovery_report = discover_services(services_root, checkers_root, selected) + validation_report = validate_preflight(specs, args.board_port) + report = merge_reports(discovery_report, validation_report) + print_report(report) + if not report.ok(): + return 2 + + runtime_dir = args.runtime_dir.resolve() + runtime_meta = generate_runtime( + runtime_dir=runtime_dir, + specs=specs, + team_count=args.team_count, + board_port=args.board_port, + round_time=args.round_time, + flag_lifetime=args.flag_lifetime, + start_delay_seconds=args.start_delay_seconds, + ) + + failure_policy = args.failure_policy.lower() + strict = failure_policy == "strict" + ordered_specs = order_services_for_deploy(specs) + + if args.auto_cleanup: + auto_cleanup_services(ordered_specs) + if args.prewarm_k3d: + prewarm_k3d_images( + ordered_specs, + strict=strict, + image_pull_retries=args.image_pull_retries, + image_pull_retry_delay=args.image_pull_retry_delay, + ) + + ready_specs, build_failed = build_checker_artifacts( + runtime_dir, + ordered_specs, + cache_dir=args.cache_dir.resolve(), + failure_policy=failure_policy, + ) + if build_failed and strict: + raise RuntimeError(f"Checker build failed: {build_failed}") + if build_failed and not strict: + warn(f"Checker build failures (best-effort): {build_failed}") + if not ready_specs: + raise RuntimeError("No services left after checker build stage.") + + deployed_specs: List[ServiceSpec] = [] + deploy_failed: Dict[str, str] = {} + for spec in ready_specs: + try: + deploy_service_with_retry( + spec, + k3d_retries=args.k3d_retries, + retry_delay=args.k3d_retry_delay, + image_pull_retries=args.image_pull_retries, + image_pull_retry_delay=args.image_pull_retry_delay, + ) + wait_for_service(spec, timeout_seconds=args.service_wait_timeout) + deployed_specs.append(spec) + except Exception as exc: # noqa: BLE001 + deploy_failed[spec.name] = str(exc) + if strict: + if args.rollback_on_failure: + rollback_services(deployed_specs + [spec]) + raise + warn(f"Skipping `{spec.name}` after deploy failure: {exc}") + + if not deployed_specs: + raise RuntimeError("No services were deployed successfully.") + if deploy_failed and not strict: + warn(f"Deploy failures (best-effort): {deploy_failed}") + + mode = args.mode.lower() + override_path = Path(runtime_meta["override_path"]) + if mode == "full": + run_forcad_up(forcad_root, override_path, fast=not args.no_fast, workers=args.workers) + info("Ожидание стабилизации ForcAD (20 секунд)...") + time.sleep(20) + run_smoke_checks(forcad_root, override_path, fast=not args.no_fast, specs=deployed_specs) + else: + info("Running in light mode: full ForcAD stack is not started.") + run_light_smoke_checks(runtime_dir, deployed_specs) + + state = { + "forcad_root": str(forcad_root), + "services_root": str(services_root), + "checkers_root": str(checkers_root), + "mode": mode, + "fast": not args.no_fast, + "board_port": args.board_port, + "runtime": runtime_meta, + "failure_policy": failure_policy, + "deploy_failed": deploy_failed, + "build_failed": build_failed, + "services": [ + { + "name": spec.name, + "profile": spec.profile, + "service_dir": str(spec.service_dir), + "checker_dir": str(spec.checker_dir), + "checker_build_enabled": spec.checker_build_enabled, + "checker_build_profile": spec.checker_build_profile, + "checker_build_command": spec.checker_build_command, + "checker_build_artifacts": spec.checker_build_artifacts, + "host_port": spec.host_port, + "start_cmd": spec.start_cmd, + "stop_cmd": spec.stop_cmd, + } + for spec in deployed_specs + ], + } + save_state(runtime_dir, state) + + if mode == "full": + info(f"Борда должна быть доступна по адресу http://127.0.0.1:{args.board_port}/") + else: + info("Light mode complete: services are up and checker liveness checks were executed.") + info("Готово.") + return 0 + + +def cmd_down(args: argparse.Namespace) -> int: + runtime_dir = args.runtime_dir.resolve() + state = load_state(runtime_dir) + if state is None and not args.services_root: + err("Не найден state.json. Укажи `--services-root` и `--checkers-root`.") + return 2 + + if state is not None: + forcad_root = Path(state["forcad_root"]) + override_path = Path(state["runtime"]["override_path"]) + fast = bool(state.get("fast", True)) + mode = str(state.get("mode", "light")).lower() + specs = [ + ServiceSpec( + name=s["name"], + profile=s["profile"], + service_dir=Path(s["service_dir"]), + checker_dir=Path(s["checker_dir"]), + checker_entry=Path("checker.py"), + checker_type="hackerdom", + places=1, + checker_timeout=30, + protocol="unknown", + checker_build_enabled=bool(s.get("checker_build_enabled", False)), + checker_build_profile=s.get("checker_build_profile"), + checker_build_command=s.get("checker_build_command"), + checker_build_artifacts=[str(x) for x in s.get("checker_build_artifacts", [])], + host_port=s.get("host_port"), + start_cmd=s.get("start_cmd"), + stop_cmd=s.get("stop_cmd"), + ) + for s in state["services"] + ] + else: + forcad_root, services_root, checkers_root = load_context_from_args(args, allow_state=False) + discovered, report = discover_services(services_root, checkers_root, set(args.service) if args.service else None) + print_report(report) + if not discovered: + return 2 + specs = discovered + override_path = runtime_dir / "forcad.override.yml" + fast = not args.no_fast + mode = args.mode.lower() + + if mode == "full": + run_forcad_down(forcad_root, override_path, fast=fast) + for spec in specs: + stop_service(spec) + + info("Остановка завершена.") + return 0 + + +def cmd_status(args: argparse.Namespace) -> int: + runtime_dir = args.runtime_dir.resolve() + state = load_state(runtime_dir) + if state is None: + err("Не найден state.json. Сначала выполни `up`.") + return 2 + + forcad_root = Path(state["forcad_root"]) + override_path = Path(state["runtime"]["override_path"]) + fast = bool(state.get("fast", True)) + mode = str(state.get("mode", "light")).lower() + if mode == "full": + run_forcad_status(forcad_root, override_path, fast=fast) + else: + info("Light mode state: ForcAD stack was not started.") + + for service in state["services"]: + port = service.get("host_port") + if not port: + warn(f"{service['name']}: порт неизвестен.") + continue + with socket.socket() as sock: + sock.settimeout(1.0) + try: + sock.connect(("127.0.0.1", int(port))) + info(f"{service['name']}: 127.0.0.1:{port} открыт") + except OSError: + warn(f"{service['name']}: 127.0.0.1:{port} недоступен") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + default_workspace = Path(__file__).resolve().parents[2] + default_forcad = default_workspace / "ForcAD" + + parser = argparse.ArgumentParser( + description="Локальный оркестратор ForcAD: auto-discovery сервисов, preflight, up/down/status." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + def add_common_flags(subparser: argparse.ArgumentParser, services_required: bool) -> None: + subparser.add_argument("--forcad-root", type=Path, default=default_forcad) + subparser.add_argument("--services-root", type=Path, required=services_required) + subparser.add_argument("--checkers-root", type=Path) + subparser.add_argument("--runtime-dir", type=Path, default=Path(__file__).resolve().parent / ".runtime") + subparser.add_argument("--service", action="append", help="Имя сервиса (можно повторять). Если не задано, берутся все.") + subparser.add_argument("--board-port", type=int, default=8080) + subparser.add_argument("--no-fast", action="store_true", help="Не использовать docker-compose-fast.yml для ForcAD.") + + validate = subparsers.add_parser("validate", help="Проверка структуры сервисов/чекеров без запуска.") + add_common_flags(validate, services_required=True) + validate.set_defaults(handler=cmd_validate) + + up = subparsers.add_parser("up", help="Поднять сервисы, борду и запустить smoke-checks.") + add_common_flags(up, services_required=True) + up.add_argument("--mode", choices=["light", "full"], default="light", help="light: без ForcAD stack; full: с полным ForcAD stack.") + up.add_argument("--cache-dir", type=Path, default=Path(__file__).resolve().parent / ".cache") + up.add_argument("--failure-policy", choices=["strict", "best-effort"], default="strict") + up.add_argument("--rollback-on-failure", action=argparse.BooleanOptionalAction, default=True) + up.add_argument("--auto-cleanup", action=argparse.BooleanOptionalAction, default=True) + up.add_argument("--prewarm-k3d", action=argparse.BooleanOptionalAction, default=True) + up.add_argument("--k3d-retries", type=int, default=1) + up.add_argument("--k3d-retry-delay", type=int, default=5) + up.add_argument( + "--image-pull-retries", + type=int, + default=2, + help="Additional retries for docker image pulls (compose/prewarm).", + ) + up.add_argument( + "--image-pull-retry-delay", + type=int, + default=10, + help="Delay (seconds) between docker pull retries.", + ) + up.add_argument("--team-count", type=int, default=2) + up.add_argument("--workers", type=int, default=2) + up.add_argument("--round-time", type=int, default=120) + up.add_argument("--flag-lifetime", type=int, default=5) + up.add_argument("--start-delay-seconds", type=int, default=120) + up.add_argument("--service-wait-timeout", type=int, default=180) + up.set_defaults(handler=cmd_up) + + down = subparsers.add_parser("down", help="Остановить ForcAD и игровые сервисы.") + add_common_flags(down, services_required=False) + down.add_argument("--mode", choices=["light", "full"], default="light", help="Используется если state.json отсутствует.") + down.set_defaults(handler=cmd_down) + + status = subparsers.add_parser("status", help="Показать статус запущенной инфраструктуры.") + status.add_argument("--runtime-dir", type=Path, default=Path(__file__).resolve().parent / ".runtime") + status.set_defaults(handler=cmd_status) + + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + try: + return int(args.handler(args)) + except KeyboardInterrupt: + err("Прервано пользователем.") + return 130 + except Exception as exc: # noqa: BLE001 + err(str(exc)) + return 1 + + +if __name__ == "__main__": + sys.exit(main())