#!/usr/bin/env python3 from __future__ import annotations import argparse import hashlib import json import os import re import shlex import signal import shutil import socket import subprocess import sys import tempfile import time from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field from pathlib import Path from typing import Dict, Iterable, List, Optional, Sequence, Tuple LOCAL_HOST_ALIASES = {"127.0.0.1", "localhost", "host.docker.internal", "local", "test-service"} BUILDER_PROFILES = { "cpp-make": { "dockerfile": "builders/cpp-make.Dockerfile", "image": "forcad-local-builder-cpp-make:1", }, } K3D_PREWARM_IMAGES = [ "ghcr.io/k3d-io/k3d-tools:5.9.0", "rancher/k3s:v1.31.5-k3s1", ] SCRIPT_START_CANDIDATES = ("server.sh", "run.sh", "start.sh") SCRIPT_STOP_CANDIDATES = ("stop.sh", "shutdown.sh") COMPOSE_FILE_CANDIDATES = ("docker-compose.yml", "docker-compose.yaml") @dataclass class ServiceSpec: name: str service_dir: Path checker_dir: Path profile: str checker_entry: Path checker_type: str places: int checker_timeout: int protocol: str host_port: Optional[int] host_transport: str = "tcp" checker_port_env: Optional[str] = None expects_host_with_port: bool = False requirements: List[str] = field(default_factory=list) checker_build_enabled: bool = False checker_build_profile: Optional[str] = None checker_build_command: Optional[str] = None checker_build_artifacts: List[str] = field(default_factory=list) start_cmd: Optional[str] = None stop_cmd: Optional[str] = None @dataclass class ValidationReport: errors: List[str] = field(default_factory=list) warnings: List[str] = field(default_factory=list) def ok(self) -> bool: return not self.errors def info(message: str) -> None: print(f"[INFO] {message}") def warn(message: str) -> None: print(f"[WARN] {message}") def err(message: str) -> None: print(f"[ERROR] {message}", file=sys.stderr) class CommandTimeoutError(RuntimeError): """Raised when a subprocess exceeds its allotted timeout.""" def __init__(self, printable: str, timeout: float): super().__init__(f"Command timed out after {timeout}s: {printable}") self.printable = printable self.timeout = timeout def run_command( command: Sequence[str], cwd: Optional[Path] = None, check: bool = True, timeout: Optional[float] = None, ) -> subprocess.CompletedProcess: printable = " ".join(command) location = f" (cwd={cwd})" if cwd else "" info(f"Running: {printable}{location}") try: result = subprocess.run(command, cwd=cwd, text=True, timeout=timeout) except subprocess.TimeoutExpired as exc: # Explicit, catchable failure instead of hanging the whole `up` forever. raise CommandTimeoutError(printable, timeout) from exc if check and result.returncode != 0: raise RuntimeError(f"Command failed with exit code {result.returncode}: {printable}") return result def run_shell( command: str, cwd: Optional[Path] = None, check: bool = True, timeout: Optional[float] = None, ) -> subprocess.CompletedProcess: location = f" (cwd={cwd})" if cwd else "" info(f"Running shell: {command}{location}") try: result = subprocess.run(["bash", "-lc", command], cwd=cwd, text=True, timeout=timeout) except subprocess.TimeoutExpired as exc: raise CommandTimeoutError(command, timeout) from exc if check and result.returncode != 0: raise RuntimeError(f"Command failed with exit code {result.returncode}: {command}") return result def run_command_with_retries( command: Sequence[str], *, cwd: Optional[Path] = None, retries: int = 0, retry_delay: int = 5, purpose: str = "command", ) -> subprocess.CompletedProcess: attempts = 1 + max(0, retries) last_exception: Optional[Exception] = None for attempt in range(1, attempts + 1): try: return run_command(command, cwd=cwd, check=True) except Exception as exc: # noqa: BLE001 last_exception = exc if attempt >= attempts: raise warn( f"{purpose} failed ({attempt}/{attempts}): {exc}. " f"Retrying in {retry_delay}s..." ) time.sleep(retry_delay) if last_exception: raise last_exception raise RuntimeError(f"{purpose} failed for unknown reason") def require_command(command: str, report: ValidationReport) -> None: if shutil.which(command) is None: report.errors.append(f"Не найдена системная команда `{command}`.") def parse_manifest(services_root: Path) -> Dict[str, dict]: manifest_path = services_root / "forcad-local.json" if not manifest_path.exists(): return {} try: data = json.loads(manifest_path.read_text()) except json.JSONDecodeError as exc: raise RuntimeError(f"Некорректный JSON в {manifest_path}: {exc}") from exc services = data.get("services", {}) if not isinstance(services, dict): raise RuntimeError(f"`services` в {manifest_path} должен быть объектом.") return services def detect_profile(service_dir: Path, overrides: dict) -> Optional[str]: if "profile" in overrides: return str(overrides["profile"]) if discover_compose_file(service_dir) is not None: return "compose" if (service_dir / "deploy.sh").exists(): return "deploy_script" if (service_dir / "redeploy.sh").exists() or (service_dir / "deploy" / "chart" / "values.yaml").exists(): return "helm_chart" if detect_service_start_script(service_dir) is not None: return "script" return None def detect_service_start_script(service_dir: Path) -> Optional[Path]: for script_name in SCRIPT_START_CANDIDATES: candidate = service_dir / script_name if candidate.exists() and candidate.is_file(): return candidate return None def detect_service_stop_script(service_dir: Path) -> Optional[Path]: for script_name in SCRIPT_STOP_CANDIDATES: candidate = service_dir / script_name if candidate.exists() and candidate.is_file(): return candidate return None def discover_compose_file(service_dir: Path) -> Optional[Path]: for compose_name in COMPOSE_FILE_CANDIDATES: candidate = service_dir / compose_name if candidate.exists() and candidate.is_file(): return candidate return None def detect_checker_entry(checker_dir: Path) -> Optional[Path]: checker_py = checker_dir / "checker.py" if checker_py.exists(): return Path("checker.py") dotted = sorted(checker_dir.glob("*.checker.py")) if len(dotted) == 1: return Path(dotted[0].name) if len(dotted) > 1: return Path(dotted[0].name) checker_named = sorted(p for p in checker_dir.glob("*.py") if "checker" in p.name.lower()) if len(checker_named) == 1: return Path(checker_named[0].name) for candidate_name in ("checker", "checker.bin", "checker.out"): candidate = checker_dir / candidate_name if candidate.exists() and candidate.is_file(): return Path(candidate.name) executable_files = sorted( p for p in checker_dir.iterdir() if p.is_file() and p.suffix != ".py" and os.access(p, os.X_OK) ) if len(executable_files) == 1: return Path(executable_files[0].name) checker_like_execs = [p for p in executable_files if "checker" in p.name.lower()] if checker_like_execs: return Path(checker_like_execs[0].name) return None def is_python_checker_entry(entry: Path) -> bool: return entry.suffix.lower() == ".py" def checker_protocol(entry_content: str) -> str: if "https://" in entry_content: return "https" if "http://" in entry_content: return "http" if ".split(':')" in entry_content or "split(\":\")" in entry_content: return "tcp" return "unknown" def checker_type(entry_content: str) -> str: return "pfr" if "OK_WITH_FLAG_ID" in entry_content else "hackerdom" def checker_places(entry_content: str) -> int: count = len(re.findall(r"define_vuln", entry_content)) return max(1, count) def detect_checker_default_port(entry_content: str) -> Optional[int]: candidates: List[int] = [] patterns = [ r"\bPORT\b[^=\n]*=\s*(\d{2,5})", r"\bport\s*=\s*(\d{2,5})", r":\s*(\d{2,5})\b", ] for pattern in patterns: for match in re.findall(pattern, entry_content): try: value = int(match) except ValueError: continue if 1 <= value <= 65535: candidates.append(value) if not candidates: return None # Prefer non-system and app-like ports when multiple constants appear. for candidate in candidates: if candidate >= 1024: return candidate return candidates[0] def detect_checker_port_env(entry_content: str) -> Optional[str]: matches = re.findall(r'os\.getenv\("([A-Z0-9_]*PORT[A-Z0-9_]*)"', entry_content) if not matches: return None # Prefer service-specific names over generic PORT. matches = [item for item in matches if item != "PORT"] + [item for item in matches if item == "PORT"] return matches[0] def checker_expects_host_with_port(entry_content: str, port_env_name: Optional[str]) -> bool: # If checker has explicit port env, pass host without port and inject env. if port_env_name: return False # Some checkers explicitly parse host:port from host argv. # Important: avoid broad ".split(':')" checks, they trigger on unrelated # values like flag_id.split(":") and cause host:port:port duplication. explicit_host_split = re.search( r"\b(?:host|hostname|request\.hostname|args\.host|argv\[2\]|sys\.argv\[2\])" r"\s*\.split\(\s*['\"]:\s*['\"]\s*\)", entry_content, ) if explicit_host_split: return True # HTTP(S) checkers often construct URL directly from hostname. if "request.hostname" in entry_content and ("http://" in entry_content or "https://" in entry_content): return True return False def infer_make_artifacts(makefile_path: Path) -> List[str]: makefile_text = makefile_path.read_text() artifacts = re.findall(r"-o\s+([A-Za-z0-9_.\-\/]+)", makefile_text) return sorted(set(artifacts)) def infer_checker_artifacts(checker_dir: Path) -> List[str]: artifacts: List[str] = [] for py_file in checker_dir.glob("*.py"): try: text = py_file.read_text() except UnicodeDecodeError: continue if "./libgen" in text or "libgen" in text: artifacts.append("libgen") return sorted(set(artifacts)) def detect_checker_build(checker_dir: Path, service_override: dict) -> Tuple[bool, Optional[str], Optional[str], List[str]]: raw_cfg = service_override.get("checker_build") if isinstance(raw_cfg, dict): enabled = bool(raw_cfg.get("enabled", True)) if not enabled: return False, None, None, [] profile = str(raw_cfg.get("profile", "cpp-make")) command = str(raw_cfg.get("command", "make")) raw_artifacts = raw_cfg.get("artifacts", []) artifacts = [str(x) for x in raw_artifacts] if isinstance(raw_artifacts, list) else [] return True, profile, command, artifacts makefile = checker_dir / "Makefile" if makefile.exists(): artifacts = infer_make_artifacts(makefile) or infer_checker_artifacts(checker_dir) return True, "cpp-make", "make", artifacts return False, None, None, [] def _read_requirements_file(path: Path) -> List[str]: requirements: List[str] = [] if not path.exists(): return requirements for line in path.read_text().splitlines(): stripped = line.strip() if not stripped or stripped.startswith("#"): continue requirements.append(stripped) return requirements def load_requirements(checker_dir: Path, checkers_root: Path) -> List[str]: requirements: List[str] = [] # Global checkers requirements are common in some CTF sets (e.g. one file for all checkers). # Add them first so per-checker requirements can extend/override pins if needed. candidates = [ checkers_root / "requirements.txt", checkers_root / "requirments.txt", checker_dir / "requirements.txt", checker_dir / "requirments.txt", ] seen: set[str] = set() for candidate in candidates: for requirement in _read_requirements_file(candidate): normalized = requirement.strip() if normalized in seen: continue seen.add(normalized) requirements.append(normalized) return requirements def _discover_compose_port_mapping( service_dir: Path, *, protocol: str = "unknown", preferred_port: Optional[int] = None, ) -> Optional[Tuple[int, str]]: compose_path = discover_compose_file(service_dir) if compose_path is None: return None text = compose_path.read_text() matches = re.findall(r"(\d{2,5})\s*:\s*(\d{2,5})(?:/(\w+))?", text) if not matches: return None mappings: List[Tuple[int, int, str]] = [] for host_port, container_port, transport in matches: mappings.append((int(host_port), int(container_port), (transport.lower() if transport else "tcp"))) if preferred_port is not None: for host_port, container_port, transport in mappings: if preferred_port in {host_port, container_port}: return host_port, transport normalized_protocol = protocol.strip().lower() if normalized_protocol == "udp": for host_port, _container_port, transport in mappings: if transport == "udp": return host_port, transport if normalized_protocol in {"http", "https"}: web_ports = {80, 443, 3000, 5000, 5001, 5173, 8000, 8080, 8081, 8888, 9000} for host_port, container_port, transport in mappings: if transport == "tcp" and (container_port in web_ports or host_port in web_ports): return host_port, transport infra_ports = {5432, 3306, 1521, 27017, 6379, 5672, 15672, 9092} for host_port, container_port, transport in mappings: if host_port not in infra_ports and container_port not in infra_ports: return host_port, transport host_port, _container_port, transport = mappings[0] return host_port, transport def discover_host_port( service_dir: Path, profile: str, protocol: str, overrides: dict, preferred_checker_port: Optional[int] = None, ) -> Optional[int]: if "port" in overrides: try: return int(overrides["port"]) except (TypeError, ValueError): return None if profile == "compose": mapping = _discover_compose_port_mapping( service_dir, protocol=protocol, preferred_port=preferred_checker_port, ) if mapping is None: return None return mapping[0] values_files = list(service_dir.glob("**/values.yaml")) node_ports: List[int] = [] for path in values_files: text = path.read_text() for match in re.findall(r"nodePort:\s*(\d{2,5})", text): node_ports.append(int(match)) # Some charts (e.g. ingress-nginx values) keep NodePorts as: # nodePorts: # http: 31080 # https: 31443 if not node_ports: for match in re.findall(r":\s*(\d{2,5})\b", text): value = int(match) if value >= 30000: node_ports.append(value) if node_ports: if protocol == "http": return min(node_ports) return max(node_ports) deploy_script = service_dir / "deploy.sh" if deploy_script.exists(): script_text = deploy_script.read_text() direct = re.search(r"localhost:(\d{2,5})", script_text) if direct: return int(direct.group(1)) start_script = detect_service_start_script(service_dir) if start_script is not None: script_text = start_script.read_text() direct = re.search(r"localhost:(\d{2,5})", script_text) if direct: return int(direct.group(1)) port_assign = re.search(r"\bPORT\s*=\s*\"?(\d{2,5})\"?", script_text) if port_assign: return int(port_assign.group(1)) return None def discover_host_transport( service_dir: Path, profile: str, protocol: str, overrides: dict, preferred_checker_port: Optional[int] = None, ) -> str: if "transport" in overrides: return str(overrides["transport"]).strip().lower() if profile == "compose": mapping = _discover_compose_port_mapping( service_dir, protocol=protocol, preferred_port=preferred_checker_port, ) if mapping is not None: return mapping[1] if str(protocol).strip().lower() == "udp": return "udp" return "tcp" def discover_services( services_root: Path, checkers_root: Path, selected_services: Optional[set[str]] = None, ) -> Tuple[List[ServiceSpec], ValidationReport]: report = ValidationReport() overrides = parse_manifest(services_root) if not services_root.exists(): report.errors.append(f"Папка сервисов не найдена: {services_root}") return [], report if not checkers_root.exists(): report.errors.append(f"Папка чекеров не найдена: {checkers_root}") return [], report specs: List[ServiceSpec] = [] service_dirs = sorted(p for p in services_root.iterdir() if p.is_dir() and not p.name.startswith(".")) for service_dir in service_dirs: name = service_dir.name if selected_services and name not in selected_services: continue service_override = overrides.get(name, {}) profile = detect_profile(service_dir, service_override) if profile is None: report.warnings.append(f"Пропускаю `{name}`: не найден поддерживаемый профиль деплоя.") continue checker_name = str(service_override.get("checker_dir", name)) checker_dir = checkers_root / checker_name if not checker_dir.exists(): report.errors.append(f"`{name}`: не найдена папка чекера `{checker_dir}`.") continue entry_override = service_override.get("checker_entry") if entry_override: checker_entry = Path(str(entry_override)) if not (checker_dir / checker_entry).exists(): report.errors.append(f"`{name}`: не найден checker entry `{checker_entry}` в `{checker_dir}`.") continue else: detected_entry = detect_checker_entry(checker_dir) if detected_entry is None: report.errors.append( f"`{name}`: не найден checker entry (`checker.py`, `*.checker.py` или бинарник checker*)." ) continue checker_entry = detected_entry checker_path = checker_dir / checker_entry entry_text = "" if is_python_checker_entry(checker_entry): try: entry_text = checker_path.read_text() except UnicodeDecodeError as exc: report.errors.append(f"`{name}`: не удалось прочитать python checker `{checker_entry}`: {exc}") continue detected_protocol = checker_protocol(entry_text) if entry_text else "unknown" detected_type = checker_type(entry_text) if entry_text else "hackerdom" detected_places = checker_places(entry_text) if entry_text else 1 detected_checker_port = detect_checker_default_port(entry_text) if entry_text else None checker_port_env = detect_checker_port_env(entry_text) if entry_text else None expects_host_with_port = checker_expects_host_with_port(entry_text, checker_port_env) if entry_text else False protocol = str(service_override.get("protocol", detected_protocol)) task_type = str(service_override.get("checker_type", detected_type)) places = int(service_override.get("places", detected_places)) timeout = int(service_override.get("checker_timeout", 30)) host_port = discover_host_port( service_dir, profile, protocol, service_override, preferred_checker_port=detected_checker_port, ) host_transport = discover_host_transport( service_dir, profile, protocol, service_override, preferred_checker_port=detected_checker_port, ) requirements = load_requirements(checker_dir, checkers_root) build_enabled, build_profile, build_command, build_artifacts = detect_checker_build(checker_dir, service_override) spec = ServiceSpec( name=name, service_dir=service_dir, checker_dir=checker_dir, profile=profile, checker_entry=checker_entry, checker_type=task_type, places=places, checker_timeout=timeout, protocol=protocol, host_port=host_port, host_transport=host_transport, checker_port_env=checker_port_env, expects_host_with_port=expects_host_with_port, requirements=requirements, checker_build_enabled=build_enabled, checker_build_profile=build_profile, checker_build_command=build_command, checker_build_artifacts=build_artifacts, start_cmd=service_override.get("start_cmd"), stop_cmd=service_override.get("stop_cmd"), ) specs.append(spec) if not specs: report.errors.append("Не найдено ни одного валидного сервиса для запуска.") return specs, report def validate_preflight(specs: List[ServiceSpec], board_port: Optional[int] = None) -> ValidationReport: report = ValidationReport() require_command("docker", report) seen_ports: Dict[int, str] = {} for spec in specs: if spec.host_port is None: report.errors.append(f"`{spec.name}`: не удалось определить host-порт (укажи `port` в `forcad-local.json`).") else: if board_port is not None and spec.host_port == board_port: report.errors.append( f"`{spec.name}`: host-порт {spec.host_port} конфликтует с портом борды {board_port}." ) if spec.host_port in seen_ports: report.errors.append( f"Конфликт портов: `{spec.name}` и `{seen_ports[spec.host_port]}` используют {spec.host_port}." ) seen_ports[spec.host_port] = spec.name if spec.profile == "compose" and discover_compose_file(spec.service_dir) is None: report.errors.append( f"`{spec.name}`: профиль compose выбран, но нет compose-файла " f"({', '.join(COMPOSE_FILE_CANDIDATES)})." ) if spec.profile in {"deploy_script", "helm_chart"}: for dep in ("helm", "kubectl"): if shutil.which(dep) is None: report.warnings.append( f"`{spec.name}`: команда `{dep}` не найдена; сервис может не подняться." ) if (spec.service_dir / "k3d-config.yaml").exists() and shutil.which("k3d") is None: report.warnings.append(f"`{spec.name}`: найден `k3d-config.yaml`, но команда `k3d` отсутствует.") if spec.profile == "script": if detect_service_start_script(spec.service_dir) is None and not spec.start_cmd: report.errors.append( f"`{spec.name}`: profile=script, но не найден start script ({', '.join(SCRIPT_START_CANDIDATES)})." ) checker_entry_path = spec.checker_dir / spec.checker_entry if not checker_entry_path.exists(): report.errors.append(f"`{spec.name}`: checker entry `{spec.checker_entry}` отсутствует.") elif not is_python_checker_entry(spec.checker_entry) and not os.access(checker_entry_path, os.X_OK): report.warnings.append( f"`{spec.name}`: binary checker `{spec.checker_entry}` не executable в исходной папке; " "попробую выставить +x в runtime." ) if spec.checker_build_enabled: if not spec.checker_build_profile: report.errors.append(f"`{spec.name}`: включена сборка чекера, но не указан checker_build_profile.") elif spec.checker_build_profile not in BUILDER_PROFILES: report.errors.append( f"`{spec.name}`: неизвестный checker build profile `{spec.checker_build_profile}`." ) if not spec.checker_build_command: report.errors.append(f"`{spec.name}`: включена сборка чекера, но не задан checker_build_command.") if not spec.checker_build_artifacts: report.warnings.append( f"`{spec.name}`: не заданы checker build artifacts, кэш сборки для него отключён." ) return report def merge_reports(*reports: ValidationReport) -> ValidationReport: result = ValidationReport() for report in reports: result.errors.extend(report.errors) result.warnings.extend(report.warnings) return result def print_report(report: ValidationReport) -> None: for warning in report.warnings: warn(warning) for error_message in report.errors: err(error_message) def resolve_checkers_root(services_root: Path, checkers_root_arg: Optional[Path]) -> Path: if checkers_root_arg: return checkers_root_arg.resolve() sibling = services_root.parent / "checkers" if sibling.exists(): return sibling.resolve() inside = services_root / "checkers" if inside.exists(): return inside.resolve() raise RuntimeError( "Не удалось определить папку чекеров автоматически. Укажи `--checkers-root` явно." ) def safe_service_var(service_name: str) -> str: normalized = re.sub(r"[^A-Za-z0-9]+", "_", service_name).upper() return f"AD_TARGET_{normalized}" def ensure_clean_dir(path: Path) -> None: if path.exists(): shutil.rmtree(path) path.mkdir(parents=True, exist_ok=True) def _script_state_dir() -> Path: base = Path(tempfile.gettempdir()) / "forcad-local-script-state" base.mkdir(parents=True, exist_ok=True) return base def _script_process_key(spec: ServiceSpec) -> str: digest = hashlib.sha256(str(spec.service_dir.resolve()).encode()).hexdigest()[:12] return f"{spec.name}-{digest}" def script_pid_path(spec: ServiceSpec) -> Path: return _script_state_dir() / f"{_script_process_key(spec)}.pid" def script_log_path(spec: ServiceSpec) -> Path: return _script_state_dir() / f"{_script_process_key(spec)}.log" def _read_script_pid(spec: ServiceSpec) -> Optional[int]: pid_file = script_pid_path(spec) if not pid_file.exists(): return None try: return int(pid_file.read_text().strip()) except (TypeError, ValueError): return None def _process_alive(pid: int) -> bool: if pid <= 0: return False try: os.kill(pid, 0) except OSError: return False return True def _terminate_pid(pid: int, *, grace_seconds: float = 3.0) -> None: if not _process_alive(pid): return try: if os.name == "posix": os.killpg(pid, signal.SIGTERM) else: os.kill(pid, signal.SIGTERM) except ProcessLookupError: return except OSError: pass deadline = time.time() + grace_seconds while time.time() < deadline: if not _process_alive(pid): return time.sleep(0.2) try: if os.name == "posix": os.killpg(pid, signal.SIGKILL) else: os.kill(pid, signal.SIGKILL) except OSError: return def _kill_by_port(port: Optional[int]) -> None: if port is None or not is_port_open(port): return if shutil.which("lsof"): run_shell(f"lsof -ti tcp:{port} | xargs -r kill -TERM", check=False) time.sleep(0.5) if is_port_open(port): run_shell(f"lsof -ti tcp:{port} | xargs -r kill -KILL", check=False) return if shutil.which("fuser"): run_command(["fuser", "-k", f"{port}/tcp"], check=False) def _kill_script_signature(spec: ServiceSpec) -> None: script = detect_service_start_script(spec.service_dir) if not script or shutil.which("pgrep") is None: return quoted = shlex.quote(str(script)) run_shell(f"pgrep -f {quoted} | xargs -r kill -TERM", check=False) def resolve_builder_image(orchestrator_root: Path, profile: str) -> str: profile_config = BUILDER_PROFILES.get(profile) if not profile_config: raise RuntimeError(f"Unknown builder profile: {profile}") image = profile_config["image"] inspect = run_command(["docker", "image", "inspect", image], check=False) if inspect.returncode == 0: return image dockerfile = orchestrator_root / profile_config["dockerfile"] if not dockerfile.exists(): raise RuntimeError(f"Builder dockerfile not found for profile `{profile}`: {dockerfile}") info(f"Building checker builder image `{image}` from `{dockerfile}`.") run_command( [ "docker", "build", "-f", str(dockerfile), "-t", image, str(orchestrator_root), ] ) return image def hash_checker_inputs(checker_dir: Path, build_profile: str, build_command: str) -> str: digest = hashlib.sha256() digest.update(build_profile.encode()) digest.update(b"\0") digest.update(build_command.encode()) digest.update(b"\0") for item in sorted(checker_dir.rglob("*")): if not item.is_file(): continue rel = item.relative_to(checker_dir).as_posix() digest.update(rel.encode()) digest.update(b"\0") digest.update(item.read_bytes()) digest.update(b"\0") return digest.hexdigest() def restore_build_cache( service: ServiceSpec, checker_runtime_dir: Path, cache_root: Path, cache_key: str, ) -> bool: if not service.checker_build_artifacts: return False entry = cache_root / service.name / cache_key if not entry.exists(): return False for artifact in service.checker_build_artifacts: src = entry / artifact dst = checker_runtime_dir / artifact if not src.exists(): return False if src.is_dir(): if dst.exists(): shutil.rmtree(dst) shutil.copytree(src, dst) else: dst.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src, dst) info(f"Checker build cache hit for `{service.name}`.") return True def store_build_cache( service: ServiceSpec, checker_runtime_dir: Path, cache_root: Path, cache_key: str, ) -> None: if not service.checker_build_artifacts: return entry = cache_root / service.name / cache_key if entry.exists(): shutil.rmtree(entry) entry.mkdir(parents=True, exist_ok=True) for artifact in service.checker_build_artifacts: src = checker_runtime_dir / artifact if not src.exists(): continue dst = entry / artifact if src.is_dir(): shutil.copytree(src, dst) else: dst.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src, dst) def create_wrapper(service: ServiceSpec, destination: Path) -> None: env_var = safe_service_var(service.name) default_target = "host.docker.internal" default_port = str(service.host_port) if service.host_port is not None else "" checker_port_env = service.checker_port_env or "" needs_host_port = "True" if service.expects_host_with_port else "False" wrapper = f"""#!/usr/bin/env python3 import os import subprocess import sys from pathlib import Path LOCAL_ALIASES = {sorted(LOCAL_HOST_ALIASES)!r} DEFAULT_TARGET = os.environ.get("{env_var}", "{default_target}") DEFAULT_PORT = "{default_port}" CHECKER_PORT_ENV = "{checker_port_env}" NEEDS_HOST_PORT = {needs_host_port} def patch_hostname(argv): if len(argv) < 3: return argv action = argv[1].lower() if action not in {{"check", "put", "get", "test"}}: return argv host = argv[2] if host in LOCAL_ALIASES: host = DEFAULT_TARGET if NEEDS_HOST_PORT and DEFAULT_PORT and ":" not in host: host = f"{{host}}:{{DEFAULT_PORT}}" argv[2] = host return argv def main(): if CHECKER_PORT_ENV and DEFAULT_PORT: os.environ.setdefault(CHECKER_PORT_ENV, DEFAULT_PORT) checker_path = Path(__file__).with_name("{service.checker_entry.as_posix()}") argv = patch_hostname(sys.argv[:]) checker_cmd = [sys.executable, str(checker_path), *argv[1:]] if checker_path.suffix.lower() != ".py": checker_cmd = [str(checker_path), *argv[1:]] completed = subprocess.run(checker_cmd, cwd=Path(__file__).parent) raise SystemExit(completed.returncode) if __name__ == "__main__": main() """ destination.write_text(wrapper) destination.chmod(0o755) def generate_runtime( runtime_dir: Path, specs: List[ServiceSpec], ) -> dict: ensure_clean_dir(runtime_dir) checkers_dir = runtime_dir / "checkers" checkers_dir.mkdir(parents=True, exist_ok=True) requirements = {"gornilo", "checklib==0.7.0"} for spec in specs: target_dir = checkers_dir / spec.name shutil.copytree(spec.checker_dir, target_dir, dirs_exist_ok=True) checker_target = target_dir / spec.checker_entry if checker_target.exists() and not is_python_checker_entry(spec.checker_entry): mode = checker_target.stat().st_mode checker_target.chmod(mode | 0o111) create_wrapper(spec, target_dir / "forcad_wrapper.py") for requirement in spec.requirements: requirements.add(requirement) (checkers_dir / "requirements.txt").write_text("\n".join(sorted(requirements)) + "\n") return { "runtime_dir": str(runtime_dir), "checkers_dir": str(checkers_dir), } def build_checker_artifacts( runtime_dir: Path, specs: List[ServiceSpec], cache_dir: Path, failure_policy: str = "strict", ) -> Tuple[List[ServiceSpec], Dict[str, str]]: orchestrator_root = Path(__file__).resolve().parent cache_root = cache_dir / "checker-builds" cache_root.mkdir(parents=True, exist_ok=True) builder_images: Dict[str, str] = {} ready_specs: List[ServiceSpec] = [] failed: Dict[str, str] = {} for spec in specs: try: if not spec.checker_build_enabled: ready_specs.append(spec) continue if not spec.checker_build_profile or not spec.checker_build_command: raise RuntimeError(f"Invalid checker build config for `{spec.name}`.") checker_runtime_dir = runtime_dir / "checkers" / spec.name cache_key = hash_checker_inputs( checker_dir=checker_runtime_dir, build_profile=spec.checker_build_profile, build_command=spec.checker_build_command, ) if restore_build_cache(spec, checker_runtime_dir, cache_root, cache_key): ready_specs.append(spec) continue if spec.checker_build_profile not in builder_images: builder_images[spec.checker_build_profile] = resolve_builder_image( orchestrator_root=orchestrator_root, profile=spec.checker_build_profile, ) image = builder_images[spec.checker_build_profile] info(f"Containerized checker build for `{spec.name}` ({spec.checker_build_profile}).") run_command( [ "docker", "run", "--rm", "--add-host", "host.docker.internal:host-gateway", "-v", f"{checker_runtime_dir}:/work", "-w", "/work", image, "bash", "-lc", spec.checker_build_command, ] ) store_build_cache(spec, checker_runtime_dir, cache_root, cache_key) ready_specs.append(spec) except Exception as exc: # noqa: BLE001 message = str(exc) failed[spec.name] = message if failure_policy == "strict": raise warn(f"Skipping `{spec.name}` due checker build failure: {message}") return ready_specs, failed def is_port_open(port: int) -> bool: with socket.socket() as sock: sock.settimeout(0.5) return sock.connect_ex(("127.0.0.1", int(port))) == 0 def is_udp_port_bound(port: int) -> bool: with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: try: sock.bind(("127.0.0.1", int(port))) return False except OSError: return True def ensure_service_port_available(spec: ServiceSpec) -> None: if spec.host_port is None: return if not is_port_open(spec.host_port): return warn(f"Port {spec.host_port} is busy before deploying `{spec.name}`, trying cleanup.") stop_service(spec, include_legacy_compose=True) time.sleep(1) if is_port_open(spec.host_port): raise RuntimeError( f"Port {spec.host_port} is still busy before deploying `{spec.name}`. " "Run cleanup and retry." ) def prewarm_k3d_images( specs: List[ServiceSpec], strict: bool, image_pull_retries: int, image_pull_retry_delay: int, ) -> None: if not any(is_k3d_service(spec) for spec in specs): return for image in K3D_PREWARM_IMAGES: try: run_command_with_retries( ["docker", "pull", image], retries=image_pull_retries, retry_delay=image_pull_retry_delay, purpose=f"prewarm pull `{image}`", ) except Exception as exc: # noqa: BLE001 message = f"Failed to prewarm image `{image}`: {exc}" if strict: raise RuntimeError(message) from exc warn(message) def auto_cleanup_services(specs: List[ServiceSpec]) -> None: info("Auto-cleanup previous service runs.") for spec in reversed(specs): stop_service(spec, include_legacy_compose=True) def rollback_services(specs: List[ServiceSpec]) -> None: if not specs: return warn("Rolling back already deployed services.") for spec in reversed(specs): stop_service(spec, include_legacy_compose=True) def deploy_service_with_retry( spec: ServiceSpec, k3d_retries: int, retry_delay: int, image_pull_retries: int, image_pull_retry_delay: int, ) -> None: attempts = 1 + (k3d_retries if is_k3d_service(spec) else 0) for attempt in range(1, attempts + 1): try: ensure_service_port_available(spec) deploy_service( spec, image_pull_retries=image_pull_retries, image_pull_retry_delay=image_pull_retry_delay, ) return except Exception as exc: # noqa: BLE001 if attempt >= attempts: raise warn(f"Deploy `{spec.name}` attempt {attempt}/{attempts} failed: {exc}") stop_service(spec, include_legacy_compose=True) time.sleep(retry_delay) def compose_project_names(spec: ServiceSpec) -> List[str]: names: List[str] = [f"forcadsvc_{spec.name}"] legacy = spec.service_dir.name if legacy not in names: names.append(legacy) return names def is_k3d_service(spec: ServiceSpec) -> bool: if (spec.service_dir / "k3d-config.yaml").exists(): return True deploy_script = spec.service_dir / "deploy.sh" if deploy_script.exists(): return "k3d cluster create" in deploy_script.read_text() return False def service_priority(spec: ServiceSpec) -> Tuple[int, str]: # Start k3d-heavy services first while Docker daemon is still idle. if is_k3d_service(spec): return (0, spec.name) if spec.profile in {"compose", "script"}: return (1, spec.name) return (2, spec.name) def order_services_for_deploy(specs: List[ServiceSpec]) -> List[ServiceSpec]: return sorted(specs, key=service_priority) def _start_script_service(spec: ServiceSpec) -> None: start_script = detect_service_start_script(spec.service_dir) if start_script is None: raise RuntimeError( f"`{spec.name}`: не найден script entry для профиля script ({', '.join(SCRIPT_START_CANDIDATES)})." ) pid_file = script_pid_path(spec) existing_pid = _read_script_pid(spec) if existing_pid and _process_alive(existing_pid): warn(f"`{spec.name}` script already running (pid={existing_pid}), stopping old process first.") _terminate_pid(existing_pid) pid_file.parent.mkdir(parents=True, exist_ok=True) log_file = script_log_path(spec) with log_file.open("ab") as output: process = subprocess.Popen( ["bash", str(start_script)], cwd=spec.service_dir, stdout=output, stderr=output, start_new_session=True, ) pid_file.write_text(str(process.pid)) info(f"Started script service `{spec.name}` via `{start_script.name}` (pid={process.pid}).") def deploy_service( spec: ServiceSpec, *, image_pull_retries: int = 0, image_pull_retry_delay: int = 5, ) -> None: if spec.start_cmd: run_shell(spec.start_cmd, cwd=spec.service_dir) return if spec.profile == "compose": compose_path = discover_compose_file(spec.service_dir) if compose_path is None: raise RuntimeError( f"`{spec.name}`: не найден compose-файл ({', '.join(COMPOSE_FILE_CANDIDATES)})." ) project = compose_project_names(spec)[0] # Pull externally referenced images with retries before "up". run_command_with_retries( [ "docker", "compose", "-f", str(compose_path), "-p", project, "pull", ], retries=image_pull_retries, retry_delay=image_pull_retry_delay, purpose=f"image pull for `{spec.name}`", ) run_command( [ "docker", "compose", "-f", str(compose_path), "-p", project, "up", "-d", "--build", "--remove-orphans", ] ) return if spec.profile == "script": _start_script_service(spec) return deploy_script = spec.service_dir / "deploy.sh" if deploy_script.exists(): run_command(["bash", str(deploy_script)], cwd=spec.service_dir) return redeploy_script = spec.service_dir / "redeploy.sh" if redeploy_script.exists(): run_command(["bash", str(redeploy_script)], cwd=spec.service_dir) return build_script = spec.service_dir / "build-docker.sh" chart_dir = spec.service_dir / "deploy" / "chart" if build_script.exists(): run_command(["bash", str(build_script)], cwd=spec.service_dir) if chart_dir.exists(): run_command( [ "helm", "upgrade", "--install", spec.name, str(chart_dir), "--create-namespace", "--namespace", spec.name, "--wait", "--timeout", "10m", ], cwd=spec.service_dir, ) return raise RuntimeError(f"Не знаю, как поднять сервис `{spec.name}` (profile={spec.profile}).") def stop_service(spec: ServiceSpec, include_legacy_compose: bool = True) -> None: if spec.stop_cmd: run_shell(spec.stop_cmd, cwd=spec.service_dir, check=False) if spec.profile != "script": return if spec.profile == "script": stop_script = detect_service_stop_script(spec.service_dir) if stop_script is not None: run_command(["bash", str(stop_script)], cwd=spec.service_dir, check=False) pid = _read_script_pid(spec) if pid is not None: _terminate_pid(pid) script_pid_path(spec).unlink(missing_ok=True) _kill_by_port(spec.host_port) if spec.host_port is None or is_port_open(spec.host_port): _kill_script_signature(spec) _kill_by_port(spec.host_port) return if spec.profile == "compose": compose_path = discover_compose_file(spec.service_dir) if compose_path is None: warn( f"`{spec.name}`: compose profile without compose file " f"({', '.join(COMPOSE_FILE_CANDIDATES)}), skip stop." ) return projects = compose_project_names(spec) if not include_legacy_compose: projects = projects[:1] for project in projects: run_command( ["docker", "compose", "-f", str(compose_path), "-p", project, "down", "-v", "--remove-orphans"], check=False, ) return if (spec.service_dir / "k3d-config.yaml").exists() and shutil.which("k3d"): config_text = (spec.service_dir / "k3d-config.yaml").read_text() match = re.search(r"\bname:\s*([A-Za-z0-9_.-]+)", config_text) cluster_name = match.group(1) if match else f"{spec.name}-cluster" run_command(["k3d", "cluster", "delete", cluster_name], check=False) if shutil.which("helm"): run_command(["helm", "uninstall", spec.name, "-n", spec.name], check=False) def wait_for_service(spec: ServiceSpec, timeout_seconds: int = 180) -> None: if spec.host_port is None: raise RuntimeError(f"У `{spec.name}` не определён host_port.") info( f"Ожидаю доступность `{spec.name}` на порту {spec.host_port} " f"(transport={spec.host_transport}) ..." ) deadline = time.time() + timeout_seconds if spec.host_transport.lower() == "udp": while time.time() < deadline: if is_udp_port_bound(spec.host_port): info(f"`{spec.name}` доступен на UDP 127.0.0.1:{spec.host_port}") return time.sleep(2) raise RuntimeError( f"`{spec.name}` не стал доступен на UDP-порту {spec.host_port} за {timeout_seconds} сек." ) while time.time() < deadline: with socket.socket() as sock: sock.settimeout(2.0) try: sock.connect(("127.0.0.1", int(spec.host_port))) info(f"`{spec.name}` доступен на 127.0.0.1:{spec.host_port}") return except OSError: time.sleep(2) raise RuntimeError(f"`{spec.name}` не стал доступен на порту {spec.host_port} за {timeout_seconds} сек.") def _smoke_check_timeout(spec: ServiceSpec) -> float: # A bit of slack on top of the checker's own timeout for process/exec overhead. return float(max(5, spec.checker_timeout) + 10) def _build_light_checkers_image(checkers_dir: Path) -> str: """Build (or reuse) an image with all checker requirements pre-installed. Building once avoids re-running `pip install` inside every parallel `docker run`, which used to happen serially inside a single shared container and made per-service timeouts impossible to enforce. """ requirements_path = checkers_dir / "requirements.txt" requirements_bytes = requirements_path.read_bytes() if requirements_path.exists() else b"" tag = hashlib.sha256(requirements_bytes).hexdigest()[:16] image = f"forcad-local-light-checkers:{tag}" inspect = run_command(["docker", "image", "inspect", image], check=False) if inspect.returncode == 0: return image build_ctx = Path(tempfile.mkdtemp(prefix="forcad-local-light-")) try: (build_ctx / "requirements.txt").write_bytes(requirements_bytes) (build_ctx / "Dockerfile").write_text( "FROM python:3.11-slim\n" "COPY requirements.txt /tmp/requirements.txt\n" "RUN python3 -m pip install --no-cache-dir -r /tmp/requirements.txt\n" ) info(f"Building light checker image `{image}` (checkers requirements.txt) ...") run_command(["docker", "build", "-t", image, str(build_ctx)]) finally: shutil.rmtree(build_ctx, ignore_errors=True) return image def _run_one_light_smoke_check(image: str, checkers_dir: Path, spec: ServiceSpec) -> Tuple[str, bool, str]: command = [ "docker", "run", "--rm", "--add-host", "host.docker.internal:host-gateway", "-v", f"{checkers_dir}:/checkers:ro", image, "python3", f"/checkers/{spec.name}/forcad_wrapper.py", "check", "host.docker.internal", ] try: result = run_command(command, check=False, timeout=_smoke_check_timeout(spec)) except CommandTimeoutError as exc: return spec.name, False, str(exc) if result.returncode == 101: return spec.name, True, "OK" return spec.name, False, f"exit code {result.returncode}" def run_light_smoke_checks( runtime_dir: Path, specs: List[ServiceSpec], max_workers: int = 8 ) -> Dict[str, Tuple[bool, str]]: checkers_dir = runtime_dir / "checkers" image = _build_light_checkers_image(checkers_dir) info(f"Running {len(specs)} light smoke-check(s) in parallel (max_workers={max_workers}) ...") results: Dict[str, Tuple[bool, str]] = {} with ThreadPoolExecutor(max_workers=max(1, max_workers)) as pool: futures = { pool.submit(_run_one_light_smoke_check, image, checkers_dir, spec): spec for spec in specs } for future in as_completed(futures): name, ok, detail = future.result() results[name] = (ok, detail) if ok: info(f"[SMOKE] `{name}`: OK") else: warn(f"[SMOKE] `{name}` failed: {detail}") failed = [name for name, (ok, _) in results.items() if not ok] if not failed: info("Smoke-checks completed successfully.") else: warn(f"Smoke-checks failed for: {', '.join(sorted(failed))}") return results def save_state(runtime_dir: Path, payload: dict) -> None: state_path = runtime_dir / "state.json" state_path.write_text(json.dumps(payload, indent=2)) def load_state(runtime_dir: Path) -> Optional[dict]: state_path = runtime_dir / "state.json" if not state_path.exists(): return None return json.loads(state_path.read_text()) def load_context_from_args(args: argparse.Namespace, allow_state: bool = False) -> Tuple[Path, Path]: runtime_dir = args.runtime_dir.resolve() services_root = args.services_root.resolve() if args.services_root else None checkers_root = args.checkers_root.resolve() if args.checkers_root else None if services_root is None and allow_state: state = load_state(runtime_dir) if state: services_root = Path(state["services_root"]) checkers_root = Path(state["checkers_root"]) if services_root is None: raise RuntimeError("Нужно указать `--services-root` или иметь сохранённый state от `up`.") if checkers_root is None: checkers_root = resolve_checkers_root(services_root, None) return services_root, checkers_root def cmd_validate(args: argparse.Namespace) -> int: services_root, checkers_root = load_context_from_args(args, allow_state=False) selected = set(args.service) if args.service else None specs, discovery_report = discover_services(services_root, checkers_root, selected) validation_report = validate_preflight(specs) report = merge_reports(discovery_report, validation_report) print_report(report) if not report.ok(): return 2 info("Preflight-проверка пройдена.") for spec in specs: info( f"service={spec.name} profile={spec.profile} checker={spec.checker_entry} " f"type={spec.checker_type} places={spec.places} port={spec.host_port}" ) return 0 def cmd_up(args: argparse.Namespace) -> int: services_root, checkers_root = load_context_from_args(args, allow_state=False) selected = set(args.service) if args.service else None specs, discovery_report = discover_services(services_root, checkers_root, selected) validation_report = validate_preflight(specs) report = merge_reports(discovery_report, validation_report) print_report(report) if not report.ok(): return 2 runtime_dir = args.runtime_dir.resolve() runtime_meta = generate_runtime( runtime_dir=runtime_dir, specs=specs, ) failure_policy = args.failure_policy.lower() strict = failure_policy == "strict" ordered_specs = order_services_for_deploy(specs) if args.auto_cleanup: auto_cleanup_services(ordered_specs) if args.prewarm_k3d: prewarm_k3d_images( ordered_specs, strict=strict, image_pull_retries=args.image_pull_retries, image_pull_retry_delay=args.image_pull_retry_delay, ) ready_specs, build_failed = build_checker_artifacts( runtime_dir, ordered_specs, cache_dir=args.cache_dir.resolve(), failure_policy=failure_policy, ) if build_failed and strict: raise RuntimeError(f"Checker build failed: {build_failed}") if build_failed and not strict: warn(f"Checker build failures (best-effort): {build_failed}") if not ready_specs: raise RuntimeError("No services left after checker build stage.") deployed_specs: List[ServiceSpec] = [] deploy_failed: Dict[str, str] = {} for spec in ready_specs: try: deploy_service_with_retry( spec, k3d_retries=args.k3d_retries, retry_delay=args.k3d_retry_delay, image_pull_retries=args.image_pull_retries, image_pull_retry_delay=args.image_pull_retry_delay, ) wait_for_service(spec, timeout_seconds=args.service_wait_timeout) deployed_specs.append(spec) except Exception as exc: # noqa: BLE001 deploy_failed[spec.name] = str(exc) if strict: if args.rollback_on_failure: rollback_services(deployed_specs + [spec]) raise warn(f"Skipping `{spec.name}` after deploy failure: {exc}") if not deployed_specs: raise RuntimeError("No services were deployed successfully.") if deploy_failed and not strict: warn(f"Deploy failures (best-effort): {deploy_failed}") if args.smoke_start_delay > 0: info(f"Waiting {args.smoke_start_delay}s before smoke-check ...") time.sleep(args.smoke_start_delay) info("Running service-only mode: ForcAD stack is disabled.") run_light_smoke_checks(runtime_dir, deployed_specs, max_workers=args.smoke_workers) state = { "services_root": str(services_root), "checkers_root": str(checkers_root), "runtime": runtime_meta, "failure_policy": failure_policy, "deploy_failed": deploy_failed, "build_failed": build_failed, "services": [ { "name": spec.name, "profile": spec.profile, "service_dir": str(spec.service_dir), "checker_dir": str(spec.checker_dir), "checker_entry": spec.checker_entry.as_posix(), "checker_type": spec.checker_type, "places": spec.places, "checker_timeout": spec.checker_timeout, "protocol": spec.protocol, "requirements": list(spec.requirements), "checker_build_enabled": spec.checker_build_enabled, "checker_build_profile": spec.checker_build_profile, "checker_build_command": spec.checker_build_command, "checker_build_artifacts": spec.checker_build_artifacts, "host_port": spec.host_port, "host_transport": spec.host_transport, "checker_port_env": spec.checker_port_env, "expects_host_with_port": spec.expects_host_with_port, "start_cmd": spec.start_cmd, "stop_cmd": spec.stop_cmd, } for spec in deployed_specs ], } save_state(runtime_dir, state) info("Service-only mode complete: services are up and checker liveness checks were executed.") info("Готово.") return 0 def cmd_down(args: argparse.Namespace) -> int: runtime_dir = args.runtime_dir.resolve() state = load_state(runtime_dir) if state is None and not args.services_root: err("Не найден state.json. Укажи `--services-root` и `--checkers-root`.") return 2 if state is not None: specs = [ ServiceSpec( name=s["name"], profile=s["profile"], service_dir=Path(s["service_dir"]), checker_dir=Path(s["checker_dir"]), checker_entry=Path(s.get("checker_entry", "checker.py")), checker_type=s.get("checker_type", "hackerdom"), places=int(s.get("places", 1)), checker_timeout=int(s.get("checker_timeout", 30)), protocol=s.get("protocol", "unknown"), requirements=[str(x) for x in s.get("requirements", [])], checker_build_enabled=bool(s.get("checker_build_enabled", False)), checker_build_profile=s.get("checker_build_profile"), checker_build_command=s.get("checker_build_command"), checker_build_artifacts=[str(x) for x in s.get("checker_build_artifacts", [])], host_port=s.get("host_port"), host_transport=str(s.get("host_transport", "tcp")).lower(), checker_port_env=s.get("checker_port_env"), expects_host_with_port=bool(s.get("expects_host_with_port", False)), start_cmd=s.get("start_cmd"), stop_cmd=s.get("stop_cmd"), ) for s in state["services"] ] else: services_root, checkers_root = load_context_from_args(args, allow_state=False) discovered, report = discover_services(services_root, checkers_root, set(args.service) if args.service else None) print_report(report) if not discovered: return 2 specs = discovered for spec in specs: stop_service(spec) info("Остановка завершена.") return 0 def cmd_status(args: argparse.Namespace) -> int: runtime_dir = args.runtime_dir.resolve() state = load_state(runtime_dir) if state is None: err("Не найден state.json. Сначала выполни `up`.") return 2 info("Service-only mode state: ForcAD stack is disabled.") for service in state["services"]: port = service.get("host_port") transport = str(service.get("host_transport", "tcp")).lower() if not port: warn(f"{service['name']}: порт неизвестен.") continue if transport == "udp": if is_udp_port_bound(int(port)): info(f"{service['name']}: UDP 127.0.0.1:{port} открыт") else: warn(f"{service['name']}: UDP 127.0.0.1:{port} недоступен") continue with socket.socket() as sock: sock.settimeout(1.0) try: sock.connect(("127.0.0.1", int(port))) info(f"{service['name']}: 127.0.0.1:{port} открыт") except OSError: warn(f"{service['name']}: 127.0.0.1:{port} недоступен") return 0 def cmd_check(args: argparse.Namespace) -> int: runtime_dir = args.runtime_dir.resolve() state = load_state(runtime_dir) services_root = args.services_root.resolve() if args.services_root else None checkers_root = args.checkers_root.resolve() if args.checkers_root else None if state is not None: if services_root is None: services_root = Path(state["services_root"]) if checkers_root is None: checkers_root = Path(state["checkers_root"]) if services_root is None: err("Нужно указать `--services-root` или иметь сохранённый state от `up`.") return 2 if checkers_root is None: checkers_root = resolve_checkers_root(services_root, None) selected = set(args.service) if args.service else None specs, discovery_report = discover_services(services_root, checkers_root, selected) validation_report = validate_preflight(specs) report = merge_reports(discovery_report, validation_report) print_report(report) if not report.ok(): return 2 runtime_meta = generate_runtime( runtime_dir=runtime_dir, specs=specs, ) failure_policy = args.failure_policy.lower() ready_specs, build_failed = build_checker_artifacts( runtime_dir, specs, cache_dir=args.cache_dir.resolve(), failure_policy=failure_policy, ) if build_failed and failure_policy == "strict": raise RuntimeError(f"Checker build failed: {build_failed}") if not ready_specs: raise RuntimeError("No checkers ready for smoke-check.") if args.smoke_start_delay > 0: info(f"Waiting {args.smoke_start_delay}s before smoke-check ...") time.sleep(args.smoke_start_delay) run_light_smoke_checks(runtime_dir, ready_specs, max_workers=args.smoke_workers) state_payload = { "services_root": str(services_root), "checkers_root": str(checkers_root), "runtime": runtime_meta, "failure_policy": failure_policy, "deploy_failed": {}, "build_failed": build_failed, "services": [ { "name": spec.name, "profile": spec.profile, "service_dir": str(spec.service_dir), "checker_dir": str(spec.checker_dir), "checker_entry": spec.checker_entry.as_posix(), "checker_type": spec.checker_type, "places": spec.places, "checker_timeout": spec.checker_timeout, "protocol": spec.protocol, "requirements": list(spec.requirements), "checker_build_enabled": spec.checker_build_enabled, "checker_build_profile": spec.checker_build_profile, "checker_build_command": spec.checker_build_command, "checker_build_artifacts": spec.checker_build_artifacts, "host_port": spec.host_port, "host_transport": spec.host_transport, "checker_port_env": spec.checker_port_env, "expects_host_with_port": spec.expects_host_with_port, "start_cmd": spec.start_cmd, "stop_cmd": spec.stop_cmd, } for spec in ready_specs ], } save_state(runtime_dir, state_payload) info("Проверка чекеров завершена.") return 0 def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Локальный оркестратор сервисов и checker smoke-check без запуска полного ForcAD." ) subparsers = parser.add_subparsers(dest="command", required=True) def add_common_flags(subparser: argparse.ArgumentParser, services_required: bool) -> None: subparser.add_argument("--services-root", type=Path, required=services_required) subparser.add_argument("--checkers-root", type=Path) subparser.add_argument("--runtime-dir", type=Path, default=Path(__file__).resolve().parent / ".runtime") subparser.add_argument("--service", action="append", help="Имя сервиса (можно повторять). Если не задано, берутся все.") validate = subparsers.add_parser("validate", help="Проверка структуры сервисов/чекеров без запуска.") add_common_flags(validate, services_required=True) validate.set_defaults(handler=cmd_validate) up = subparsers.add_parser("up", help="Поднять сервисы и выполнить checker smoke-check.") add_common_flags(up, services_required=True) up.add_argument("--cache-dir", type=Path, default=Path(__file__).resolve().parent / ".cache") up.add_argument("--failure-policy", choices=["strict", "best-effort"], default="strict") up.add_argument("--rollback-on-failure", action=argparse.BooleanOptionalAction, default=True) up.add_argument("--auto-cleanup", action=argparse.BooleanOptionalAction, default=True) up.add_argument("--prewarm-k3d", action=argparse.BooleanOptionalAction, default=True) up.add_argument("--k3d-retries", type=int, default=1) up.add_argument("--k3d-retry-delay", type=int, default=5) up.add_argument( "--image-pull-retries", type=int, default=2, help="Additional retries for docker image pulls (compose/prewarm).", ) up.add_argument( "--image-pull-retry-delay", type=int, default=10, help="Delay (seconds) between docker pull retries.", ) up.add_argument("--service-wait-timeout", type=int, default=180) up.add_argument( "--smoke-workers", type=int, default=8, help="Максимум параллельных чекеров при smoke-check.", ) up.add_argument( "--smoke-start-delay", type=int, default=30, help="Задержка (сек) перед запуском smoke-check после деплоя.", ) up.set_defaults(handler=cmd_up) down = subparsers.add_parser("down", help="Остановить игровые сервисы.") add_common_flags(down, services_required=False) down.set_defaults(handler=cmd_down) status = subparsers.add_parser("status", help="Показать статус запущенной инфраструктуры.") status.add_argument("--runtime-dir", type=Path, default=Path(__file__).resolve().parent / ".runtime") status.set_defaults(handler=cmd_status) check = subparsers.add_parser( "check", help="Прогнать checker smoke-check по уже поднятым сервисам (использует state или явные roots).", ) add_common_flags(check, services_required=False) check.add_argument("--cache-dir", type=Path, default=Path(__file__).resolve().parent / ".cache") check.add_argument("--failure-policy", choices=["strict", "best-effort"], default="strict") check.add_argument( "--smoke-workers", type=int, default=8, help="Максимум параллельных чекеров при smoke-check.", ) check.add_argument( "--smoke-start-delay", type=int, default=30, help="Задержка (сек) перед запуском smoke-check.", ) check.set_defaults(handler=cmd_check) return parser def main() -> int: parser = build_parser() args = parser.parse_args() try: return int(args.handler(args)) except KeyboardInterrupt: err("Прервано пользователем.") return 130 except Exception as exc: # noqa: BLE001 err(str(exc)) return 1 if __name__ == "__main__": sys.exit(main())