1925 lines
68 KiB
Python
1925 lines
68 KiB
Python
#!/usr/bin/env python3
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import re
|
||
import shlex
|
||
import signal
|
||
import shutil
|
||
import socket
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Dict, Iterable, List, Optional, Sequence, Tuple
|
||
|
||
|
||
LOCAL_HOST_ALIASES = {"127.0.0.1", "localhost", "host.docker.internal", "local", "test-service"}
|
||
|
||
BUILDER_PROFILES = {
|
||
"cpp-make": {
|
||
"dockerfile": "builders/cpp-make.Dockerfile",
|
||
"image": "forcad-local-builder-cpp-make:1",
|
||
},
|
||
}
|
||
|
||
K3D_PREWARM_IMAGES = [
|
||
"ghcr.io/k3d-io/k3d-tools:5.9.0",
|
||
"rancher/k3s:v1.31.5-k3s1",
|
||
]
|
||
|
||
SCRIPT_START_CANDIDATES = ("server.sh", "run.sh", "start.sh")
|
||
SCRIPT_STOP_CANDIDATES = ("stop.sh", "shutdown.sh")
|
||
COMPOSE_FILE_CANDIDATES = ("docker-compose.yml", "docker-compose.yaml")
|
||
|
||
|
||
@dataclass
|
||
class ServiceSpec:
|
||
name: str
|
||
service_dir: Path
|
||
checker_dir: Path
|
||
profile: str
|
||
checker_entry: Path
|
||
checker_type: str
|
||
places: int
|
||
checker_timeout: int
|
||
protocol: str
|
||
host_port: Optional[int]
|
||
host_transport: str = "tcp"
|
||
checker_port_env: Optional[str] = None
|
||
expects_host_with_port: bool = False
|
||
requirements: List[str] = field(default_factory=list)
|
||
checker_build_enabled: bool = False
|
||
checker_build_profile: Optional[str] = None
|
||
checker_build_command: Optional[str] = None
|
||
checker_build_artifacts: List[str] = field(default_factory=list)
|
||
start_cmd: Optional[str] = None
|
||
stop_cmd: Optional[str] = None
|
||
|
||
|
||
@dataclass
|
||
class ValidationReport:
|
||
errors: List[str] = field(default_factory=list)
|
||
warnings: List[str] = field(default_factory=list)
|
||
|
||
def ok(self) -> bool:
|
||
return not self.errors
|
||
|
||
|
||
def info(message: str) -> None:
|
||
print(f"[INFO] {message}")
|
||
|
||
|
||
def warn(message: str) -> None:
|
||
print(f"[WARN] {message}")
|
||
|
||
|
||
def err(message: str) -> None:
|
||
print(f"[ERROR] {message}", file=sys.stderr)
|
||
|
||
|
||
class CommandTimeoutError(RuntimeError):
|
||
"""Raised when a subprocess exceeds its allotted timeout."""
|
||
|
||
def __init__(self, printable: str, timeout: float):
|
||
super().__init__(f"Command timed out after {timeout}s: {printable}")
|
||
self.printable = printable
|
||
self.timeout = timeout
|
||
|
||
|
||
def run_command(
|
||
command: Sequence[str],
|
||
cwd: Optional[Path] = None,
|
||
check: bool = True,
|
||
timeout: Optional[float] = None,
|
||
) -> subprocess.CompletedProcess:
|
||
printable = " ".join(command)
|
||
location = f" (cwd={cwd})" if cwd else ""
|
||
info(f"Running: {printable}{location}")
|
||
try:
|
||
result = subprocess.run(command, cwd=cwd, text=True, timeout=timeout)
|
||
except subprocess.TimeoutExpired as exc:
|
||
# Explicit, catchable failure instead of hanging the whole `up` forever.
|
||
raise CommandTimeoutError(printable, timeout) from exc
|
||
if check and result.returncode != 0:
|
||
raise RuntimeError(f"Command failed with exit code {result.returncode}: {printable}")
|
||
return result
|
||
|
||
|
||
def run_shell(
|
||
command: str,
|
||
cwd: Optional[Path] = None,
|
||
check: bool = True,
|
||
timeout: Optional[float] = None,
|
||
) -> subprocess.CompletedProcess:
|
||
location = f" (cwd={cwd})" if cwd else ""
|
||
info(f"Running shell: {command}{location}")
|
||
try:
|
||
result = subprocess.run(["bash", "-lc", command], cwd=cwd, text=True, timeout=timeout)
|
||
except subprocess.TimeoutExpired as exc:
|
||
raise CommandTimeoutError(command, timeout) from exc
|
||
if check and result.returncode != 0:
|
||
raise RuntimeError(f"Command failed with exit code {result.returncode}: {command}")
|
||
return result
|
||
|
||
|
||
def run_command_with_retries(
|
||
command: Sequence[str],
|
||
*,
|
||
cwd: Optional[Path] = None,
|
||
retries: int = 0,
|
||
retry_delay: int = 5,
|
||
purpose: str = "command",
|
||
) -> subprocess.CompletedProcess:
|
||
attempts = 1 + max(0, retries)
|
||
last_exception: Optional[Exception] = None
|
||
for attempt in range(1, attempts + 1):
|
||
try:
|
||
return run_command(command, cwd=cwd, check=True)
|
||
except Exception as exc: # noqa: BLE001
|
||
last_exception = exc
|
||
if attempt >= attempts:
|
||
raise
|
||
warn(
|
||
f"{purpose} failed ({attempt}/{attempts}): {exc}. "
|
||
f"Retrying in {retry_delay}s..."
|
||
)
|
||
time.sleep(retry_delay)
|
||
if last_exception:
|
||
raise last_exception
|
||
raise RuntimeError(f"{purpose} failed for unknown reason")
|
||
|
||
|
||
def require_command(command: str, report: ValidationReport) -> None:
|
||
if shutil.which(command) is None:
|
||
report.errors.append(f"Не найдена системная команда `{command}`.")
|
||
|
||
|
||
def parse_manifest(services_root: Path) -> Dict[str, dict]:
|
||
manifest_path = services_root / "forcad-local.json"
|
||
if not manifest_path.exists():
|
||
return {}
|
||
try:
|
||
data = json.loads(manifest_path.read_text())
|
||
except json.JSONDecodeError as exc:
|
||
raise RuntimeError(f"Некорректный JSON в {manifest_path}: {exc}") from exc
|
||
services = data.get("services", {})
|
||
if not isinstance(services, dict):
|
||
raise RuntimeError(f"`services` в {manifest_path} должен быть объектом.")
|
||
return services
|
||
|
||
|
||
def detect_profile(service_dir: Path, overrides: dict) -> Optional[str]:
|
||
if "profile" in overrides:
|
||
return str(overrides["profile"])
|
||
if discover_compose_file(service_dir) is not None:
|
||
return "compose"
|
||
if (service_dir / "deploy.sh").exists():
|
||
return "deploy_script"
|
||
if (service_dir / "redeploy.sh").exists() or (service_dir / "deploy" / "chart" / "values.yaml").exists():
|
||
return "helm_chart"
|
||
if detect_service_start_script(service_dir) is not None:
|
||
return "script"
|
||
return None
|
||
|
||
|
||
def detect_service_start_script(service_dir: Path) -> Optional[Path]:
|
||
for script_name in SCRIPT_START_CANDIDATES:
|
||
candidate = service_dir / script_name
|
||
if candidate.exists() and candidate.is_file():
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def detect_service_stop_script(service_dir: Path) -> Optional[Path]:
|
||
for script_name in SCRIPT_STOP_CANDIDATES:
|
||
candidate = service_dir / script_name
|
||
if candidate.exists() and candidate.is_file():
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def discover_compose_file(service_dir: Path) -> Optional[Path]:
|
||
for compose_name in COMPOSE_FILE_CANDIDATES:
|
||
candidate = service_dir / compose_name
|
||
if candidate.exists() and candidate.is_file():
|
||
return candidate
|
||
return None
|
||
|
||
|
||
def detect_checker_entry(checker_dir: Path) -> Optional[Path]:
|
||
checker_py = checker_dir / "checker.py"
|
||
if checker_py.exists():
|
||
return Path("checker.py")
|
||
|
||
dotted = sorted(checker_dir.glob("*.checker.py"))
|
||
if len(dotted) == 1:
|
||
return Path(dotted[0].name)
|
||
if len(dotted) > 1:
|
||
return Path(dotted[0].name)
|
||
|
||
checker_named = sorted(p for p in checker_dir.glob("*.py") if "checker" in p.name.lower())
|
||
if len(checker_named) == 1:
|
||
return Path(checker_named[0].name)
|
||
|
||
for candidate_name in ("checker", "checker.bin", "checker.out"):
|
||
candidate = checker_dir / candidate_name
|
||
if candidate.exists() and candidate.is_file():
|
||
return Path(candidate.name)
|
||
|
||
executable_files = sorted(
|
||
p
|
||
for p in checker_dir.iterdir()
|
||
if p.is_file() and p.suffix != ".py" and os.access(p, os.X_OK)
|
||
)
|
||
if len(executable_files) == 1:
|
||
return Path(executable_files[0].name)
|
||
checker_like_execs = [p for p in executable_files if "checker" in p.name.lower()]
|
||
if checker_like_execs:
|
||
return Path(checker_like_execs[0].name)
|
||
return None
|
||
|
||
|
||
def is_python_checker_entry(entry: Path) -> bool:
|
||
return entry.suffix.lower() == ".py"
|
||
|
||
|
||
def checker_protocol(entry_content: str) -> str:
|
||
if "https://" in entry_content:
|
||
return "https"
|
||
if "http://" in entry_content:
|
||
return "http"
|
||
if ".split(':')" in entry_content or "split(\":\")" in entry_content:
|
||
return "tcp"
|
||
return "unknown"
|
||
|
||
|
||
def checker_type(entry_content: str) -> str:
|
||
return "pfr" if "OK_WITH_FLAG_ID" in entry_content else "hackerdom"
|
||
|
||
|
||
def checker_places(entry_content: str) -> int:
|
||
count = len(re.findall(r"define_vuln", entry_content))
|
||
return max(1, count)
|
||
|
||
|
||
def detect_checker_default_port(entry_content: str) -> Optional[int]:
|
||
candidates: List[int] = []
|
||
patterns = [
|
||
r"\bPORT\b[^=\n]*=\s*(\d{2,5})",
|
||
r"\bport\s*=\s*(\d{2,5})",
|
||
r":\s*(\d{2,5})\b",
|
||
]
|
||
for pattern in patterns:
|
||
for match in re.findall(pattern, entry_content):
|
||
try:
|
||
value = int(match)
|
||
except ValueError:
|
||
continue
|
||
if 1 <= value <= 65535:
|
||
candidates.append(value)
|
||
if not candidates:
|
||
return None
|
||
# Prefer non-system and app-like ports when multiple constants appear.
|
||
for candidate in candidates:
|
||
if candidate >= 1024:
|
||
return candidate
|
||
return candidates[0]
|
||
|
||
|
||
def detect_checker_port_env(entry_content: str) -> Optional[str]:
|
||
matches = re.findall(r'os\.getenv\("([A-Z0-9_]*PORT[A-Z0-9_]*)"', entry_content)
|
||
if not matches:
|
||
return None
|
||
# Prefer service-specific names over generic PORT.
|
||
matches = [item for item in matches if item != "PORT"] + [item for item in matches if item == "PORT"]
|
||
return matches[0]
|
||
|
||
|
||
def checker_expects_host_with_port(entry_content: str, port_env_name: Optional[str]) -> bool:
|
||
# If checker has explicit port env, pass host without port and inject env.
|
||
if port_env_name:
|
||
return False
|
||
|
||
# Some checkers explicitly parse host:port from host argv.
|
||
# Important: avoid broad ".split(':')" checks, they trigger on unrelated
|
||
# values like flag_id.split(":") and cause host:port:port duplication.
|
||
explicit_host_split = re.search(
|
||
r"\b(?:host|hostname|request\.hostname|args\.host|argv\[2\]|sys\.argv\[2\])"
|
||
r"\s*\.split\(\s*['\"]:\s*['\"]\s*\)",
|
||
entry_content,
|
||
)
|
||
if explicit_host_split:
|
||
return True
|
||
|
||
# HTTP(S) checkers often construct URL directly from hostname.
|
||
if "request.hostname" in entry_content and ("http://" in entry_content or "https://" in entry_content):
|
||
return True
|
||
|
||
return False
|
||
|
||
|
||
def infer_make_artifacts(makefile_path: Path) -> List[str]:
|
||
makefile_text = makefile_path.read_text()
|
||
artifacts = re.findall(r"-o\s+([A-Za-z0-9_.\-\/]+)", makefile_text)
|
||
return sorted(set(artifacts))
|
||
|
||
|
||
def infer_checker_artifacts(checker_dir: Path) -> List[str]:
|
||
artifacts: List[str] = []
|
||
for py_file in checker_dir.glob("*.py"):
|
||
try:
|
||
text = py_file.read_text()
|
||
except UnicodeDecodeError:
|
||
continue
|
||
if "./libgen" in text or "libgen" in text:
|
||
artifacts.append("libgen")
|
||
return sorted(set(artifacts))
|
||
|
||
|
||
def detect_checker_build(checker_dir: Path, service_override: dict) -> Tuple[bool, Optional[str], Optional[str], List[str]]:
|
||
raw_cfg = service_override.get("checker_build")
|
||
if isinstance(raw_cfg, dict):
|
||
enabled = bool(raw_cfg.get("enabled", True))
|
||
if not enabled:
|
||
return False, None, None, []
|
||
profile = str(raw_cfg.get("profile", "cpp-make"))
|
||
command = str(raw_cfg.get("command", "make"))
|
||
raw_artifacts = raw_cfg.get("artifacts", [])
|
||
artifacts = [str(x) for x in raw_artifacts] if isinstance(raw_artifacts, list) else []
|
||
return True, profile, command, artifacts
|
||
|
||
makefile = checker_dir / "Makefile"
|
||
if makefile.exists():
|
||
artifacts = infer_make_artifacts(makefile) or infer_checker_artifacts(checker_dir)
|
||
return True, "cpp-make", "make", artifacts
|
||
|
||
return False, None, None, []
|
||
|
||
|
||
def _read_requirements_file(path: Path) -> List[str]:
|
||
requirements: List[str] = []
|
||
if not path.exists():
|
||
return requirements
|
||
for line in path.read_text().splitlines():
|
||
stripped = line.strip()
|
||
if not stripped or stripped.startswith("#"):
|
||
continue
|
||
requirements.append(stripped)
|
||
return requirements
|
||
|
||
|
||
def load_requirements(checker_dir: Path, checkers_root: Path) -> List[str]:
|
||
requirements: List[str] = []
|
||
# Global checkers requirements are common in some CTF sets (e.g. one file for all checkers).
|
||
# Add them first so per-checker requirements can extend/override pins if needed.
|
||
candidates = [
|
||
checkers_root / "requirements.txt",
|
||
checkers_root / "requirments.txt",
|
||
checker_dir / "requirements.txt",
|
||
checker_dir / "requirments.txt",
|
||
]
|
||
seen: set[str] = set()
|
||
for candidate in candidates:
|
||
for requirement in _read_requirements_file(candidate):
|
||
normalized = requirement.strip()
|
||
if normalized in seen:
|
||
continue
|
||
seen.add(normalized)
|
||
requirements.append(normalized)
|
||
return requirements
|
||
|
||
|
||
def _discover_compose_port_mapping(
|
||
service_dir: Path,
|
||
*,
|
||
protocol: str = "unknown",
|
||
preferred_port: Optional[int] = None,
|
||
) -> Optional[Tuple[int, str]]:
|
||
compose_path = discover_compose_file(service_dir)
|
||
if compose_path is None:
|
||
return None
|
||
text = compose_path.read_text()
|
||
matches = re.findall(r"(\d{2,5})\s*:\s*(\d{2,5})(?:/(\w+))?", text)
|
||
if not matches:
|
||
return None
|
||
mappings: List[Tuple[int, int, str]] = []
|
||
for host_port, container_port, transport in matches:
|
||
mappings.append((int(host_port), int(container_port), (transport.lower() if transport else "tcp")))
|
||
|
||
if preferred_port is not None:
|
||
for host_port, container_port, transport in mappings:
|
||
if preferred_port in {host_port, container_port}:
|
||
return host_port, transport
|
||
|
||
normalized_protocol = protocol.strip().lower()
|
||
if normalized_protocol == "udp":
|
||
for host_port, _container_port, transport in mappings:
|
||
if transport == "udp":
|
||
return host_port, transport
|
||
|
||
if normalized_protocol in {"http", "https"}:
|
||
web_ports = {80, 443, 3000, 5000, 5001, 5173, 8000, 8080, 8081, 8888, 9000}
|
||
for host_port, container_port, transport in mappings:
|
||
if transport == "tcp" and (container_port in web_ports or host_port in web_ports):
|
||
return host_port, transport
|
||
|
||
infra_ports = {5432, 3306, 1521, 27017, 6379, 5672, 15672, 9092}
|
||
for host_port, container_port, transport in mappings:
|
||
if host_port not in infra_ports and container_port not in infra_ports:
|
||
return host_port, transport
|
||
|
||
host_port, _container_port, transport = mappings[0]
|
||
return host_port, transport
|
||
|
||
|
||
def discover_host_port(
|
||
service_dir: Path,
|
||
profile: str,
|
||
protocol: str,
|
||
overrides: dict,
|
||
preferred_checker_port: Optional[int] = None,
|
||
) -> Optional[int]:
|
||
if "port" in overrides:
|
||
try:
|
||
return int(overrides["port"])
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
if profile == "compose":
|
||
mapping = _discover_compose_port_mapping(
|
||
service_dir,
|
||
protocol=protocol,
|
||
preferred_port=preferred_checker_port,
|
||
)
|
||
if mapping is None:
|
||
return None
|
||
return mapping[0]
|
||
|
||
values_files = list(service_dir.glob("**/values.yaml"))
|
||
node_ports: List[int] = []
|
||
for path in values_files:
|
||
text = path.read_text()
|
||
for match in re.findall(r"nodePort:\s*(\d{2,5})", text):
|
||
node_ports.append(int(match))
|
||
# Some charts (e.g. ingress-nginx values) keep NodePorts as:
|
||
# nodePorts:
|
||
# http: 31080
|
||
# https: 31443
|
||
if not node_ports:
|
||
for match in re.findall(r":\s*(\d{2,5})\b", text):
|
||
value = int(match)
|
||
if value >= 30000:
|
||
node_ports.append(value)
|
||
|
||
if node_ports:
|
||
if protocol == "http":
|
||
return min(node_ports)
|
||
return max(node_ports)
|
||
|
||
deploy_script = service_dir / "deploy.sh"
|
||
if deploy_script.exists():
|
||
script_text = deploy_script.read_text()
|
||
direct = re.search(r"localhost:(\d{2,5})", script_text)
|
||
if direct:
|
||
return int(direct.group(1))
|
||
|
||
start_script = detect_service_start_script(service_dir)
|
||
if start_script is not None:
|
||
script_text = start_script.read_text()
|
||
direct = re.search(r"localhost:(\d{2,5})", script_text)
|
||
if direct:
|
||
return int(direct.group(1))
|
||
port_assign = re.search(r"\bPORT\s*=\s*\"?(\d{2,5})\"?", script_text)
|
||
if port_assign:
|
||
return int(port_assign.group(1))
|
||
|
||
return None
|
||
|
||
|
||
def discover_host_transport(
|
||
service_dir: Path,
|
||
profile: str,
|
||
protocol: str,
|
||
overrides: dict,
|
||
preferred_checker_port: Optional[int] = None,
|
||
) -> str:
|
||
if "transport" in overrides:
|
||
return str(overrides["transport"]).strip().lower()
|
||
if profile == "compose":
|
||
mapping = _discover_compose_port_mapping(
|
||
service_dir,
|
||
protocol=protocol,
|
||
preferred_port=preferred_checker_port,
|
||
)
|
||
if mapping is not None:
|
||
return mapping[1]
|
||
if str(protocol).strip().lower() == "udp":
|
||
return "udp"
|
||
return "tcp"
|
||
|
||
|
||
def discover_services(
|
||
services_root: Path,
|
||
checkers_root: Path,
|
||
selected_services: Optional[set[str]] = None,
|
||
) -> Tuple[List[ServiceSpec], ValidationReport]:
|
||
report = ValidationReport()
|
||
overrides = parse_manifest(services_root)
|
||
|
||
if not services_root.exists():
|
||
report.errors.append(f"Папка сервисов не найдена: {services_root}")
|
||
return [], report
|
||
if not checkers_root.exists():
|
||
report.errors.append(f"Папка чекеров не найдена: {checkers_root}")
|
||
return [], report
|
||
|
||
specs: List[ServiceSpec] = []
|
||
service_dirs = sorted(p for p in services_root.iterdir() if p.is_dir() and not p.name.startswith("."))
|
||
|
||
for service_dir in service_dirs:
|
||
name = service_dir.name
|
||
if selected_services and name not in selected_services:
|
||
continue
|
||
|
||
service_override = overrides.get(name, {})
|
||
profile = detect_profile(service_dir, service_override)
|
||
if profile is None:
|
||
report.warnings.append(f"Пропускаю `{name}`: не найден поддерживаемый профиль деплоя.")
|
||
continue
|
||
|
||
checker_name = str(service_override.get("checker_dir", name))
|
||
checker_dir = checkers_root / checker_name
|
||
if not checker_dir.exists():
|
||
report.errors.append(f"`{name}`: не найдена папка чекера `{checker_dir}`.")
|
||
continue
|
||
|
||
entry_override = service_override.get("checker_entry")
|
||
if entry_override:
|
||
checker_entry = Path(str(entry_override))
|
||
if not (checker_dir / checker_entry).exists():
|
||
report.errors.append(f"`{name}`: не найден checker entry `{checker_entry}` в `{checker_dir}`.")
|
||
continue
|
||
else:
|
||
detected_entry = detect_checker_entry(checker_dir)
|
||
if detected_entry is None:
|
||
report.errors.append(
|
||
f"`{name}`: не найден checker entry (`checker.py`, `*.checker.py` или бинарник checker*)."
|
||
)
|
||
continue
|
||
checker_entry = detected_entry
|
||
|
||
checker_path = checker_dir / checker_entry
|
||
entry_text = ""
|
||
if is_python_checker_entry(checker_entry):
|
||
try:
|
||
entry_text = checker_path.read_text()
|
||
except UnicodeDecodeError as exc:
|
||
report.errors.append(f"`{name}`: не удалось прочитать python checker `{checker_entry}`: {exc}")
|
||
continue
|
||
|
||
detected_protocol = checker_protocol(entry_text) if entry_text else "unknown"
|
||
detected_type = checker_type(entry_text) if entry_text else "hackerdom"
|
||
detected_places = checker_places(entry_text) if entry_text else 1
|
||
detected_checker_port = detect_checker_default_port(entry_text) if entry_text else None
|
||
checker_port_env = detect_checker_port_env(entry_text) if entry_text else None
|
||
expects_host_with_port = checker_expects_host_with_port(entry_text, checker_port_env) if entry_text else False
|
||
|
||
protocol = str(service_override.get("protocol", detected_protocol))
|
||
task_type = str(service_override.get("checker_type", detected_type))
|
||
places = int(service_override.get("places", detected_places))
|
||
timeout = int(service_override.get("checker_timeout", 30))
|
||
host_port = discover_host_port(
|
||
service_dir,
|
||
profile,
|
||
protocol,
|
||
service_override,
|
||
preferred_checker_port=detected_checker_port,
|
||
)
|
||
host_transport = discover_host_transport(
|
||
service_dir,
|
||
profile,
|
||
protocol,
|
||
service_override,
|
||
preferred_checker_port=detected_checker_port,
|
||
)
|
||
requirements = load_requirements(checker_dir, checkers_root)
|
||
build_enabled, build_profile, build_command, build_artifacts = detect_checker_build(checker_dir, service_override)
|
||
|
||
spec = ServiceSpec(
|
||
name=name,
|
||
service_dir=service_dir,
|
||
checker_dir=checker_dir,
|
||
profile=profile,
|
||
checker_entry=checker_entry,
|
||
checker_type=task_type,
|
||
places=places,
|
||
checker_timeout=timeout,
|
||
protocol=protocol,
|
||
host_port=host_port,
|
||
host_transport=host_transport,
|
||
checker_port_env=checker_port_env,
|
||
expects_host_with_port=expects_host_with_port,
|
||
requirements=requirements,
|
||
checker_build_enabled=build_enabled,
|
||
checker_build_profile=build_profile,
|
||
checker_build_command=build_command,
|
||
checker_build_artifacts=build_artifacts,
|
||
start_cmd=service_override.get("start_cmd"),
|
||
stop_cmd=service_override.get("stop_cmd"),
|
||
)
|
||
specs.append(spec)
|
||
|
||
if not specs:
|
||
report.errors.append("Не найдено ни одного валидного сервиса для запуска.")
|
||
return specs, report
|
||
|
||
|
||
def validate_preflight(specs: List[ServiceSpec], board_port: Optional[int] = None) -> ValidationReport:
|
||
report = ValidationReport()
|
||
require_command("docker", report)
|
||
seen_ports: Dict[int, str] = {}
|
||
|
||
for spec in specs:
|
||
if spec.host_port is None:
|
||
report.errors.append(f"`{spec.name}`: не удалось определить host-порт (укажи `port` в `forcad-local.json`).")
|
||
else:
|
||
if board_port is not None and spec.host_port == board_port:
|
||
report.errors.append(
|
||
f"`{spec.name}`: host-порт {spec.host_port} конфликтует с портом борды {board_port}."
|
||
)
|
||
if spec.host_port in seen_ports:
|
||
report.errors.append(
|
||
f"Конфликт портов: `{spec.name}` и `{seen_ports[spec.host_port]}` используют {spec.host_port}."
|
||
)
|
||
seen_ports[spec.host_port] = spec.name
|
||
|
||
if spec.profile == "compose" and discover_compose_file(spec.service_dir) is None:
|
||
report.errors.append(
|
||
f"`{spec.name}`: профиль compose выбран, но нет compose-файла "
|
||
f"({', '.join(COMPOSE_FILE_CANDIDATES)})."
|
||
)
|
||
|
||
if spec.profile in {"deploy_script", "helm_chart"}:
|
||
for dep in ("helm", "kubectl"):
|
||
if shutil.which(dep) is None:
|
||
report.warnings.append(
|
||
f"`{spec.name}`: команда `{dep}` не найдена; сервис может не подняться."
|
||
)
|
||
if (spec.service_dir / "k3d-config.yaml").exists() and shutil.which("k3d") is None:
|
||
report.warnings.append(f"`{spec.name}`: найден `k3d-config.yaml`, но команда `k3d` отсутствует.")
|
||
|
||
if spec.profile == "script":
|
||
if detect_service_start_script(spec.service_dir) is None and not spec.start_cmd:
|
||
report.errors.append(
|
||
f"`{spec.name}`: profile=script, но не найден start script ({', '.join(SCRIPT_START_CANDIDATES)})."
|
||
)
|
||
|
||
checker_entry_path = spec.checker_dir / spec.checker_entry
|
||
if not checker_entry_path.exists():
|
||
report.errors.append(f"`{spec.name}`: checker entry `{spec.checker_entry}` отсутствует.")
|
||
elif not is_python_checker_entry(spec.checker_entry) and not os.access(checker_entry_path, os.X_OK):
|
||
report.warnings.append(
|
||
f"`{spec.name}`: binary checker `{spec.checker_entry}` не executable в исходной папке; "
|
||
"попробую выставить +x в runtime."
|
||
)
|
||
|
||
if spec.checker_build_enabled:
|
||
if not spec.checker_build_profile:
|
||
report.errors.append(f"`{spec.name}`: включена сборка чекера, но не указан checker_build_profile.")
|
||
elif spec.checker_build_profile not in BUILDER_PROFILES:
|
||
report.errors.append(
|
||
f"`{spec.name}`: неизвестный checker build profile `{spec.checker_build_profile}`."
|
||
)
|
||
if not spec.checker_build_command:
|
||
report.errors.append(f"`{spec.name}`: включена сборка чекера, но не задан checker_build_command.")
|
||
if not spec.checker_build_artifacts:
|
||
report.warnings.append(
|
||
f"`{spec.name}`: не заданы checker build artifacts, кэш сборки для него отключён."
|
||
)
|
||
|
||
return report
|
||
|
||
|
||
def merge_reports(*reports: ValidationReport) -> ValidationReport:
|
||
result = ValidationReport()
|
||
for report in reports:
|
||
result.errors.extend(report.errors)
|
||
result.warnings.extend(report.warnings)
|
||
return result
|
||
|
||
|
||
def print_report(report: ValidationReport) -> None:
|
||
for warning in report.warnings:
|
||
warn(warning)
|
||
for error_message in report.errors:
|
||
err(error_message)
|
||
|
||
|
||
def resolve_checkers_root(services_root: Path, checkers_root_arg: Optional[Path]) -> Path:
|
||
if checkers_root_arg:
|
||
return checkers_root_arg.resolve()
|
||
|
||
sibling = services_root.parent / "checkers"
|
||
if sibling.exists():
|
||
return sibling.resolve()
|
||
|
||
inside = services_root / "checkers"
|
||
if inside.exists():
|
||
return inside.resolve()
|
||
|
||
raise RuntimeError(
|
||
"Не удалось определить папку чекеров автоматически. Укажи `--checkers-root` явно."
|
||
)
|
||
|
||
|
||
def safe_service_var(service_name: str) -> str:
|
||
normalized = re.sub(r"[^A-Za-z0-9]+", "_", service_name).upper()
|
||
return f"AD_TARGET_{normalized}"
|
||
|
||
|
||
def ensure_clean_dir(path: Path) -> None:
|
||
if path.exists():
|
||
shutil.rmtree(path)
|
||
path.mkdir(parents=True, exist_ok=True)
|
||
|
||
|
||
def _script_state_dir() -> Path:
|
||
base = Path(tempfile.gettempdir()) / "forcad-local-script-state"
|
||
base.mkdir(parents=True, exist_ok=True)
|
||
return base
|
||
|
||
|
||
def _script_process_key(spec: ServiceSpec) -> str:
|
||
digest = hashlib.sha256(str(spec.service_dir.resolve()).encode()).hexdigest()[:12]
|
||
return f"{spec.name}-{digest}"
|
||
|
||
|
||
def script_pid_path(spec: ServiceSpec) -> Path:
|
||
return _script_state_dir() / f"{_script_process_key(spec)}.pid"
|
||
|
||
|
||
def script_log_path(spec: ServiceSpec) -> Path:
|
||
return _script_state_dir() / f"{_script_process_key(spec)}.log"
|
||
|
||
|
||
def _read_script_pid(spec: ServiceSpec) -> Optional[int]:
|
||
pid_file = script_pid_path(spec)
|
||
if not pid_file.exists():
|
||
return None
|
||
try:
|
||
return int(pid_file.read_text().strip())
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def _process_alive(pid: int) -> bool:
|
||
if pid <= 0:
|
||
return False
|
||
try:
|
||
os.kill(pid, 0)
|
||
except OSError:
|
||
return False
|
||
return True
|
||
|
||
|
||
def _terminate_pid(pid: int, *, grace_seconds: float = 3.0) -> None:
|
||
if not _process_alive(pid):
|
||
return
|
||
try:
|
||
if os.name == "posix":
|
||
os.killpg(pid, signal.SIGTERM)
|
||
else:
|
||
os.kill(pid, signal.SIGTERM)
|
||
except ProcessLookupError:
|
||
return
|
||
except OSError:
|
||
pass
|
||
deadline = time.time() + grace_seconds
|
||
while time.time() < deadline:
|
||
if not _process_alive(pid):
|
||
return
|
||
time.sleep(0.2)
|
||
try:
|
||
if os.name == "posix":
|
||
os.killpg(pid, signal.SIGKILL)
|
||
else:
|
||
os.kill(pid, signal.SIGKILL)
|
||
except OSError:
|
||
return
|
||
|
||
|
||
def _kill_by_port(port: Optional[int]) -> None:
|
||
if port is None or not is_port_open(port):
|
||
return
|
||
if shutil.which("lsof"):
|
||
run_shell(f"lsof -ti tcp:{port} | xargs -r kill -TERM", check=False)
|
||
time.sleep(0.5)
|
||
if is_port_open(port):
|
||
run_shell(f"lsof -ti tcp:{port} | xargs -r kill -KILL", check=False)
|
||
return
|
||
if shutil.which("fuser"):
|
||
run_command(["fuser", "-k", f"{port}/tcp"], check=False)
|
||
|
||
|
||
def _kill_script_signature(spec: ServiceSpec) -> None:
|
||
script = detect_service_start_script(spec.service_dir)
|
||
if not script or shutil.which("pgrep") is None:
|
||
return
|
||
quoted = shlex.quote(str(script))
|
||
run_shell(f"pgrep -f {quoted} | xargs -r kill -TERM", check=False)
|
||
|
||
|
||
def resolve_builder_image(orchestrator_root: Path, profile: str) -> str:
|
||
profile_config = BUILDER_PROFILES.get(profile)
|
||
if not profile_config:
|
||
raise RuntimeError(f"Unknown builder profile: {profile}")
|
||
|
||
image = profile_config["image"]
|
||
inspect = run_command(["docker", "image", "inspect", image], check=False)
|
||
if inspect.returncode == 0:
|
||
return image
|
||
|
||
dockerfile = orchestrator_root / profile_config["dockerfile"]
|
||
if not dockerfile.exists():
|
||
raise RuntimeError(f"Builder dockerfile not found for profile `{profile}`: {dockerfile}")
|
||
|
||
info(f"Building checker builder image `{image}` from `{dockerfile}`.")
|
||
run_command(
|
||
[
|
||
"docker",
|
||
"build",
|
||
"-f",
|
||
str(dockerfile),
|
||
"-t",
|
||
image,
|
||
str(orchestrator_root),
|
||
]
|
||
)
|
||
return image
|
||
|
||
|
||
def hash_checker_inputs(checker_dir: Path, build_profile: str, build_command: str) -> str:
|
||
digest = hashlib.sha256()
|
||
digest.update(build_profile.encode())
|
||
digest.update(b"\0")
|
||
digest.update(build_command.encode())
|
||
digest.update(b"\0")
|
||
|
||
for item in sorted(checker_dir.rglob("*")):
|
||
if not item.is_file():
|
||
continue
|
||
rel = item.relative_to(checker_dir).as_posix()
|
||
digest.update(rel.encode())
|
||
digest.update(b"\0")
|
||
digest.update(item.read_bytes())
|
||
digest.update(b"\0")
|
||
return digest.hexdigest()
|
||
|
||
|
||
def restore_build_cache(
|
||
service: ServiceSpec,
|
||
checker_runtime_dir: Path,
|
||
cache_root: Path,
|
||
cache_key: str,
|
||
) -> bool:
|
||
if not service.checker_build_artifacts:
|
||
return False
|
||
|
||
entry = cache_root / service.name / cache_key
|
||
if not entry.exists():
|
||
return False
|
||
|
||
for artifact in service.checker_build_artifacts:
|
||
src = entry / artifact
|
||
dst = checker_runtime_dir / artifact
|
||
if not src.exists():
|
||
return False
|
||
if src.is_dir():
|
||
if dst.exists():
|
||
shutil.rmtree(dst)
|
||
shutil.copytree(src, dst)
|
||
else:
|
||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(src, dst)
|
||
|
||
info(f"Checker build cache hit for `{service.name}`.")
|
||
return True
|
||
|
||
|
||
def store_build_cache(
|
||
service: ServiceSpec,
|
||
checker_runtime_dir: Path,
|
||
cache_root: Path,
|
||
cache_key: str,
|
||
) -> None:
|
||
if not service.checker_build_artifacts:
|
||
return
|
||
|
||
entry = cache_root / service.name / cache_key
|
||
if entry.exists():
|
||
shutil.rmtree(entry)
|
||
entry.mkdir(parents=True, exist_ok=True)
|
||
|
||
for artifact in service.checker_build_artifacts:
|
||
src = checker_runtime_dir / artifact
|
||
if not src.exists():
|
||
continue
|
||
dst = entry / artifact
|
||
if src.is_dir():
|
||
shutil.copytree(src, dst)
|
||
else:
|
||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(src, dst)
|
||
|
||
|
||
def create_wrapper(service: ServiceSpec, destination: Path) -> None:
|
||
env_var = safe_service_var(service.name)
|
||
default_target = "host.docker.internal"
|
||
default_port = str(service.host_port) if service.host_port is not None else ""
|
||
checker_port_env = service.checker_port_env or ""
|
||
needs_host_port = "True" if service.expects_host_with_port else "False"
|
||
wrapper = f"""#!/usr/bin/env python3
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
LOCAL_ALIASES = {sorted(LOCAL_HOST_ALIASES)!r}
|
||
DEFAULT_TARGET = os.environ.get("{env_var}", "{default_target}")
|
||
DEFAULT_PORT = "{default_port}"
|
||
CHECKER_PORT_ENV = "{checker_port_env}"
|
||
NEEDS_HOST_PORT = {needs_host_port}
|
||
|
||
def patch_hostname(argv):
|
||
if len(argv) < 3:
|
||
return argv
|
||
action = argv[1].lower()
|
||
if action not in {{"check", "put", "get", "test"}}:
|
||
return argv
|
||
host = argv[2]
|
||
if host in LOCAL_ALIASES:
|
||
host = DEFAULT_TARGET
|
||
if NEEDS_HOST_PORT and DEFAULT_PORT and ":" not in host:
|
||
host = f"{{host}}:{{DEFAULT_PORT}}"
|
||
argv[2] = host
|
||
return argv
|
||
|
||
def main():
|
||
if CHECKER_PORT_ENV and DEFAULT_PORT:
|
||
os.environ.setdefault(CHECKER_PORT_ENV, DEFAULT_PORT)
|
||
checker_path = Path(__file__).with_name("{service.checker_entry.as_posix()}")
|
||
argv = patch_hostname(sys.argv[:])
|
||
checker_cmd = [sys.executable, str(checker_path), *argv[1:]]
|
||
if checker_path.suffix.lower() != ".py":
|
||
checker_cmd = [str(checker_path), *argv[1:]]
|
||
completed = subprocess.run(checker_cmd, cwd=Path(__file__).parent)
|
||
raise SystemExit(completed.returncode)
|
||
|
||
if __name__ == "__main__":
|
||
main()
|
||
"""
|
||
destination.write_text(wrapper)
|
||
destination.chmod(0o755)
|
||
|
||
|
||
def generate_runtime(
|
||
runtime_dir: Path,
|
||
specs: List[ServiceSpec],
|
||
) -> dict:
|
||
ensure_clean_dir(runtime_dir)
|
||
checkers_dir = runtime_dir / "checkers"
|
||
checkers_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
requirements = {"gornilo", "checklib==0.7.0"}
|
||
|
||
for spec in specs:
|
||
target_dir = checkers_dir / spec.name
|
||
shutil.copytree(spec.checker_dir, target_dir, dirs_exist_ok=True)
|
||
checker_target = target_dir / spec.checker_entry
|
||
if checker_target.exists() and not is_python_checker_entry(spec.checker_entry):
|
||
mode = checker_target.stat().st_mode
|
||
checker_target.chmod(mode | 0o111)
|
||
create_wrapper(spec, target_dir / "forcad_wrapper.py")
|
||
|
||
for requirement in spec.requirements:
|
||
requirements.add(requirement)
|
||
|
||
(checkers_dir / "requirements.txt").write_text("\n".join(sorted(requirements)) + "\n")
|
||
|
||
return {
|
||
"runtime_dir": str(runtime_dir),
|
||
"checkers_dir": str(checkers_dir),
|
||
}
|
||
|
||
|
||
def build_checker_artifacts(
|
||
runtime_dir: Path,
|
||
specs: List[ServiceSpec],
|
||
cache_dir: Path,
|
||
failure_policy: str = "strict",
|
||
) -> Tuple[List[ServiceSpec], Dict[str, str]]:
|
||
orchestrator_root = Path(__file__).resolve().parent
|
||
cache_root = cache_dir / "checker-builds"
|
||
cache_root.mkdir(parents=True, exist_ok=True)
|
||
builder_images: Dict[str, str] = {}
|
||
ready_specs: List[ServiceSpec] = []
|
||
failed: Dict[str, str] = {}
|
||
|
||
for spec in specs:
|
||
try:
|
||
if not spec.checker_build_enabled:
|
||
ready_specs.append(spec)
|
||
continue
|
||
if not spec.checker_build_profile or not spec.checker_build_command:
|
||
raise RuntimeError(f"Invalid checker build config for `{spec.name}`.")
|
||
|
||
checker_runtime_dir = runtime_dir / "checkers" / spec.name
|
||
cache_key = hash_checker_inputs(
|
||
checker_dir=checker_runtime_dir,
|
||
build_profile=spec.checker_build_profile,
|
||
build_command=spec.checker_build_command,
|
||
)
|
||
if restore_build_cache(spec, checker_runtime_dir, cache_root, cache_key):
|
||
ready_specs.append(spec)
|
||
continue
|
||
|
||
if spec.checker_build_profile not in builder_images:
|
||
builder_images[spec.checker_build_profile] = resolve_builder_image(
|
||
orchestrator_root=orchestrator_root,
|
||
profile=spec.checker_build_profile,
|
||
)
|
||
image = builder_images[spec.checker_build_profile]
|
||
|
||
info(f"Containerized checker build for `{spec.name}` ({spec.checker_build_profile}).")
|
||
run_command(
|
||
[
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--add-host",
|
||
"host.docker.internal:host-gateway",
|
||
"-v",
|
||
f"{checker_runtime_dir}:/work",
|
||
"-w",
|
||
"/work",
|
||
image,
|
||
"bash",
|
||
"-lc",
|
||
spec.checker_build_command,
|
||
]
|
||
)
|
||
store_build_cache(spec, checker_runtime_dir, cache_root, cache_key)
|
||
ready_specs.append(spec)
|
||
except Exception as exc: # noqa: BLE001
|
||
message = str(exc)
|
||
failed[spec.name] = message
|
||
if failure_policy == "strict":
|
||
raise
|
||
warn(f"Skipping `{spec.name}` due checker build failure: {message}")
|
||
|
||
return ready_specs, failed
|
||
|
||
|
||
def is_port_open(port: int) -> bool:
|
||
with socket.socket() as sock:
|
||
sock.settimeout(0.5)
|
||
return sock.connect_ex(("127.0.0.1", int(port))) == 0
|
||
|
||
|
||
def is_udp_port_bound(port: int) -> bool:
|
||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock:
|
||
try:
|
||
sock.bind(("127.0.0.1", int(port)))
|
||
return False
|
||
except OSError:
|
||
return True
|
||
|
||
|
||
def ensure_service_port_available(spec: ServiceSpec) -> None:
|
||
if spec.host_port is None:
|
||
return
|
||
if not is_port_open(spec.host_port):
|
||
return
|
||
warn(f"Port {spec.host_port} is busy before deploying `{spec.name}`, trying cleanup.")
|
||
stop_service(spec, include_legacy_compose=True)
|
||
time.sleep(1)
|
||
if is_port_open(spec.host_port):
|
||
raise RuntimeError(
|
||
f"Port {spec.host_port} is still busy before deploying `{spec.name}`. "
|
||
"Run cleanup and retry."
|
||
)
|
||
|
||
|
||
def prewarm_k3d_images(
|
||
specs: List[ServiceSpec],
|
||
strict: bool,
|
||
image_pull_retries: int,
|
||
image_pull_retry_delay: int,
|
||
) -> None:
|
||
if not any(is_k3d_service(spec) for spec in specs):
|
||
return
|
||
for image in K3D_PREWARM_IMAGES:
|
||
try:
|
||
run_command_with_retries(
|
||
["docker", "pull", image],
|
||
retries=image_pull_retries,
|
||
retry_delay=image_pull_retry_delay,
|
||
purpose=f"prewarm pull `{image}`",
|
||
)
|
||
except Exception as exc: # noqa: BLE001
|
||
message = f"Failed to prewarm image `{image}`: {exc}"
|
||
if strict:
|
||
raise RuntimeError(message) from exc
|
||
warn(message)
|
||
|
||
|
||
def auto_cleanup_services(specs: List[ServiceSpec]) -> None:
|
||
info("Auto-cleanup previous service runs.")
|
||
for spec in reversed(specs):
|
||
stop_service(spec, include_legacy_compose=True)
|
||
|
||
|
||
def rollback_services(specs: List[ServiceSpec]) -> None:
|
||
if not specs:
|
||
return
|
||
warn("Rolling back already deployed services.")
|
||
for spec in reversed(specs):
|
||
stop_service(spec, include_legacy_compose=True)
|
||
|
||
|
||
def deploy_service_with_retry(
|
||
spec: ServiceSpec,
|
||
k3d_retries: int,
|
||
retry_delay: int,
|
||
image_pull_retries: int,
|
||
image_pull_retry_delay: int,
|
||
) -> None:
|
||
attempts = 1 + (k3d_retries if is_k3d_service(spec) else 0)
|
||
for attempt in range(1, attempts + 1):
|
||
try:
|
||
ensure_service_port_available(spec)
|
||
deploy_service(
|
||
spec,
|
||
image_pull_retries=image_pull_retries,
|
||
image_pull_retry_delay=image_pull_retry_delay,
|
||
)
|
||
return
|
||
except Exception as exc: # noqa: BLE001
|
||
if attempt >= attempts:
|
||
raise
|
||
warn(f"Deploy `{spec.name}` attempt {attempt}/{attempts} failed: {exc}")
|
||
stop_service(spec, include_legacy_compose=True)
|
||
time.sleep(retry_delay)
|
||
|
||
|
||
def compose_project_names(spec: ServiceSpec) -> List[str]:
|
||
names: List[str] = [f"forcadsvc_{spec.name}"]
|
||
legacy = spec.service_dir.name
|
||
if legacy not in names:
|
||
names.append(legacy)
|
||
return names
|
||
|
||
|
||
def is_k3d_service(spec: ServiceSpec) -> bool:
|
||
if (spec.service_dir / "k3d-config.yaml").exists():
|
||
return True
|
||
deploy_script = spec.service_dir / "deploy.sh"
|
||
if deploy_script.exists():
|
||
return "k3d cluster create" in deploy_script.read_text()
|
||
return False
|
||
|
||
|
||
def service_priority(spec: ServiceSpec) -> Tuple[int, str]:
|
||
# Start k3d-heavy services first while Docker daemon is still idle.
|
||
if is_k3d_service(spec):
|
||
return (0, spec.name)
|
||
if spec.profile in {"compose", "script"}:
|
||
return (1, spec.name)
|
||
return (2, spec.name)
|
||
|
||
|
||
def order_services_for_deploy(specs: List[ServiceSpec]) -> List[ServiceSpec]:
|
||
return sorted(specs, key=service_priority)
|
||
|
||
|
||
def _start_script_service(spec: ServiceSpec) -> None:
|
||
start_script = detect_service_start_script(spec.service_dir)
|
||
if start_script is None:
|
||
raise RuntimeError(
|
||
f"`{spec.name}`: не найден script entry для профиля script ({', '.join(SCRIPT_START_CANDIDATES)})."
|
||
)
|
||
|
||
pid_file = script_pid_path(spec)
|
||
existing_pid = _read_script_pid(spec)
|
||
if existing_pid and _process_alive(existing_pid):
|
||
warn(f"`{spec.name}` script already running (pid={existing_pid}), stopping old process first.")
|
||
_terminate_pid(existing_pid)
|
||
pid_file.parent.mkdir(parents=True, exist_ok=True)
|
||
log_file = script_log_path(spec)
|
||
with log_file.open("ab") as output:
|
||
process = subprocess.Popen(
|
||
["bash", str(start_script)],
|
||
cwd=spec.service_dir,
|
||
stdout=output,
|
||
stderr=output,
|
||
start_new_session=True,
|
||
)
|
||
pid_file.write_text(str(process.pid))
|
||
info(f"Started script service `{spec.name}` via `{start_script.name}` (pid={process.pid}).")
|
||
|
||
|
||
def deploy_service(
|
||
spec: ServiceSpec,
|
||
*,
|
||
image_pull_retries: int = 0,
|
||
image_pull_retry_delay: int = 5,
|
||
) -> None:
|
||
if spec.start_cmd:
|
||
run_shell(spec.start_cmd, cwd=spec.service_dir)
|
||
return
|
||
|
||
if spec.profile == "compose":
|
||
compose_path = discover_compose_file(spec.service_dir)
|
||
if compose_path is None:
|
||
raise RuntimeError(
|
||
f"`{spec.name}`: не найден compose-файл ({', '.join(COMPOSE_FILE_CANDIDATES)})."
|
||
)
|
||
project = compose_project_names(spec)[0]
|
||
# Pull externally referenced images with retries before "up".
|
||
run_command_with_retries(
|
||
[
|
||
"docker",
|
||
"compose",
|
||
"-f",
|
||
str(compose_path),
|
||
"-p",
|
||
project,
|
||
"pull",
|
||
],
|
||
retries=image_pull_retries,
|
||
retry_delay=image_pull_retry_delay,
|
||
purpose=f"image pull for `{spec.name}`",
|
||
)
|
||
run_command(
|
||
[
|
||
"docker",
|
||
"compose",
|
||
"-f",
|
||
str(compose_path),
|
||
"-p",
|
||
project,
|
||
"up",
|
||
"-d",
|
||
"--build",
|
||
"--remove-orphans",
|
||
]
|
||
)
|
||
return
|
||
|
||
if spec.profile == "script":
|
||
_start_script_service(spec)
|
||
return
|
||
|
||
deploy_script = spec.service_dir / "deploy.sh"
|
||
if deploy_script.exists():
|
||
run_command(["bash", str(deploy_script)], cwd=spec.service_dir)
|
||
return
|
||
|
||
redeploy_script = spec.service_dir / "redeploy.sh"
|
||
if redeploy_script.exists():
|
||
run_command(["bash", str(redeploy_script)], cwd=spec.service_dir)
|
||
return
|
||
|
||
build_script = spec.service_dir / "build-docker.sh"
|
||
chart_dir = spec.service_dir / "deploy" / "chart"
|
||
if build_script.exists():
|
||
run_command(["bash", str(build_script)], cwd=spec.service_dir)
|
||
if chart_dir.exists():
|
||
run_command(
|
||
[
|
||
"helm",
|
||
"upgrade",
|
||
"--install",
|
||
spec.name,
|
||
str(chart_dir),
|
||
"--create-namespace",
|
||
"--namespace",
|
||
spec.name,
|
||
"--wait",
|
||
"--timeout",
|
||
"10m",
|
||
],
|
||
cwd=spec.service_dir,
|
||
)
|
||
return
|
||
|
||
raise RuntimeError(f"Не знаю, как поднять сервис `{spec.name}` (profile={spec.profile}).")
|
||
|
||
|
||
def stop_service(spec: ServiceSpec, include_legacy_compose: bool = True) -> None:
|
||
if spec.stop_cmd:
|
||
run_shell(spec.stop_cmd, cwd=spec.service_dir, check=False)
|
||
if spec.profile != "script":
|
||
return
|
||
|
||
if spec.profile == "script":
|
||
stop_script = detect_service_stop_script(spec.service_dir)
|
||
if stop_script is not None:
|
||
run_command(["bash", str(stop_script)], cwd=spec.service_dir, check=False)
|
||
pid = _read_script_pid(spec)
|
||
if pid is not None:
|
||
_terminate_pid(pid)
|
||
script_pid_path(spec).unlink(missing_ok=True)
|
||
_kill_by_port(spec.host_port)
|
||
if spec.host_port is None or is_port_open(spec.host_port):
|
||
_kill_script_signature(spec)
|
||
_kill_by_port(spec.host_port)
|
||
return
|
||
|
||
if spec.profile == "compose":
|
||
compose_path = discover_compose_file(spec.service_dir)
|
||
if compose_path is None:
|
||
warn(
|
||
f"`{spec.name}`: compose profile without compose file "
|
||
f"({', '.join(COMPOSE_FILE_CANDIDATES)}), skip stop."
|
||
)
|
||
return
|
||
projects = compose_project_names(spec)
|
||
if not include_legacy_compose:
|
||
projects = projects[:1]
|
||
for project in projects:
|
||
run_command(
|
||
["docker", "compose", "-f", str(compose_path), "-p", project, "down", "-v", "--remove-orphans"],
|
||
check=False,
|
||
)
|
||
return
|
||
|
||
if (spec.service_dir / "k3d-config.yaml").exists() and shutil.which("k3d"):
|
||
config_text = (spec.service_dir / "k3d-config.yaml").read_text()
|
||
match = re.search(r"\bname:\s*([A-Za-z0-9_.-]+)", config_text)
|
||
cluster_name = match.group(1) if match else f"{spec.name}-cluster"
|
||
run_command(["k3d", "cluster", "delete", cluster_name], check=False)
|
||
|
||
if shutil.which("helm"):
|
||
run_command(["helm", "uninstall", spec.name, "-n", spec.name], check=False)
|
||
|
||
|
||
def wait_for_service(spec: ServiceSpec, timeout_seconds: int = 180) -> None:
|
||
if spec.host_port is None:
|
||
raise RuntimeError(f"У `{spec.name}` не определён host_port.")
|
||
|
||
info(
|
||
f"Ожидаю доступность `{spec.name}` на порту {spec.host_port} "
|
||
f"(transport={spec.host_transport}) ..."
|
||
)
|
||
deadline = time.time() + timeout_seconds
|
||
if spec.host_transport.lower() == "udp":
|
||
while time.time() < deadline:
|
||
if is_udp_port_bound(spec.host_port):
|
||
info(f"`{spec.name}` доступен на UDP 127.0.0.1:{spec.host_port}")
|
||
return
|
||
time.sleep(2)
|
||
raise RuntimeError(
|
||
f"`{spec.name}` не стал доступен на UDP-порту {spec.host_port} за {timeout_seconds} сек."
|
||
)
|
||
|
||
while time.time() < deadline:
|
||
with socket.socket() as sock:
|
||
sock.settimeout(2.0)
|
||
try:
|
||
sock.connect(("127.0.0.1", int(spec.host_port)))
|
||
info(f"`{spec.name}` доступен на 127.0.0.1:{spec.host_port}")
|
||
return
|
||
except OSError:
|
||
time.sleep(2)
|
||
raise RuntimeError(f"`{spec.name}` не стал доступен на порту {spec.host_port} за {timeout_seconds} сек.")
|
||
|
||
|
||
def _smoke_check_timeout(spec: ServiceSpec) -> float:
|
||
# A bit of slack on top of the checker's own timeout for process/exec overhead.
|
||
return float(max(5, spec.checker_timeout) + 10)
|
||
|
||
|
||
def _build_light_checkers_image(checkers_dir: Path) -> str:
|
||
"""Build (or reuse) an image with all checker requirements pre-installed.
|
||
|
||
Building once avoids re-running `pip install` inside every parallel
|
||
`docker run`, which used to happen serially inside a single shared
|
||
container and made per-service timeouts impossible to enforce.
|
||
"""
|
||
requirements_path = checkers_dir / "requirements.txt"
|
||
requirements_bytes = requirements_path.read_bytes() if requirements_path.exists() else b""
|
||
tag = hashlib.sha256(requirements_bytes).hexdigest()[:16]
|
||
image = f"forcad-local-light-checkers:{tag}"
|
||
|
||
inspect = run_command(["docker", "image", "inspect", image], check=False)
|
||
if inspect.returncode == 0:
|
||
return image
|
||
|
||
build_ctx = Path(tempfile.mkdtemp(prefix="forcad-local-light-"))
|
||
try:
|
||
(build_ctx / "requirements.txt").write_bytes(requirements_bytes)
|
||
(build_ctx / "Dockerfile").write_text(
|
||
"FROM python:3.11-slim\n"
|
||
"COPY requirements.txt /tmp/requirements.txt\n"
|
||
"RUN python3 -m pip install --no-cache-dir -r /tmp/requirements.txt\n"
|
||
)
|
||
info(f"Building light checker image `{image}` (checkers requirements.txt) ...")
|
||
run_command(["docker", "build", "-t", image, str(build_ctx)])
|
||
finally:
|
||
shutil.rmtree(build_ctx, ignore_errors=True)
|
||
return image
|
||
|
||
|
||
def _run_one_light_smoke_check(image: str, checkers_dir: Path, spec: ServiceSpec) -> Tuple[str, bool, str]:
|
||
command = [
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--add-host",
|
||
"host.docker.internal:host-gateway",
|
||
"-v",
|
||
f"{checkers_dir}:/checkers:ro",
|
||
image,
|
||
"python3",
|
||
f"/checkers/{spec.name}/forcad_wrapper.py",
|
||
"check",
|
||
"host.docker.internal",
|
||
]
|
||
try:
|
||
result = run_command(command, check=False, timeout=_smoke_check_timeout(spec))
|
||
except CommandTimeoutError as exc:
|
||
return spec.name, False, str(exc)
|
||
if result.returncode == 101:
|
||
return spec.name, True, "OK"
|
||
return spec.name, False, f"exit code {result.returncode}"
|
||
|
||
|
||
def run_light_smoke_checks(
|
||
runtime_dir: Path, specs: List[ServiceSpec], max_workers: int = 8
|
||
) -> Dict[str, Tuple[bool, str]]:
|
||
checkers_dir = runtime_dir / "checkers"
|
||
image = _build_light_checkers_image(checkers_dir)
|
||
|
||
info(f"Running {len(specs)} light smoke-check(s) in parallel (max_workers={max_workers}) ...")
|
||
results: Dict[str, Tuple[bool, str]] = {}
|
||
with ThreadPoolExecutor(max_workers=max(1, max_workers)) as pool:
|
||
futures = {
|
||
pool.submit(_run_one_light_smoke_check, image, checkers_dir, spec): spec
|
||
for spec in specs
|
||
}
|
||
for future in as_completed(futures):
|
||
name, ok, detail = future.result()
|
||
results[name] = (ok, detail)
|
||
if ok:
|
||
info(f"[SMOKE] `{name}`: OK")
|
||
else:
|
||
warn(f"[SMOKE] `{name}` failed: {detail}")
|
||
|
||
failed = [name for name, (ok, _) in results.items() if not ok]
|
||
if not failed:
|
||
info("Smoke-checks completed successfully.")
|
||
else:
|
||
warn(f"Smoke-checks failed for: {', '.join(sorted(failed))}")
|
||
return results
|
||
|
||
|
||
def save_state(runtime_dir: Path, payload: dict) -> None:
|
||
state_path = runtime_dir / "state.json"
|
||
state_path.write_text(json.dumps(payload, indent=2))
|
||
|
||
|
||
def load_state(runtime_dir: Path) -> Optional[dict]:
|
||
state_path = runtime_dir / "state.json"
|
||
if not state_path.exists():
|
||
return None
|
||
return json.loads(state_path.read_text())
|
||
|
||
|
||
def load_context_from_args(args: argparse.Namespace, allow_state: bool = False) -> Tuple[Path, Path]:
|
||
runtime_dir = args.runtime_dir.resolve()
|
||
|
||
services_root = args.services_root.resolve() if args.services_root else None
|
||
checkers_root = args.checkers_root.resolve() if args.checkers_root else None
|
||
|
||
if services_root is None and allow_state:
|
||
state = load_state(runtime_dir)
|
||
if state:
|
||
services_root = Path(state["services_root"])
|
||
checkers_root = Path(state["checkers_root"])
|
||
|
||
if services_root is None:
|
||
raise RuntimeError("Нужно указать `--services-root` или иметь сохранённый state от `up`.")
|
||
if checkers_root is None:
|
||
checkers_root = resolve_checkers_root(services_root, None)
|
||
|
||
return services_root, checkers_root
|
||
|
||
|
||
def cmd_validate(args: argparse.Namespace) -> int:
|
||
services_root, checkers_root = load_context_from_args(args, allow_state=False)
|
||
selected = set(args.service) if args.service else None
|
||
specs, discovery_report = discover_services(services_root, checkers_root, selected)
|
||
validation_report = validate_preflight(specs)
|
||
report = merge_reports(discovery_report, validation_report)
|
||
|
||
print_report(report)
|
||
if not report.ok():
|
||
return 2
|
||
|
||
info("Preflight-проверка пройдена.")
|
||
for spec in specs:
|
||
info(
|
||
f"service={spec.name} profile={spec.profile} checker={spec.checker_entry} "
|
||
f"type={spec.checker_type} places={spec.places} port={spec.host_port}"
|
||
)
|
||
return 0
|
||
|
||
|
||
def cmd_up(args: argparse.Namespace) -> int:
|
||
services_root, checkers_root = load_context_from_args(args, allow_state=False)
|
||
selected = set(args.service) if args.service else None
|
||
specs, discovery_report = discover_services(services_root, checkers_root, selected)
|
||
validation_report = validate_preflight(specs)
|
||
report = merge_reports(discovery_report, validation_report)
|
||
print_report(report)
|
||
if not report.ok():
|
||
return 2
|
||
|
||
runtime_dir = args.runtime_dir.resolve()
|
||
runtime_meta = generate_runtime(
|
||
runtime_dir=runtime_dir,
|
||
specs=specs,
|
||
)
|
||
|
||
failure_policy = args.failure_policy.lower()
|
||
strict = failure_policy == "strict"
|
||
ordered_specs = order_services_for_deploy(specs)
|
||
|
||
if args.auto_cleanup:
|
||
auto_cleanup_services(ordered_specs)
|
||
if args.prewarm_k3d:
|
||
prewarm_k3d_images(
|
||
ordered_specs,
|
||
strict=strict,
|
||
image_pull_retries=args.image_pull_retries,
|
||
image_pull_retry_delay=args.image_pull_retry_delay,
|
||
)
|
||
|
||
ready_specs, build_failed = build_checker_artifacts(
|
||
runtime_dir,
|
||
ordered_specs,
|
||
cache_dir=args.cache_dir.resolve(),
|
||
failure_policy=failure_policy,
|
||
)
|
||
if build_failed and strict:
|
||
raise RuntimeError(f"Checker build failed: {build_failed}")
|
||
if build_failed and not strict:
|
||
warn(f"Checker build failures (best-effort): {build_failed}")
|
||
if not ready_specs:
|
||
raise RuntimeError("No services left after checker build stage.")
|
||
|
||
deployed_specs: List[ServiceSpec] = []
|
||
deploy_failed: Dict[str, str] = {}
|
||
for spec in ready_specs:
|
||
try:
|
||
deploy_service_with_retry(
|
||
spec,
|
||
k3d_retries=args.k3d_retries,
|
||
retry_delay=args.k3d_retry_delay,
|
||
image_pull_retries=args.image_pull_retries,
|
||
image_pull_retry_delay=args.image_pull_retry_delay,
|
||
)
|
||
wait_for_service(spec, timeout_seconds=args.service_wait_timeout)
|
||
deployed_specs.append(spec)
|
||
except Exception as exc: # noqa: BLE001
|
||
deploy_failed[spec.name] = str(exc)
|
||
if strict:
|
||
if args.rollback_on_failure:
|
||
rollback_services(deployed_specs + [spec])
|
||
raise
|
||
warn(f"Skipping `{spec.name}` after deploy failure: {exc}")
|
||
|
||
if not deployed_specs:
|
||
raise RuntimeError("No services were deployed successfully.")
|
||
if deploy_failed and not strict:
|
||
warn(f"Deploy failures (best-effort): {deploy_failed}")
|
||
|
||
if args.smoke_start_delay > 0:
|
||
info(f"Waiting {args.smoke_start_delay}s before smoke-check ...")
|
||
time.sleep(args.smoke_start_delay)
|
||
|
||
info("Running service-only mode: ForcAD stack is disabled.")
|
||
run_light_smoke_checks(runtime_dir, deployed_specs, max_workers=args.smoke_workers)
|
||
|
||
state = {
|
||
"services_root": str(services_root),
|
||
"checkers_root": str(checkers_root),
|
||
"runtime": runtime_meta,
|
||
"failure_policy": failure_policy,
|
||
"deploy_failed": deploy_failed,
|
||
"build_failed": build_failed,
|
||
"services": [
|
||
{
|
||
"name": spec.name,
|
||
"profile": spec.profile,
|
||
"service_dir": str(spec.service_dir),
|
||
"checker_dir": str(spec.checker_dir),
|
||
"checker_entry": spec.checker_entry.as_posix(),
|
||
"checker_type": spec.checker_type,
|
||
"places": spec.places,
|
||
"checker_timeout": spec.checker_timeout,
|
||
"protocol": spec.protocol,
|
||
"requirements": list(spec.requirements),
|
||
"checker_build_enabled": spec.checker_build_enabled,
|
||
"checker_build_profile": spec.checker_build_profile,
|
||
"checker_build_command": spec.checker_build_command,
|
||
"checker_build_artifacts": spec.checker_build_artifacts,
|
||
"host_port": spec.host_port,
|
||
"host_transport": spec.host_transport,
|
||
"checker_port_env": spec.checker_port_env,
|
||
"expects_host_with_port": spec.expects_host_with_port,
|
||
"start_cmd": spec.start_cmd,
|
||
"stop_cmd": spec.stop_cmd,
|
||
}
|
||
for spec in deployed_specs
|
||
],
|
||
}
|
||
save_state(runtime_dir, state)
|
||
|
||
info("Service-only mode complete: services are up and checker liveness checks were executed.")
|
||
info("Готово.")
|
||
return 0
|
||
|
||
|
||
def cmd_down(args: argparse.Namespace) -> int:
|
||
runtime_dir = args.runtime_dir.resolve()
|
||
state = load_state(runtime_dir)
|
||
if state is None and not args.services_root:
|
||
err("Не найден state.json. Укажи `--services-root` и `--checkers-root`.")
|
||
return 2
|
||
|
||
if state is not None:
|
||
specs = [
|
||
ServiceSpec(
|
||
name=s["name"],
|
||
profile=s["profile"],
|
||
service_dir=Path(s["service_dir"]),
|
||
checker_dir=Path(s["checker_dir"]),
|
||
checker_entry=Path(s.get("checker_entry", "checker.py")),
|
||
checker_type=s.get("checker_type", "hackerdom"),
|
||
places=int(s.get("places", 1)),
|
||
checker_timeout=int(s.get("checker_timeout", 30)),
|
||
protocol=s.get("protocol", "unknown"),
|
||
requirements=[str(x) for x in s.get("requirements", [])],
|
||
checker_build_enabled=bool(s.get("checker_build_enabled", False)),
|
||
checker_build_profile=s.get("checker_build_profile"),
|
||
checker_build_command=s.get("checker_build_command"),
|
||
checker_build_artifacts=[str(x) for x in s.get("checker_build_artifacts", [])],
|
||
host_port=s.get("host_port"),
|
||
host_transport=str(s.get("host_transport", "tcp")).lower(),
|
||
checker_port_env=s.get("checker_port_env"),
|
||
expects_host_with_port=bool(s.get("expects_host_with_port", False)),
|
||
start_cmd=s.get("start_cmd"),
|
||
stop_cmd=s.get("stop_cmd"),
|
||
)
|
||
for s in state["services"]
|
||
]
|
||
else:
|
||
services_root, checkers_root = load_context_from_args(args, allow_state=False)
|
||
discovered, report = discover_services(services_root, checkers_root, set(args.service) if args.service else None)
|
||
print_report(report)
|
||
if not discovered:
|
||
return 2
|
||
specs = discovered
|
||
|
||
for spec in specs:
|
||
stop_service(spec)
|
||
|
||
info("Остановка завершена.")
|
||
return 0
|
||
|
||
|
||
def cmd_status(args: argparse.Namespace) -> int:
|
||
runtime_dir = args.runtime_dir.resolve()
|
||
state = load_state(runtime_dir)
|
||
if state is None:
|
||
err("Не найден state.json. Сначала выполни `up`.")
|
||
return 2
|
||
|
||
info("Service-only mode state: ForcAD stack is disabled.")
|
||
|
||
for service in state["services"]:
|
||
port = service.get("host_port")
|
||
transport = str(service.get("host_transport", "tcp")).lower()
|
||
if not port:
|
||
warn(f"{service['name']}: порт неизвестен.")
|
||
continue
|
||
if transport == "udp":
|
||
if is_udp_port_bound(int(port)):
|
||
info(f"{service['name']}: UDP 127.0.0.1:{port} открыт")
|
||
else:
|
||
warn(f"{service['name']}: UDP 127.0.0.1:{port} недоступен")
|
||
continue
|
||
with socket.socket() as sock:
|
||
sock.settimeout(1.0)
|
||
try:
|
||
sock.connect(("127.0.0.1", int(port)))
|
||
info(f"{service['name']}: 127.0.0.1:{port} открыт")
|
||
except OSError:
|
||
warn(f"{service['name']}: 127.0.0.1:{port} недоступен")
|
||
return 0
|
||
|
||
|
||
def cmd_check(args: argparse.Namespace) -> int:
|
||
runtime_dir = args.runtime_dir.resolve()
|
||
state = load_state(runtime_dir)
|
||
services_root = args.services_root.resolve() if args.services_root else None
|
||
checkers_root = args.checkers_root.resolve() if args.checkers_root else None
|
||
if state is not None:
|
||
if services_root is None:
|
||
services_root = Path(state["services_root"])
|
||
if checkers_root is None:
|
||
checkers_root = Path(state["checkers_root"])
|
||
|
||
if services_root is None:
|
||
err("Нужно указать `--services-root` или иметь сохранённый state от `up`.")
|
||
return 2
|
||
if checkers_root is None:
|
||
checkers_root = resolve_checkers_root(services_root, None)
|
||
|
||
selected = set(args.service) if args.service else None
|
||
specs, discovery_report = discover_services(services_root, checkers_root, selected)
|
||
validation_report = validate_preflight(specs)
|
||
report = merge_reports(discovery_report, validation_report)
|
||
print_report(report)
|
||
if not report.ok():
|
||
return 2
|
||
|
||
runtime_meta = generate_runtime(
|
||
runtime_dir=runtime_dir,
|
||
specs=specs,
|
||
)
|
||
failure_policy = args.failure_policy.lower()
|
||
ready_specs, build_failed = build_checker_artifacts(
|
||
runtime_dir,
|
||
specs,
|
||
cache_dir=args.cache_dir.resolve(),
|
||
failure_policy=failure_policy,
|
||
)
|
||
if build_failed and failure_policy == "strict":
|
||
raise RuntimeError(f"Checker build failed: {build_failed}")
|
||
if not ready_specs:
|
||
raise RuntimeError("No checkers ready for smoke-check.")
|
||
|
||
if args.smoke_start_delay > 0:
|
||
info(f"Waiting {args.smoke_start_delay}s before smoke-check ...")
|
||
time.sleep(args.smoke_start_delay)
|
||
|
||
run_light_smoke_checks(runtime_dir, ready_specs, max_workers=args.smoke_workers)
|
||
state_payload = {
|
||
"services_root": str(services_root),
|
||
"checkers_root": str(checkers_root),
|
||
"runtime": runtime_meta,
|
||
"failure_policy": failure_policy,
|
||
"deploy_failed": {},
|
||
"build_failed": build_failed,
|
||
"services": [
|
||
{
|
||
"name": spec.name,
|
||
"profile": spec.profile,
|
||
"service_dir": str(spec.service_dir),
|
||
"checker_dir": str(spec.checker_dir),
|
||
"checker_entry": spec.checker_entry.as_posix(),
|
||
"checker_type": spec.checker_type,
|
||
"places": spec.places,
|
||
"checker_timeout": spec.checker_timeout,
|
||
"protocol": spec.protocol,
|
||
"requirements": list(spec.requirements),
|
||
"checker_build_enabled": spec.checker_build_enabled,
|
||
"checker_build_profile": spec.checker_build_profile,
|
||
"checker_build_command": spec.checker_build_command,
|
||
"checker_build_artifacts": spec.checker_build_artifacts,
|
||
"host_port": spec.host_port,
|
||
"host_transport": spec.host_transport,
|
||
"checker_port_env": spec.checker_port_env,
|
||
"expects_host_with_port": spec.expects_host_with_port,
|
||
"start_cmd": spec.start_cmd,
|
||
"stop_cmd": spec.stop_cmd,
|
||
}
|
||
for spec in ready_specs
|
||
],
|
||
}
|
||
save_state(runtime_dir, state_payload)
|
||
info("Проверка чекеров завершена.")
|
||
return 0
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(
|
||
description="Локальный оркестратор сервисов и checker smoke-check без запуска полного ForcAD."
|
||
)
|
||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||
|
||
def add_common_flags(subparser: argparse.ArgumentParser, services_required: bool) -> None:
|
||
subparser.add_argument("--services-root", type=Path, required=services_required)
|
||
subparser.add_argument("--checkers-root", type=Path)
|
||
subparser.add_argument("--runtime-dir", type=Path, default=Path(__file__).resolve().parent / ".runtime")
|
||
subparser.add_argument("--service", action="append", help="Имя сервиса (можно повторять). Если не задано, берутся все.")
|
||
|
||
validate = subparsers.add_parser("validate", help="Проверка структуры сервисов/чекеров без запуска.")
|
||
add_common_flags(validate, services_required=True)
|
||
validate.set_defaults(handler=cmd_validate)
|
||
|
||
up = subparsers.add_parser("up", help="Поднять сервисы и выполнить checker smoke-check.")
|
||
add_common_flags(up, services_required=True)
|
||
up.add_argument("--cache-dir", type=Path, default=Path(__file__).resolve().parent / ".cache")
|
||
up.add_argument("--failure-policy", choices=["strict", "best-effort"], default="strict")
|
||
up.add_argument("--rollback-on-failure", action=argparse.BooleanOptionalAction, default=True)
|
||
up.add_argument("--auto-cleanup", action=argparse.BooleanOptionalAction, default=True)
|
||
up.add_argument("--prewarm-k3d", action=argparse.BooleanOptionalAction, default=True)
|
||
up.add_argument("--k3d-retries", type=int, default=1)
|
||
up.add_argument("--k3d-retry-delay", type=int, default=5)
|
||
up.add_argument(
|
||
"--image-pull-retries",
|
||
type=int,
|
||
default=2,
|
||
help="Additional retries for docker image pulls (compose/prewarm).",
|
||
)
|
||
up.add_argument(
|
||
"--image-pull-retry-delay",
|
||
type=int,
|
||
default=10,
|
||
help="Delay (seconds) between docker pull retries.",
|
||
)
|
||
up.add_argument("--service-wait-timeout", type=int, default=180)
|
||
up.add_argument(
|
||
"--smoke-workers",
|
||
type=int,
|
||
default=8,
|
||
help="Максимум параллельных чекеров при smoke-check.",
|
||
)
|
||
up.add_argument(
|
||
"--smoke-start-delay",
|
||
type=int,
|
||
default=30,
|
||
help="Задержка (сек) перед запуском smoke-check после деплоя.",
|
||
)
|
||
up.set_defaults(handler=cmd_up)
|
||
|
||
down = subparsers.add_parser("down", help="Остановить игровые сервисы.")
|
||
add_common_flags(down, services_required=False)
|
||
down.set_defaults(handler=cmd_down)
|
||
|
||
status = subparsers.add_parser("status", help="Показать статус запущенной инфраструктуры.")
|
||
status.add_argument("--runtime-dir", type=Path, default=Path(__file__).resolve().parent / ".runtime")
|
||
status.set_defaults(handler=cmd_status)
|
||
|
||
check = subparsers.add_parser(
|
||
"check",
|
||
help="Прогнать checker smoke-check по уже поднятым сервисам (использует state или явные roots).",
|
||
)
|
||
add_common_flags(check, services_required=False)
|
||
check.add_argument("--cache-dir", type=Path, default=Path(__file__).resolve().parent / ".cache")
|
||
check.add_argument("--failure-policy", choices=["strict", "best-effort"], default="strict")
|
||
check.add_argument(
|
||
"--smoke-workers",
|
||
type=int,
|
||
default=8,
|
||
help="Максимум параллельных чекеров при smoke-check.",
|
||
)
|
||
check.add_argument(
|
||
"--smoke-start-delay",
|
||
type=int,
|
||
default=30,
|
||
help="Задержка (сек) перед запуском smoke-check.",
|
||
)
|
||
check.set_defaults(handler=cmd_check)
|
||
|
||
return parser
|
||
|
||
|
||
def main() -> int:
|
||
parser = build_parser()
|
||
args = parser.parse_args()
|
||
try:
|
||
return int(args.handler(args))
|
||
except KeyboardInterrupt:
|
||
err("Прервано пользователем.")
|
||
return 130
|
||
except Exception as exc: # noqa: BLE001
|
||
err(str(exc))
|
||
return 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|
||
|