using System.Collections.Concurrent; using System.ComponentModel; using System.Diagnostics; using System.Text; using System.Text.Json; namespace Zcbot.WindowsNode; internal sealed class AdapterProcessRunner(AdapterDescriptor descriptor, JobInboxStore inbox) { private readonly ConcurrentDictionary active = new(); private readonly ConcurrentDictionary cancellations = new(); internal bool HasActiveJobs => !active.IsEmpty; internal string RuntimePath => ResolveCommand(descriptor.EntrypointPath).Filename; internal Task RunAsync(RecoverableJob job) => active.GetOrAdd(job.JobId, _ => RunOnceAsync(job, CancellationFor(job.JobId).Token)); internal void Cancel(Guid jobId) => CancellationFor(jobId).Cancel(); internal AdapterRuntimeStatus Probe() { try { var command = ResolveCommand(descriptor.EntrypointPath); using var process = Start(command, descriptor.DirectoryPath, ["--probe"]); var stdout = process.StandardOutput.ReadToEndAsync(); var stderr = process.StandardError.ReadToEndAsync(); using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); try { process.WaitForExitAsync(timeout.Token).GetAwaiter().GetResult(); } catch (OperationCanceledException) { process.Kill(entireProcessTree: true); return Unavailable("Adapter probe timed out."); } var output = stdout.GetAwaiter().GetResult(); var error = stderr.GetAwaiter().GetResult(); if (process.ExitCode != 0) { return Unavailable( string.IsNullOrWhiteSpace(error) ? "Adapter probe failed." : error.Trim()); } using var document = JsonDocument.Parse(output); var root = document.RootElement; var reportedVersion = root.GetProperty("adapter_version").GetString(); if (!descriptor.Manifest.AdapterVersion.Equals(reportedVersion, StringComparison.Ordinal)) { return Unavailable("Adapter manifest and worker versions do not match."); } return new AdapterRuntimeStatus( root.GetProperty("software").GetString() ?? descriptor.Contract.DisplayName, root.TryGetProperty("software_version", out var softwareVersion) ? softwareVersion.GetString() : null, descriptor.Manifest.AdapterVersion, root.GetProperty("health").GetString() ?? "unavailable", root.GetProperty("detail").GetString() ?? "Adapter probe returned no detail."); } catch (Exception exception) when ( exception is IOException or JsonException or UnauthorizedAccessException or InvalidOperationException or Win32Exception) { return Unavailable(exception.Message); } } private AdapterRuntimeStatus Unavailable(string detail) => new( descriptor.Contract.DisplayName, null, descriptor.Manifest.AdapterVersion, "unavailable", detail[..Math.Min(500, detail.Length)]); private CancellationTokenSource CancellationFor(Guid jobId) => cancellations.GetOrAdd(jobId, _ => new CancellationTokenSource()); internal async Task RunAcceptanceAsync( string workRoot, IProgress progress, CancellationToken cancellationToken) { if (HasActiveJobs) { throw new InvalidOperationException( "Adapter acceptance cannot run while a dispatched job is active."); } var root = Path.GetFullPath(workRoot); var parent = Path.GetDirectoryName(root); if (Directory.Exists(root) || string.IsNullOrWhiteSpace(parent) || !Directory.Exists(parent)) { throw new InvalidDataException( "Acceptance work root must be a new directory under an existing parent."); } var acceptancePath = Path.GetFullPath( Path.Combine(descriptor.DirectoryPath, "acceptance.py")); var fixturePath = Path.GetFullPath( Path.Combine(descriptor.DirectoryPath, "acceptance-coupon.step")); if (!acceptancePath.StartsWith( descriptor.DirectoryPath + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) || !fixturePath.StartsWith( descriptor.DirectoryPath + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) || !File.Exists(acceptancePath) || !File.Exists(fixturePath)) { throw new InvalidDataException("Bundled adapter acceptance fixture is unavailable."); } var command = ResolveCommand(acceptancePath); using var process = Start( command, descriptor.DirectoryPath, ["--work-root", root, "--repeat", "20", "--cancel-after", "10", "--release-wait", "60"]); var stdout = CaptureOutputAsync(process.StandardOutput, progress); var stderr = CaptureOutputAsync(process.StandardError, null); try { await process.WaitForExitAsync(cancellationToken); } catch (OperationCanceledException) { await TerminateProcessTreeAsync(process); throw; } var output = await stdout; var error = await stderr; var reportPath = Path.Combine(root, "acceptance-report.json"); if (!File.Exists(reportPath)) { return new AdapterAcceptanceResult( false, reportPath, string.IsNullOrWhiteSpace(error) ? $"Acceptance exited with code {process.ExitCode}." : error[^Math.Min(error.Length, 1000)..]); } using var report = JsonDocument.Parse(File.ReadAllBytes(reportPath)); var passed = report.RootElement.TryGetProperty("passed", out var passedValue) && passedValue.ValueKind == JsonValueKind.True; var detail = passed ? "Cancellation and 20 consecutive solves passed." : report.RootElement.TryGetProperty("failure", out var failure) ? failure.ToString() : string.IsNullOrWhiteSpace(error) ? output[^Math.Min(output.Length, 1000)..] : error[^Math.Min(error.Length, 1000)..]; return new AdapterAcceptanceResult(passed, reportPath, detail); } private async Task RunOnceAsync(RecoverableJob job, CancellationToken cancellationToken) { try { var paths = NodePaths.ForCurrentMachine(); var jobDirectory = Path.Combine(paths.JobsDirectory, job.JobId.ToString("D")); var terminalPath = Path.Combine(jobDirectory, "terminal.json"); if (File.Exists(terminalPath)) return; var markerPath = Path.Combine(jobDirectory, "worker-started.json"); if (File.Exists(markerPath)) { inbox.WriteTerminal( job, "failed", ErrorCode("NODE_RESTARTED_DURING_JOB"), "The node restarted after adapter execution began and cannot prove the prior worker state."); return; } var command = ResolveCommand(descriptor.EntrypointPath); WriteMarker(markerPath, command); using var process = Start(command, jobDirectory, [jobDirectory]); var stdout = process.StandardOutput.ReadToEndAsync(); var stderr = process.StandardError.ReadToEndAsync(); using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); timeout.CancelAfter(TimeSpan.FromMinutes(descriptor.Manifest.WorkerTimeoutMinutes)); try { await process.WaitForExitAsync(timeout.Token); } catch (OperationCanceledException) { await TerminateProcessTreeAsync(process); if (cancellationToken.IsCancellationRequested) { inbox.WriteTerminal(job, "cancelled", "USER_CANCELLED", "Cancelled by user."); } else { inbox.WriteTerminal( job, "failed", ErrorCode("WORKER_TIMEOUT"), $"Adapter worker exceeded {descriptor.Manifest.WorkerTimeoutMinutes} minutes."); } return; } var output = await stdout; var error = await stderr; WriteDiagnostic(jobDirectory, output, error, process.ExitCode); if (!File.Exists(terminalPath)) { inbox.WriteTerminal( job, "failed", ErrorCode("WORKER_NO_TERMINAL"), $"Adapter worker exited with code {process.ExitCode} without terminal.json."); } } catch (Exception exception) when ( exception is IOException or JsonException or UnauthorizedAccessException or InvalidOperationException or Win32Exception) { inbox.WriteTerminal( job, "failed", ErrorCode("WORKER_START_FAILED"), exception.Message[..Math.Min(500, exception.Message.Length)]); } finally { active.TryRemove(job.JobId, out _); if (cancellations.TryRemove(job.JobId, out var cancellation)) cancellation.Dispose(); } } private string ErrorCode(string suffix) => descriptor.Manifest.Capability.StartsWith("origin.", StringComparison.Ordinal) ? suffix == "NODE_RESTARTED_DURING_JOB" ? suffix : $"ORIGIN_{suffix}" : suffix == "NODE_RESTARTED_DURING_JOB" ? suffix : $"ADAPTER_{suffix}"; private ProcessCommand ResolveCommand(string entrypointPath) { if (descriptor.Manifest.Runtime == "executable") { return new ProcessCommand(entrypointPath, []); } var runtimeId = descriptor.Manifest.RuntimeId!; var environmentName = "ZCBOT_ADAPTER_" + runtimeId.ToUpperInvariant().Replace('-', '_') + "_PYTHON"; var configured = Environment.GetEnvironmentVariable(environmentName); if (string.IsNullOrWhiteSpace(configured) && runtimeId == "origin") { configured = Environment.GetEnvironmentVariable("ZCBOT_ORIGIN_PYTHON"); } var paths = NodePaths.ForCurrentMachine(); var candidate = string.IsNullOrWhiteSpace(configured) ? Path.Combine(paths.RootDirectory, "runtimes", runtimeId, "Scripts", "python.exe") : configured; if (!Path.IsPathFullyQualified(candidate)) { throw new InvalidOperationException("Managed Python interpreter path is not absolute."); } var interpreter = Path.GetFullPath(candidate); if (!File.Exists(interpreter) || !Path.GetFileName(interpreter).Equals("python.exe", StringComparison.OrdinalIgnoreCase)) { throw new InvalidOperationException("Managed Python interpreter is unavailable."); } return new ProcessCommand(interpreter, [entrypointPath]); } private Process Start( ProcessCommand command, string workingDirectory, IReadOnlyList arguments) { var startInfo = new ProcessStartInfo { FileName = command.Filename, WorkingDirectory = workingDirectory, UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = true, RedirectStandardError = true, StandardOutputEncoding = Encoding.UTF8, StandardErrorEncoding = Encoding.UTF8, }; foreach (var argument in command.PrefixArguments.Concat(arguments)) { startInfo.ArgumentList.Add(argument); } if (descriptor.Manifest.Runtime == "python") { startInfo.Environment["PYTHONUTF8"] = "1"; startInfo.Environment["PYTHONIOENCODING"] = "utf-8"; } return Process.Start(startInfo) ?? throw new InvalidOperationException("Adapter worker did not start."); } private void WriteMarker(string path, ProcessCommand command) { var value = JsonSerializer.SerializeToUtf8Bytes(new { started_at = DateTimeOffset.UtcNow, node_pid = Environment.ProcessId, capability = descriptor.Manifest.Capability, adapter_version = descriptor.Manifest.AdapterVersion, runtime = descriptor.Manifest.Runtime, executable = command.Filename, entrypoint = descriptor.EntrypointPath, }); using var stream = new FileStream( path, FileMode.CreateNew, FileAccess.Write, FileShare.None, bufferSize: 4096, FileOptions.WriteThrough); stream.Write(value); stream.Flush(flushToDisk: true); } private static void WriteDiagnostic(string jobDirectory, string output, string error, int exitCode) { var logs = Path.Combine(jobDirectory, "logs"); Directory.CreateDirectory(logs); var value = JsonSerializer.Serialize(new { exit_code = exitCode, stdout = output[..Math.Min(output.Length, 16 * 1024)], stderr = error[..Math.Min(error.Length, 16 * 1024)], }); File.WriteAllText(Path.Combine(logs, "worker-process.json"), value, Encoding.UTF8); } private static async Task TerminateProcessTreeAsync(Process process) { if (process.HasExited) return; process.Kill(entireProcessTree: true); using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); try { await process.WaitForExitAsync(timeout.Token); } catch (OperationCanceledException exception) { throw new InvalidOperationException( "Adapter worker process tree did not terminate within 30 seconds.", exception); } } private static async Task CaptureOutputAsync( StreamReader reader, IProgress? progress) { var output = new StringBuilder(); while (await reader.ReadLineAsync() is { } line) { if (output.Length < 16 * 1024) { output.AppendLine(line); } progress?.Report(line); } return output.ToString(); } private sealed record ProcessCommand(string Filename, IReadOnlyList PrefixArguments); }