368 lines
15 KiB
C#
368 lines
15 KiB
C#
using System.Collections.Concurrent;
|
|
using System.ComponentModel;
|
|
using System.Diagnostics;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
|
|
namespace Zcbot.WindowsNode;
|
|
|
|
internal sealed class AdapterProcessRunner(AdapterDescriptor descriptor, JobInboxStore inbox)
|
|
{
|
|
private readonly ConcurrentDictionary<Guid, Task> active = new();
|
|
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> cancellations = new();
|
|
|
|
internal bool HasActiveJobs => !active.IsEmpty;
|
|
internal string RuntimePath => ResolveCommand(descriptor.EntrypointPath).Filename;
|
|
|
|
internal Task RunAsync(RecoverableJob job) =>
|
|
active.GetOrAdd(job.JobId, _ => RunOnceAsync(job, CancellationFor(job.JobId).Token));
|
|
|
|
internal void Cancel(Guid jobId) => CancellationFor(jobId).Cancel();
|
|
|
|
internal AdapterRuntimeStatus Probe()
|
|
{
|
|
try
|
|
{
|
|
var command = ResolveCommand(descriptor.EntrypointPath);
|
|
using var process = Start(command, descriptor.DirectoryPath, ["--probe"]);
|
|
var stdout = process.StandardOutput.ReadToEndAsync();
|
|
var stderr = process.StandardError.ReadToEndAsync();
|
|
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10));
|
|
try
|
|
{
|
|
process.WaitForExitAsync(timeout.Token).GetAwaiter().GetResult();
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
process.Kill(entireProcessTree: true);
|
|
return Unavailable("Adapter probe timed out.");
|
|
}
|
|
var output = stdout.GetAwaiter().GetResult();
|
|
var error = stderr.GetAwaiter().GetResult();
|
|
if (process.ExitCode != 0)
|
|
{
|
|
return Unavailable(
|
|
string.IsNullOrWhiteSpace(error) ? "Adapter probe failed." : error.Trim());
|
|
}
|
|
using var document = JsonDocument.Parse(output);
|
|
var root = document.RootElement;
|
|
var reportedVersion = root.GetProperty("adapter_version").GetString();
|
|
if (!descriptor.Manifest.AdapterVersion.Equals(reportedVersion, StringComparison.Ordinal))
|
|
{
|
|
return Unavailable("Adapter manifest and worker versions do not match.");
|
|
}
|
|
return new AdapterRuntimeStatus(
|
|
root.GetProperty("software").GetString() ?? descriptor.Contract.DisplayName,
|
|
root.TryGetProperty("software_version", out var softwareVersion)
|
|
? softwareVersion.GetString()
|
|
: null,
|
|
descriptor.Manifest.AdapterVersion,
|
|
root.GetProperty("health").GetString() ?? "unavailable",
|
|
root.GetProperty("detail").GetString() ?? "Adapter probe returned no detail.");
|
|
}
|
|
catch (Exception exception) when (
|
|
exception is IOException
|
|
or JsonException
|
|
or UnauthorizedAccessException
|
|
or InvalidOperationException
|
|
or Win32Exception)
|
|
{
|
|
return Unavailable(exception.Message);
|
|
}
|
|
}
|
|
|
|
private AdapterRuntimeStatus Unavailable(string detail) =>
|
|
new(
|
|
descriptor.Contract.DisplayName,
|
|
null,
|
|
descriptor.Manifest.AdapterVersion,
|
|
"unavailable",
|
|
detail[..Math.Min(500, detail.Length)]);
|
|
|
|
private CancellationTokenSource CancellationFor(Guid jobId) =>
|
|
cancellations.GetOrAdd(jobId, _ => new CancellationTokenSource());
|
|
|
|
internal async Task<AdapterAcceptanceResult> RunAcceptanceAsync(
|
|
string workRoot,
|
|
IProgress<string> progress,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
if (HasActiveJobs)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"Adapter acceptance cannot run while a dispatched job is active.");
|
|
}
|
|
var root = Path.GetFullPath(workRoot);
|
|
var parent = Path.GetDirectoryName(root);
|
|
if (Directory.Exists(root)
|
|
|| string.IsNullOrWhiteSpace(parent)
|
|
|| !Directory.Exists(parent))
|
|
{
|
|
throw new InvalidDataException(
|
|
"Acceptance work root must be a new directory under an existing parent.");
|
|
}
|
|
var acceptancePath = Path.GetFullPath(
|
|
Path.Combine(descriptor.DirectoryPath, "acceptance.py"));
|
|
var fixturePath = Path.GetFullPath(
|
|
Path.Combine(descriptor.DirectoryPath, "acceptance-coupon.step"));
|
|
if (!acceptancePath.StartsWith(
|
|
descriptor.DirectoryPath + Path.DirectorySeparatorChar,
|
|
StringComparison.OrdinalIgnoreCase)
|
|
|| !fixturePath.StartsWith(
|
|
descriptor.DirectoryPath + Path.DirectorySeparatorChar,
|
|
StringComparison.OrdinalIgnoreCase)
|
|
|| !File.Exists(acceptancePath)
|
|
|| !File.Exists(fixturePath))
|
|
{
|
|
throw new InvalidDataException("Bundled adapter acceptance fixture is unavailable.");
|
|
}
|
|
|
|
var command = ResolveCommand(acceptancePath);
|
|
using var process = Start(
|
|
command,
|
|
descriptor.DirectoryPath,
|
|
["--work-root", root, "--repeat", "20",
|
|
"--cancel-after", "10", "--release-wait", "60"]);
|
|
var stdout = CaptureOutputAsync(process.StandardOutput, progress);
|
|
var stderr = CaptureOutputAsync(process.StandardError, null);
|
|
try
|
|
{
|
|
await process.WaitForExitAsync(cancellationToken);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
await TerminateProcessTreeAsync(process);
|
|
throw;
|
|
}
|
|
var output = await stdout;
|
|
var error = await stderr;
|
|
var reportPath = Path.Combine(root, "acceptance-report.json");
|
|
if (!File.Exists(reportPath))
|
|
{
|
|
return new AdapterAcceptanceResult(
|
|
false,
|
|
reportPath,
|
|
string.IsNullOrWhiteSpace(error)
|
|
? $"Acceptance exited with code {process.ExitCode}."
|
|
: error[^Math.Min(error.Length, 1000)..]);
|
|
}
|
|
using var report = JsonDocument.Parse(File.ReadAllBytes(reportPath));
|
|
var passed = report.RootElement.TryGetProperty("passed", out var passedValue)
|
|
&& passedValue.ValueKind == JsonValueKind.True;
|
|
var detail = passed
|
|
? "Cancellation and 20 consecutive solves passed."
|
|
: report.RootElement.TryGetProperty("failure", out var failure)
|
|
? failure.ToString()
|
|
: string.IsNullOrWhiteSpace(error)
|
|
? output[^Math.Min(output.Length, 1000)..]
|
|
: error[^Math.Min(error.Length, 1000)..];
|
|
return new AdapterAcceptanceResult(passed, reportPath, detail);
|
|
}
|
|
|
|
private async Task RunOnceAsync(RecoverableJob job, CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
var paths = NodePaths.ForCurrentMachine();
|
|
var jobDirectory = Path.Combine(paths.JobsDirectory, job.JobId.ToString("D"));
|
|
var terminalPath = Path.Combine(jobDirectory, "terminal.json");
|
|
if (File.Exists(terminalPath)) return;
|
|
|
|
var markerPath = Path.Combine(jobDirectory, "worker-started.json");
|
|
if (File.Exists(markerPath))
|
|
{
|
|
inbox.WriteTerminal(
|
|
job,
|
|
"failed",
|
|
ErrorCode("NODE_RESTARTED_DURING_JOB"),
|
|
"The node restarted after adapter execution began and cannot prove the prior worker state.");
|
|
return;
|
|
}
|
|
|
|
var command = ResolveCommand(descriptor.EntrypointPath);
|
|
WriteMarker(markerPath, command);
|
|
using var process = Start(command, jobDirectory, [jobDirectory]);
|
|
var stdout = process.StandardOutput.ReadToEndAsync();
|
|
var stderr = process.StandardError.ReadToEndAsync();
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
|
timeout.CancelAfter(TimeSpan.FromMinutes(descriptor.Manifest.WorkerTimeoutMinutes));
|
|
try
|
|
{
|
|
await process.WaitForExitAsync(timeout.Token);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
await TerminateProcessTreeAsync(process);
|
|
if (cancellationToken.IsCancellationRequested)
|
|
{
|
|
inbox.WriteTerminal(job, "cancelled", "USER_CANCELLED", "Cancelled by user.");
|
|
}
|
|
else
|
|
{
|
|
inbox.WriteTerminal(
|
|
job, "failed", ErrorCode("WORKER_TIMEOUT"),
|
|
$"Adapter worker exceeded {descriptor.Manifest.WorkerTimeoutMinutes} minutes.");
|
|
}
|
|
return;
|
|
}
|
|
var output = await stdout;
|
|
var error = await stderr;
|
|
WriteDiagnostic(jobDirectory, output, error, process.ExitCode);
|
|
if (!File.Exists(terminalPath))
|
|
{
|
|
inbox.WriteTerminal(
|
|
job,
|
|
"failed",
|
|
ErrorCode("WORKER_NO_TERMINAL"),
|
|
$"Adapter worker exited with code {process.ExitCode} without terminal.json.");
|
|
}
|
|
}
|
|
catch (Exception exception) when (
|
|
exception is IOException
|
|
or JsonException
|
|
or UnauthorizedAccessException
|
|
or InvalidOperationException
|
|
or Win32Exception)
|
|
{
|
|
inbox.WriteTerminal(
|
|
job,
|
|
"failed",
|
|
ErrorCode("WORKER_START_FAILED"),
|
|
exception.Message[..Math.Min(500, exception.Message.Length)]);
|
|
}
|
|
finally
|
|
{
|
|
active.TryRemove(job.JobId, out _);
|
|
if (cancellations.TryRemove(job.JobId, out var cancellation)) cancellation.Dispose();
|
|
}
|
|
}
|
|
|
|
private string ErrorCode(string suffix) =>
|
|
descriptor.Manifest.Capability.StartsWith("origin.", StringComparison.Ordinal)
|
|
? suffix == "NODE_RESTARTED_DURING_JOB" ? suffix : $"ORIGIN_{suffix}"
|
|
: suffix == "NODE_RESTARTED_DURING_JOB" ? suffix : $"ADAPTER_{suffix}";
|
|
|
|
private ProcessCommand ResolveCommand(string entrypointPath)
|
|
{
|
|
if (descriptor.Manifest.Runtime == "executable")
|
|
{
|
|
return new ProcessCommand(entrypointPath, []);
|
|
}
|
|
var runtimeId = descriptor.Manifest.RuntimeId!;
|
|
var environmentName = "ZCBOT_ADAPTER_"
|
|
+ runtimeId.ToUpperInvariant().Replace('-', '_') + "_PYTHON";
|
|
var configured = Environment.GetEnvironmentVariable(environmentName);
|
|
if (string.IsNullOrWhiteSpace(configured) && runtimeId == "origin")
|
|
{
|
|
configured = Environment.GetEnvironmentVariable("ZCBOT_ORIGIN_PYTHON");
|
|
}
|
|
var paths = NodePaths.ForCurrentMachine();
|
|
var candidate = string.IsNullOrWhiteSpace(configured)
|
|
? Path.Combine(paths.RootDirectory, "runtimes", runtimeId, "Scripts", "python.exe")
|
|
: configured;
|
|
if (!Path.IsPathFullyQualified(candidate))
|
|
{
|
|
throw new InvalidOperationException("Managed Python interpreter path is not absolute.");
|
|
}
|
|
var interpreter = Path.GetFullPath(candidate);
|
|
if (!File.Exists(interpreter)
|
|
|| !Path.GetFileName(interpreter).Equals("python.exe", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
throw new InvalidOperationException("Managed Python interpreter is unavailable.");
|
|
}
|
|
return new ProcessCommand(interpreter, [entrypointPath]);
|
|
}
|
|
|
|
private Process Start(
|
|
ProcessCommand command, string workingDirectory, IReadOnlyList<string> arguments)
|
|
{
|
|
var startInfo = new ProcessStartInfo
|
|
{
|
|
FileName = command.Filename,
|
|
WorkingDirectory = workingDirectory,
|
|
UseShellExecute = false,
|
|
CreateNoWindow = true,
|
|
RedirectStandardOutput = true,
|
|
RedirectStandardError = true,
|
|
StandardOutputEncoding = Encoding.UTF8,
|
|
StandardErrorEncoding = Encoding.UTF8,
|
|
};
|
|
foreach (var argument in command.PrefixArguments.Concat(arguments))
|
|
{
|
|
startInfo.ArgumentList.Add(argument);
|
|
}
|
|
if (descriptor.Manifest.Runtime == "python")
|
|
{
|
|
startInfo.Environment["PYTHONUTF8"] = "1";
|
|
startInfo.Environment["PYTHONIOENCODING"] = "utf-8";
|
|
}
|
|
return Process.Start(startInfo)
|
|
?? throw new InvalidOperationException("Adapter worker did not start.");
|
|
}
|
|
|
|
private void WriteMarker(string path, ProcessCommand command)
|
|
{
|
|
var value = JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
started_at = DateTimeOffset.UtcNow,
|
|
node_pid = Environment.ProcessId,
|
|
capability = descriptor.Manifest.Capability,
|
|
adapter_version = descriptor.Manifest.AdapterVersion,
|
|
runtime = descriptor.Manifest.Runtime,
|
|
executable = command.Filename,
|
|
entrypoint = descriptor.EntrypointPath,
|
|
});
|
|
using var stream = new FileStream(
|
|
path, FileMode.CreateNew, FileAccess.Write, FileShare.None,
|
|
bufferSize: 4096, FileOptions.WriteThrough);
|
|
stream.Write(value);
|
|
stream.Flush(flushToDisk: true);
|
|
}
|
|
|
|
private static void WriteDiagnostic(string jobDirectory, string output, string error, int exitCode)
|
|
{
|
|
var logs = Path.Combine(jobDirectory, "logs");
|
|
Directory.CreateDirectory(logs);
|
|
var value = JsonSerializer.Serialize(new
|
|
{
|
|
exit_code = exitCode,
|
|
stdout = output[..Math.Min(output.Length, 16 * 1024)],
|
|
stderr = error[..Math.Min(error.Length, 16 * 1024)],
|
|
});
|
|
File.WriteAllText(Path.Combine(logs, "worker-process.json"), value, Encoding.UTF8);
|
|
}
|
|
|
|
private static async Task TerminateProcessTreeAsync(Process process)
|
|
{
|
|
if (process.HasExited) return;
|
|
process.Kill(entireProcessTree: true);
|
|
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
|
try
|
|
{
|
|
await process.WaitForExitAsync(timeout.Token);
|
|
}
|
|
catch (OperationCanceledException exception)
|
|
{
|
|
throw new InvalidOperationException(
|
|
"Adapter worker process tree did not terminate within 30 seconds.", exception);
|
|
}
|
|
}
|
|
|
|
private static async Task<string> CaptureOutputAsync(
|
|
StreamReader reader, IProgress<string>? progress)
|
|
{
|
|
var output = new StringBuilder();
|
|
while (await reader.ReadLineAsync() is { } line)
|
|
{
|
|
if (output.Length < 16 * 1024)
|
|
{
|
|
output.AppendLine(line);
|
|
}
|
|
progress?.Report(line);
|
|
}
|
|
return output.ToString();
|
|
}
|
|
|
|
private sealed record ProcessCommand(string Filename, IReadOnlyList<string> PrefixArguments);
|
|
}
|