260 lines
10 KiB
C#
260 lines
10 KiB
C#
using System.Collections.Concurrent;
|
|
using System.ComponentModel;
|
|
using System.Diagnostics;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
|
|
namespace Zcbot.WindowsNode;
|
|
|
|
internal sealed class AdapterProcessRunner(AdapterDescriptor descriptor, JobInboxStore inbox)
|
|
{
|
|
private static readonly TimeSpan WorkerTimeout = TimeSpan.FromMinutes(30);
|
|
private readonly ConcurrentDictionary<Guid, Task> active = new();
|
|
private readonly ConcurrentDictionary<Guid, CancellationTokenSource> cancellations = new();
|
|
|
|
internal bool HasActiveJobs => !active.IsEmpty;
|
|
|
|
internal Task RunAsync(RecoverableJob job) =>
|
|
active.GetOrAdd(job.JobId, _ => RunOnceAsync(job, CancellationFor(job.JobId).Token));
|
|
|
|
internal void Cancel(Guid jobId) => CancellationFor(jobId).Cancel();
|
|
|
|
internal AdapterRuntimeStatus Probe()
|
|
{
|
|
try
|
|
{
|
|
var command = ResolveCommand();
|
|
using var process = Start(command, descriptor.DirectoryPath, ["--probe"]);
|
|
var stdout = process.StandardOutput.ReadToEndAsync();
|
|
var stderr = process.StandardError.ReadToEndAsync();
|
|
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10));
|
|
try
|
|
{
|
|
process.WaitForExitAsync(timeout.Token).GetAwaiter().GetResult();
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
process.Kill(entireProcessTree: true);
|
|
return Unavailable("Adapter probe timed out.");
|
|
}
|
|
var output = stdout.GetAwaiter().GetResult();
|
|
var error = stderr.GetAwaiter().GetResult();
|
|
if (process.ExitCode != 0)
|
|
{
|
|
return Unavailable(
|
|
string.IsNullOrWhiteSpace(error) ? "Adapter probe failed." : error.Trim());
|
|
}
|
|
using var document = JsonDocument.Parse(output);
|
|
var root = document.RootElement;
|
|
var reportedVersion = root.GetProperty("adapter_version").GetString();
|
|
if (!descriptor.Manifest.AdapterVersion.Equals(reportedVersion, StringComparison.Ordinal))
|
|
{
|
|
return Unavailable("Adapter manifest and worker versions do not match.");
|
|
}
|
|
return new AdapterRuntimeStatus(
|
|
root.GetProperty("software").GetString() ?? descriptor.Contract.DisplayName,
|
|
root.TryGetProperty("software_version", out var softwareVersion)
|
|
? softwareVersion.GetString()
|
|
: null,
|
|
descriptor.Manifest.AdapterVersion,
|
|
root.GetProperty("health").GetString() ?? "unavailable",
|
|
root.GetProperty("detail").GetString() ?? "Adapter probe returned no detail.");
|
|
}
|
|
catch (Exception exception) when (
|
|
exception is IOException
|
|
or JsonException
|
|
or UnauthorizedAccessException
|
|
or InvalidOperationException
|
|
or Win32Exception)
|
|
{
|
|
return Unavailable(exception.Message);
|
|
}
|
|
}
|
|
|
|
private AdapterRuntimeStatus Unavailable(string detail) =>
|
|
new(
|
|
descriptor.Contract.DisplayName,
|
|
null,
|
|
descriptor.Manifest.AdapterVersion,
|
|
"unavailable",
|
|
detail[..Math.Min(500, detail.Length)]);
|
|
|
|
private CancellationTokenSource CancellationFor(Guid jobId) =>
|
|
cancellations.GetOrAdd(jobId, _ => new CancellationTokenSource());
|
|
|
|
private async Task RunOnceAsync(RecoverableJob job, CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
var paths = NodePaths.ForCurrentMachine();
|
|
var jobDirectory = Path.Combine(paths.JobsDirectory, job.JobId.ToString("D"));
|
|
var terminalPath = Path.Combine(jobDirectory, "terminal.json");
|
|
if (File.Exists(terminalPath)) return;
|
|
|
|
var markerPath = Path.Combine(jobDirectory, "worker-started.json");
|
|
if (File.Exists(markerPath))
|
|
{
|
|
inbox.WriteTerminal(
|
|
job,
|
|
"failed",
|
|
ErrorCode("NODE_RESTARTED_DURING_JOB"),
|
|
"The node restarted after adapter execution began and cannot prove the prior worker state.");
|
|
return;
|
|
}
|
|
|
|
var command = ResolveCommand();
|
|
WriteMarker(markerPath, command);
|
|
using var process = Start(command, jobDirectory, [jobDirectory]);
|
|
var stdout = process.StandardOutput.ReadToEndAsync();
|
|
var stderr = process.StandardError.ReadToEndAsync();
|
|
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
|
timeout.CancelAfter(WorkerTimeout);
|
|
try
|
|
{
|
|
await process.WaitForExitAsync(timeout.Token);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
process.Kill(entireProcessTree: true);
|
|
if (cancellationToken.IsCancellationRequested)
|
|
{
|
|
inbox.WriteTerminal(job, "cancelled", "USER_CANCELLED", "Cancelled by user.");
|
|
}
|
|
else
|
|
{
|
|
inbox.WriteTerminal(
|
|
job, "failed", ErrorCode("WORKER_TIMEOUT"),
|
|
"Adapter worker exceeded 30 minutes.");
|
|
}
|
|
return;
|
|
}
|
|
var output = await stdout;
|
|
var error = await stderr;
|
|
WriteDiagnostic(jobDirectory, output, error, process.ExitCode);
|
|
if (!File.Exists(terminalPath))
|
|
{
|
|
inbox.WriteTerminal(
|
|
job,
|
|
"failed",
|
|
ErrorCode("WORKER_NO_TERMINAL"),
|
|
$"Adapter worker exited with code {process.ExitCode} without terminal.json.");
|
|
}
|
|
}
|
|
catch (Exception exception) when (
|
|
exception is IOException
|
|
or JsonException
|
|
or UnauthorizedAccessException
|
|
or InvalidOperationException
|
|
or Win32Exception)
|
|
{
|
|
inbox.WriteTerminal(
|
|
job,
|
|
"failed",
|
|
ErrorCode("WORKER_START_FAILED"),
|
|
exception.Message[..Math.Min(500, exception.Message.Length)]);
|
|
}
|
|
finally
|
|
{
|
|
active.TryRemove(job.JobId, out _);
|
|
if (cancellations.TryRemove(job.JobId, out var cancellation)) cancellation.Dispose();
|
|
}
|
|
}
|
|
|
|
private string ErrorCode(string suffix) =>
|
|
descriptor.Manifest.Capability.StartsWith("origin.", StringComparison.Ordinal)
|
|
? suffix == "NODE_RESTARTED_DURING_JOB" ? suffix : $"ORIGIN_{suffix}"
|
|
: suffix == "NODE_RESTARTED_DURING_JOB" ? suffix : $"ADAPTER_{suffix}";
|
|
|
|
private ProcessCommand ResolveCommand()
|
|
{
|
|
if (descriptor.Manifest.Runtime == "executable")
|
|
{
|
|
return new ProcessCommand(descriptor.EntrypointPath, []);
|
|
}
|
|
var runtimeId = descriptor.Manifest.RuntimeId!;
|
|
var environmentName = "ZCBOT_ADAPTER_"
|
|
+ runtimeId.ToUpperInvariant().Replace('-', '_') + "_PYTHON";
|
|
var configured = Environment.GetEnvironmentVariable(environmentName);
|
|
if (string.IsNullOrWhiteSpace(configured) && runtimeId == "origin")
|
|
{
|
|
configured = Environment.GetEnvironmentVariable("ZCBOT_ORIGIN_PYTHON");
|
|
}
|
|
var paths = NodePaths.ForCurrentMachine();
|
|
var candidate = string.IsNullOrWhiteSpace(configured)
|
|
? Path.Combine(paths.RootDirectory, "runtimes", runtimeId, "Scripts", "python.exe")
|
|
: configured;
|
|
if (!Path.IsPathFullyQualified(candidate))
|
|
{
|
|
throw new InvalidOperationException("Managed Python interpreter path is not absolute.");
|
|
}
|
|
var interpreter = Path.GetFullPath(candidate);
|
|
if (!File.Exists(interpreter)
|
|
|| !Path.GetFileName(interpreter).Equals("python.exe", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
throw new InvalidOperationException("Managed Python interpreter is unavailable.");
|
|
}
|
|
return new ProcessCommand(interpreter, [descriptor.EntrypointPath]);
|
|
}
|
|
|
|
private Process Start(
|
|
ProcessCommand command, string workingDirectory, IReadOnlyList<string> arguments)
|
|
{
|
|
var startInfo = new ProcessStartInfo
|
|
{
|
|
FileName = command.Filename,
|
|
WorkingDirectory = workingDirectory,
|
|
UseShellExecute = false,
|
|
CreateNoWindow = true,
|
|
RedirectStandardOutput = true,
|
|
RedirectStandardError = true,
|
|
StandardOutputEncoding = Encoding.UTF8,
|
|
StandardErrorEncoding = Encoding.UTF8,
|
|
};
|
|
foreach (var argument in command.PrefixArguments.Concat(arguments))
|
|
{
|
|
startInfo.ArgumentList.Add(argument);
|
|
}
|
|
if (descriptor.Manifest.Runtime == "python")
|
|
{
|
|
startInfo.Environment["PYTHONUTF8"] = "1";
|
|
startInfo.Environment["PYTHONIOENCODING"] = "utf-8";
|
|
}
|
|
return Process.Start(startInfo)
|
|
?? throw new InvalidOperationException("Adapter worker did not start.");
|
|
}
|
|
|
|
private void WriteMarker(string path, ProcessCommand command)
|
|
{
|
|
var value = JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
started_at = DateTimeOffset.UtcNow,
|
|
node_pid = Environment.ProcessId,
|
|
capability = descriptor.Manifest.Capability,
|
|
adapter_version = descriptor.Manifest.AdapterVersion,
|
|
runtime = descriptor.Manifest.Runtime,
|
|
executable = command.Filename,
|
|
entrypoint = descriptor.EntrypointPath,
|
|
});
|
|
using var stream = new FileStream(
|
|
path, FileMode.CreateNew, FileAccess.Write, FileShare.None,
|
|
bufferSize: 4096, FileOptions.WriteThrough);
|
|
stream.Write(value);
|
|
stream.Flush(flushToDisk: true);
|
|
}
|
|
|
|
private static void WriteDiagnostic(string jobDirectory, string output, string error, int exitCode)
|
|
{
|
|
var logs = Path.Combine(jobDirectory, "logs");
|
|
Directory.CreateDirectory(logs);
|
|
var value = JsonSerializer.Serialize(new
|
|
{
|
|
exit_code = exitCode,
|
|
stdout = output[..Math.Min(output.Length, 16 * 1024)],
|
|
stderr = error[..Math.Min(error.Length, 16 * 1024)],
|
|
});
|
|
File.WriteAllText(Path.Combine(logs, "worker-process.json"), value, Encoding.UTF8);
|
|
}
|
|
|
|
private sealed record ProcessCommand(string Filename, IReadOnlyList<string> PrefixArguments);
|
|
}
|