feat(S2-b): connect durable MQTT sync to capture workflow
This commit is contained in:
25
src/FieldLogger.Sync/BackoffPolicy.cs
Normal file
25
src/FieldLogger.Sync/BackoffPolicy.cs
Normal file
@@ -0,0 +1,25 @@
|
||||
namespace FieldLogger.Sync;
|
||||
|
||||
/// <summary>Exponential retry backoff with full jitter and a hard cap.</summary>
|
||||
public sealed class BackoffPolicy
|
||||
{
|
||||
private readonly TimeSpan _base;
|
||||
private readonly TimeSpan _cap;
|
||||
private readonly Random _random;
|
||||
|
||||
public BackoffPolicy(TimeSpan baseDelay, TimeSpan cap, Random? random = null)
|
||||
{
|
||||
if (baseDelay <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(baseDelay));
|
||||
if (cap < baseDelay) throw new ArgumentOutOfRangeException(nameof(cap));
|
||||
_base = baseDelay;
|
||||
_cap = cap;
|
||||
_random = random ?? Random.Shared;
|
||||
}
|
||||
|
||||
public TimeSpan NextDelay(int attempt)
|
||||
{
|
||||
int exponent = Math.Min(Math.Max(1, attempt) - 1, 30);
|
||||
double ceilingMs = Math.Min(_cap.TotalMilliseconds, _base.TotalMilliseconds * Math.Pow(2, exponent));
|
||||
return TimeSpan.FromMilliseconds(_random.NextDouble() * ceilingMs);
|
||||
}
|
||||
}
|
||||
27
src/FieldLogger.Sync/FieldLogger.Sync.csproj
Normal file
27
src/FieldLogger.Sync/FieldLogger.Sync.csproj
Normal file
@@ -0,0 +1,27 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<!--
|
||||
Cloud-sync core for Field Logger: the durable outbound queue, the Point wire mapping,
|
||||
and the MQTTS publish/ack engine (SRS-SYN-2/5/7, §3.4.2, telemetry-schema.md).
|
||||
Deliberately a plain net9.0 library (NO MAUI head) so the queue-durability, ack-release,
|
||||
and exactly-once behaviour run headless in the QA gate without the Android/iOS workloads.
|
||||
The MAUI app references this and supplies a DB path + broker config; tests supply a fake
|
||||
transport and a temp-file store.
|
||||
-->
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net9.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<LangVersion>latest</LangVersion>
|
||||
<RootNamespace>FieldLogger.Sync</RootNamespace>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!-- Real broker transport. Tests do not touch this (they use a fake IMqttTransport). -->
|
||||
<PackageReference Include="MQTTnet" Version="4.3.7.1207" />
|
||||
<PackageReference Include="sqlite-net-pcl" Version="1.9.172" />
|
||||
<PackageReference Include="SQLitePCLRaw.bundle_green" Version="2.1.10" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
56
src/FieldLogger.Sync/IMqttTransport.cs
Normal file
56
src/FieldLogger.Sync/IMqttTransport.cs
Normal file
@@ -0,0 +1,56 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace FieldLogger.Sync;
|
||||
|
||||
/// <summary>
|
||||
/// Minimal MQTTS transport the sync engine drives. Abstracted so the engine's
|
||||
/// durability/ack/backoff behaviour is tested headless with a fake, while the app uses the
|
||||
/// MQTTnet-backed implementation against the real broker (§3.4.2).
|
||||
/// </summary>
|
||||
public interface IMqttTransport
|
||||
{
|
||||
bool IsConnected { get; }
|
||||
|
||||
/// <summary>Connect (MQTTS/TLS) and subscribe to the ack topic. Throws on failure.</summary>
|
||||
Task ConnectAsync(string ackTopic, CancellationToken ct = default);
|
||||
|
||||
Task DisconnectAsync();
|
||||
|
||||
/// <summary>Publish a payload at QoS 1 (durable). Completes on broker PUBACK; throws on failure.</summary>
|
||||
Task PublishAsync(string topic, byte[] payload, string schemaVersion, CancellationToken ct = default);
|
||||
|
||||
/// <summary>Raised when an application-level ack payload arrives on the ack topic.</summary>
|
||||
event Func<AckBatch, Task>? AckReceived;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Application-level ack payload on ul/{orgId}/app/{clientId}/ack (SRS §3.4.2 "…/ack:
|
||||
/// UUIDs accepted/rejected + reason"). Distinct from the broker PUBACK — the queue releases a
|
||||
/// record only on the application ack referencing its pointId.
|
||||
/// </summary>
|
||||
public sealed record AckBatch
|
||||
{
|
||||
[JsonPropertyName("schemaVersion")] public string SchemaVersion { get; init; } = "1";
|
||||
[JsonPropertyName("results")] public List<AckItem> Results { get; init; } = new();
|
||||
|
||||
private static readonly JsonSerializerOptions Opts = new()
|
||||
{
|
||||
Converters = { new JsonStringEnumConverter() },
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
public static AckBatch FromJsonUtf8(ReadOnlySpan<byte> utf8) =>
|
||||
JsonSerializer.Deserialize<AckBatch>(utf8, Opts) ?? throw new JsonException("null AckBatch");
|
||||
public byte[] ToJsonUtf8() => JsonSerializer.SerializeToUtf8Bytes(this, Opts);
|
||||
}
|
||||
|
||||
public enum AckOutcome { ACCEPTED, DUPLICATE, REJECTED }
|
||||
|
||||
public sealed record AckItem
|
||||
{
|
||||
[JsonPropertyName("pointId")] public required string PointId { get; init; }
|
||||
[JsonPropertyName("outcome")] public AckOutcome Outcome { get; init; }
|
||||
/// <summary>Machine code when REJECTED (api.yaml Error catalog); null on ACCEPTED.</summary>
|
||||
[JsonPropertyName("reasonCode")] public string? ReasonCode { get; init; }
|
||||
}
|
||||
134
src/FieldLogger.Sync/MqttSyncEngine.cs
Normal file
134
src/FieldLogger.Sync/MqttSyncEngine.cs
Normal file
@@ -0,0 +1,134 @@
|
||||
namespace FieldLogger.Sync;
|
||||
|
||||
/// <summary>Tunables for the publish/retry/ack loop.</summary>
|
||||
public sealed record SyncOptions
|
||||
{
|
||||
/// <summary>ul/{orgId}/app/{clientId} — the app's own namespace root. The engine only ever
|
||||
/// publishes/subscribes under this prefix (namespace confinement; SRS §3.4.2 / §10.2).</summary>
|
||||
public required string OrgId { get; init; }
|
||||
public required string ClientId { get; init; }
|
||||
|
||||
public string PointsTopic => $"ul/{OrgId}/app/{ClientId}/log/points";
|
||||
public string AckTopic => $"ul/{OrgId}/app/{ClientId}/ack";
|
||||
|
||||
/// <summary>How long to wait for the application-level ack before republishing (idempotent
|
||||
/// via UUID; cloud dedups). Not a failure — release still requires the ack.</summary>
|
||||
public TimeSpan AckWindow { get; init; } = TimeSpan.FromSeconds(10);
|
||||
/// <summary>Backoff on broker/publish failure: base, doubled per attempt, capped.</summary>
|
||||
public TimeSpan BackoffBase { get; init; } = TimeSpan.FromSeconds(1);
|
||||
public TimeSpan BackoffCap { get; init; } = TimeSpan.FromSeconds(60);
|
||||
public int DrainBatch { get; init; } = 50;
|
||||
|
||||
/// <summary>Reason codes that make a REJECTED ack terminal (drop from queue + surface for
|
||||
/// LOG-7) rather than retryable. Everything else is retried with backoff. Default: schema /
|
||||
/// validation / authorization failures — resending won't help.</summary>
|
||||
public IReadOnlySet<string> TerminalRejectCodes { get; init; } = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"VALIDATION_ERROR", "POINT_ID_NOT_UUIDV7", "ENVELOPE_VALIDATION_ERROR",
|
||||
"UNKNOWN_JOB_OR_WRONG_ORG", "SCHEMA_INVALID", "FORBIDDEN", "UNAUTHENTICATED",
|
||||
"NOT_FOUND", "PAYLOAD_TOO_LARGE",
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drives the durable outbound queue to the broker and releases records only on the cloud's
|
||||
/// application-level ack (SRS-SYN-2/7, §3.4.2). Publish is idempotent via the UUIDv7 pointId,
|
||||
/// so an unacked record is safely republished after the ack window and the cloud de-duplicates.
|
||||
/// Broker loss never loses a capture — the record stays durably queued (SRS-SYN-1). Transport-
|
||||
/// and store-agnostic so it runs headless in tests.
|
||||
/// </summary>
|
||||
public sealed class MqttSyncEngine
|
||||
{
|
||||
private readonly IMqttTransport _transport;
|
||||
private readonly IOutboundStore _store;
|
||||
private readonly SyncOptions _opts;
|
||||
private readonly Func<DateTimeOffset> _now;
|
||||
private readonly BackoffPolicy _backoff;
|
||||
|
||||
/// <summary>Raised when the cloud terminally rejects a record (LOG-7 surface: never silent).</summary>
|
||||
public event Func<OutboundMessage, Task>? PointRejected;
|
||||
/// <summary>Raised when a record is accepted and released from the queue.</summary>
|
||||
public event Func<string, Task>? PointAccepted;
|
||||
|
||||
public MqttSyncEngine(IMqttTransport transport, IOutboundStore store, SyncOptions opts,
|
||||
Func<DateTimeOffset>? now = null, BackoffPolicy? backoff = null)
|
||||
{
|
||||
_transport = transport;
|
||||
_store = store;
|
||||
_opts = opts;
|
||||
_now = now ?? (() => DateTimeOffset.UtcNow);
|
||||
_backoff = backoff ?? new BackoffPolicy(opts.BackoffBase, opts.BackoffCap);
|
||||
_transport.AckReceived += ApplyAckAsync;
|
||||
}
|
||||
|
||||
/// <summary>Connect + subscribe to the ack topic. Safe to call when a broker is reachable.</summary>
|
||||
public Task ConnectAsync(CancellationToken ct = default) => _transport.ConnectAsync(_opts.AckTopic, ct);
|
||||
|
||||
/// <summary>
|
||||
/// Publish every ready record once. Returns the number published this pass. On publish
|
||||
/// failure (e.g. broker down) the record is rescheduled with backoff and kept — no capture
|
||||
/// is lost. Records published successfully stay queued until their app-level ack arrives.
|
||||
/// </summary>
|
||||
public async Task<int> DrainOnceAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (!_transport.IsConnected)
|
||||
return 0; // broker unreachable → leave everything durably queued (SRS-SYN-1)
|
||||
|
||||
var ready = await _store.DequeueReadyAsync(_now(), _opts.DrainBatch);
|
||||
int published = 0;
|
||||
foreach (var m in ready)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
await _transport.PublishAsync(m.Topic, System.Text.Encoding.UTF8.GetBytes(m.PayloadJson),
|
||||
m.SchemaVersion, ct);
|
||||
// Published (broker PUBACK). Do NOT release — wait for the application ack.
|
||||
// Reschedule a republish after the ack window in case the ack is lost.
|
||||
await _store.TouchAwaitingAckAsync(m.PointId, _now() + _opts.AckWindow);
|
||||
published++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
await _store.RescheduleAsync(m.PointId, _now() + _backoff.NextDelay(m.AttemptCount + 1), ex.Message);
|
||||
}
|
||||
}
|
||||
return published;
|
||||
}
|
||||
|
||||
private async Task ApplyAckAsync(AckBatch batch)
|
||||
{
|
||||
foreach (var item in batch.Results)
|
||||
{
|
||||
if (item.Outcome is AckOutcome.ACCEPTED or AckOutcome.DUPLICATE)
|
||||
{
|
||||
await _store.ReleaseAckedAsync(item.PointId); // release only on ACCEPTED
|
||||
if (PointAccepted is { } acceptedHandlers)
|
||||
{
|
||||
foreach (Func<string, Task> handler in acceptedHandlers.GetInvocationList())
|
||||
await handler(item.PointId);
|
||||
}
|
||||
}
|
||||
else // REJECTED
|
||||
{
|
||||
var code = item.ReasonCode ?? "REJECTED";
|
||||
if (_opts.TerminalRejectCodes.Contains(code))
|
||||
{
|
||||
await _store.MarkRejectedAsync(item.PointId, code); // LOG-7: surface, never silent
|
||||
var rejected = (await _store.GetRejectedAsync()).FirstOrDefault(r => r.PointId == item.PointId);
|
||||
if (rejected is not null && PointRejected is { } rejectedHandlers)
|
||||
{
|
||||
foreach (Func<OutboundMessage, Task> handler in rejectedHandlers.GetInvocationList())
|
||||
await handler(rejected);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Retryable rejection — reschedule immediately-ish with backoff.
|
||||
await _store.RescheduleAsync(item.PointId, _now() + _opts.BackoffBase, code);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
112
src/FieldLogger.Sync/MqttnetTransport.cs
Normal file
112
src/FieldLogger.Sync/MqttnetTransport.cs
Normal file
@@ -0,0 +1,112 @@
|
||||
using System.Text;
|
||||
using MQTTnet;
|
||||
using MQTTnet.Client;
|
||||
using MQTTnet.Protocol;
|
||||
|
||||
namespace FieldLogger.Sync;
|
||||
|
||||
/// <summary>Interim app→broker connection config (SRS §3.4.2). MQTTS/TLS only; namespace-scoped
|
||||
/// credential. OIDC-derived tokens (SRS-SYN-3) are deferred to the auth sprint — see decisions.md.
|
||||
/// Reviewed by `security` (S2-sec).</summary>
|
||||
public sealed record MqttBrokerConfig
|
||||
{
|
||||
public required string Host { get; init; }
|
||||
public int Port { get; init; } = 8884;
|
||||
public bool UseTls { get; init; } = true;
|
||||
public required string ClientId { get; init; }
|
||||
/// <summary>Interim credential (username = orgId, password = scoped per-org secret).
|
||||
/// Supplied at runtime from secure storage / config — never hard-coded.</summary>
|
||||
public string? Username { get; init; }
|
||||
public string? Password { get; init; }
|
||||
public TimeSpan SessionExpiry { get; init; } = TimeSpan.FromHours(24);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// MQTTnet-backed <see cref="IMqttTransport"/> against the real UlHub broker. QoS 1 + persistent
|
||||
/// session so durable records survive reconnect; TLS mandatory. Ack payloads on the subscribed
|
||||
/// ack topic are surfaced via <see cref="AckReceived"/>. Not exercised by the headless tests
|
||||
/// (they use a fake transport) — this is the production path.
|
||||
/// </summary>
|
||||
public sealed class MqttnetTransport : IMqttTransport, IAsyncDisposable
|
||||
{
|
||||
private readonly MqttBrokerConfig _config;
|
||||
private readonly IMqttClient _client;
|
||||
private string? _ackTopic;
|
||||
|
||||
public event Func<AckBatch, Task>? AckReceived;
|
||||
public bool IsConnected => _client.IsConnected;
|
||||
|
||||
public MqttnetTransport(MqttBrokerConfig config)
|
||||
{
|
||||
_config = config;
|
||||
_client = new MqttFactory().CreateMqttClient();
|
||||
_client.ApplicationMessageReceivedAsync += OnMessageAsync;
|
||||
}
|
||||
|
||||
public async Task ConnectAsync(string ackTopic, CancellationToken ct = default)
|
||||
{
|
||||
if (!_config.UseTls)
|
||||
throw new InvalidOperationException("The app MQTT transport requires TLS.");
|
||||
if (string.IsNullOrWhiteSpace(_config.Username) || string.IsNullOrWhiteSpace(_config.Password))
|
||||
throw new InvalidOperationException("A scoped app MQTT username and password are required.");
|
||||
|
||||
_ackTopic = ackTopic;
|
||||
|
||||
var options = new MqttClientOptionsBuilder()
|
||||
.WithTcpServer(_config.Host, _config.Port)
|
||||
.WithClientId(_config.ClientId)
|
||||
.WithProtocolVersion(MQTTnet.Formatter.MqttProtocolVersion.V500)
|
||||
.WithCleanSession(false) // persistent session (durable)
|
||||
.WithSessionExpiryInterval((uint)_config.SessionExpiry.TotalSeconds)
|
||||
.WithTlsOptions(o => o.UseTls(_config.UseTls))
|
||||
.WithCredentials(_config.Username, _config.Password)
|
||||
.Build();
|
||||
|
||||
await _client.ConnectAsync(options, ct);
|
||||
await _client.SubscribeAsync(ackTopic, MqttQualityOfServiceLevel.AtLeastOnce, ct);
|
||||
}
|
||||
|
||||
public Task DisconnectAsync() =>
|
||||
_client.IsConnected ? _client.DisconnectAsync() : Task.CompletedTask;
|
||||
|
||||
public async Task PublishAsync(string topic, byte[] payload, string schemaVersion, CancellationToken ct = default)
|
||||
{
|
||||
var msg = new MqttApplicationMessageBuilder()
|
||||
.WithTopic(topic)
|
||||
.WithPayload(payload)
|
||||
.WithQualityOfServiceLevel(MqttQualityOfServiceLevel.AtLeastOnce) // QoS 1 (durable)
|
||||
.WithContentType("application/json")
|
||||
.WithUserProperty("schemaVersion", schemaVersion)
|
||||
.Build();
|
||||
|
||||
var result = await _client.PublishAsync(msg, ct);
|
||||
if (!result.IsSuccess)
|
||||
throw new InvalidOperationException($"publish rejected: {result.ReasonCode}");
|
||||
}
|
||||
|
||||
private async Task OnMessageAsync(MqttApplicationMessageReceivedEventArgs e)
|
||||
{
|
||||
if (e.ApplicationMessage.Topic == _ackTopic)
|
||||
{
|
||||
try
|
||||
{
|
||||
var batch = AckBatch.FromJsonUtf8(e.ApplicationMessage.PayloadSegment);
|
||||
if (AckReceived is { } handlers)
|
||||
{
|
||||
foreach (Func<AckBatch, Task> handler in handlers.GetInvocationList())
|
||||
await handler(batch);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Malformed ack payload — ignore; the record stays queued and is republished.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await DisconnectAsync();
|
||||
_client.Dispose();
|
||||
}
|
||||
}
|
||||
159
src/FieldLogger.Sync/OutboundStore.cs
Normal file
159
src/FieldLogger.Sync/OutboundStore.cs
Normal file
@@ -0,0 +1,159 @@
|
||||
using SQLite;
|
||||
|
||||
namespace FieldLogger.Sync;
|
||||
|
||||
/// <summary>Lifecycle of a queued outbound record.</summary>
|
||||
public enum OutboundStatus
|
||||
{
|
||||
/// <summary>Ready to publish.</summary>
|
||||
Pending = 0,
|
||||
/// <summary>Published and waiting for an application acknowledgement.</summary>
|
||||
InFlight = 1,
|
||||
/// <summary>Terminally rejected by the cloud (schema/validation/authz) — not retried; surfaced for LOG-7.</summary>
|
||||
RejectedTerminal = 2,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A durably-queued outbound point. Survives app restart (persisted in SQLite) and is
|
||||
/// released only when the cloud application-level ack references its <see cref="PointId"/>
|
||||
/// (SRS-SYN-2/7). Broker PUBACK alone does NOT release it.
|
||||
/// </summary>
|
||||
[Table("outbound")]
|
||||
public sealed class OutboundMessage
|
||||
{
|
||||
[PrimaryKey, AutoIncrement] public int Id { get; set; }
|
||||
|
||||
/// <summary>UUIDv7 idempotency key. Unique — a re-enqueue of the same point is a no-op.</summary>
|
||||
[Indexed(Name = "ux_pointid", Order = 1, Unique = true)]
|
||||
public string PointId { get; set; } = "";
|
||||
|
||||
public string Topic { get; set; } = "";
|
||||
public string PayloadJson { get; set; } = "";
|
||||
public string SchemaVersion { get; set; } = "1";
|
||||
|
||||
public long EnqueuedUnixMs { get; set; }
|
||||
public int AttemptCount { get; set; }
|
||||
public long NextAttemptUnixMs { get; set; }
|
||||
public int Status { get; set; } = (int)OutboundStatus.Pending;
|
||||
public string? LastReason { get; set; }
|
||||
|
||||
public static OutboundMessage FromPoint(
|
||||
PointRecord point,
|
||||
string topic,
|
||||
DateTimeOffset enqueuedAt,
|
||||
string? jobId = null,
|
||||
string? ticket = null)
|
||||
{
|
||||
var payload = point.ToAppLogPayloadUtf8(jobId, ticket);
|
||||
return new OutboundMessage
|
||||
{
|
||||
PointId = point.PointId,
|
||||
Topic = topic,
|
||||
PayloadJson = System.Text.Encoding.UTF8.GetString(payload),
|
||||
SchemaVersion = "1",
|
||||
EnqueuedUnixMs = enqueuedAt.ToUnixTimeMilliseconds(),
|
||||
NextAttemptUnixMs = enqueuedAt.ToUnixTimeMilliseconds(),
|
||||
Status = (int)OutboundStatus.Pending,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Durable outbound-queue store. Implementations must survive process restart.</summary>
|
||||
public interface IOutboundStore
|
||||
{
|
||||
Task InitAsync();
|
||||
/// <summary>Enqueue idempotently; returns false if a row with this pointId already exists.</summary>
|
||||
Task<bool> EnqueueAsync(OutboundMessage message);
|
||||
/// <summary>Pending rows whose next-attempt time has arrived, oldest first.</summary>
|
||||
Task<IReadOnlyList<OutboundMessage>> DequeueReadyAsync(DateTimeOffset now, int max = 50);
|
||||
/// <summary>Release a record — accepted by the cloud. Removes it from the queue.</summary>
|
||||
Task ReleaseAckedAsync(string pointId);
|
||||
/// <summary>Record a retryable failure: bump attempt count, schedule the next attempt.</summary>
|
||||
Task RescheduleAsync(string pointId, DateTimeOffset nextAttempt, string reason);
|
||||
/// <summary>Mark a record published-and-awaiting-ack: reschedule a republish (idempotent via
|
||||
/// UUID) after the ack window WITHOUT counting it as a failed attempt.</summary>
|
||||
Task TouchAwaitingAckAsync(string pointId, DateTimeOffset nextAttempt);
|
||||
/// <summary>Terminal rejection (LOG-7): keep the row for surfacing, mark it not-retryable.</summary>
|
||||
Task MarkRejectedAsync(string pointId, string reason);
|
||||
Task<int> PendingCountAsync();
|
||||
Task<IReadOnlyList<OutboundMessage>> GetRejectedAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// SQLite-backed <see cref="IOutboundStore"/>. The DB path is injected (not taken from a
|
||||
/// platform API) so it is testable headless with a temp file; reopening the same path is a
|
||||
/// process restart. Idempotent enqueue relies on the unique index on pointId.
|
||||
/// </summary>
|
||||
public sealed class SqliteOutboundStore : IOutboundStore
|
||||
{
|
||||
private readonly SQLiteAsyncConnection _db;
|
||||
|
||||
public SqliteOutboundStore(string dbPath)
|
||||
{
|
||||
_db = new SQLiteAsyncConnection(dbPath,
|
||||
SQLiteOpenFlags.ReadWrite | SQLiteOpenFlags.Create | SQLiteOpenFlags.SharedCache);
|
||||
}
|
||||
|
||||
public async Task InitAsync()
|
||||
{
|
||||
await _db.CreateTableAsync<OutboundMessage>();
|
||||
// A process crash can leave rows in-flight after the broker accepted them but before
|
||||
// the application ack was applied. Requeue them; pointId makes the replay idempotent.
|
||||
await _db.ExecuteAsync(
|
||||
"UPDATE outbound SET Status = ?, NextAttemptUnixMs = ? WHERE Status = ?",
|
||||
(int)OutboundStatus.Pending, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
|
||||
(int)OutboundStatus.InFlight);
|
||||
}
|
||||
|
||||
public async Task<bool> EnqueueAsync(OutboundMessage message)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _db.InsertAsync(message);
|
||||
return true;
|
||||
}
|
||||
catch (SQLiteException)
|
||||
{
|
||||
// Unique-index violation on pointId → already queued/known. Idempotent no-op.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<OutboundMessage>> DequeueReadyAsync(DateTimeOffset now, int max = 50)
|
||||
{
|
||||
long nowMs = now.ToUnixTimeMilliseconds();
|
||||
return await _db.Table<OutboundMessage>()
|
||||
.Where(m => (m.Status == (int)OutboundStatus.Pending || m.Status == (int)OutboundStatus.InFlight)
|
||||
&& m.NextAttemptUnixMs <= nowMs)
|
||||
.OrderBy(m => m.EnqueuedUnixMs)
|
||||
.Take(max)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
public Task ReleaseAckedAsync(string pointId) =>
|
||||
_db.ExecuteAsync("DELETE FROM outbound WHERE PointId = ?", pointId);
|
||||
|
||||
public Task RescheduleAsync(string pointId, DateTimeOffset nextAttempt, string reason) =>
|
||||
_db.ExecuteAsync(
|
||||
"UPDATE outbound SET AttemptCount = AttemptCount + 1, NextAttemptUnixMs = ?, LastReason = ? WHERE PointId = ? AND Status = ?",
|
||||
nextAttempt.ToUnixTimeMilliseconds(), reason, pointId, (int)OutboundStatus.Pending);
|
||||
|
||||
public Task TouchAwaitingAckAsync(string pointId, DateTimeOffset nextAttempt) =>
|
||||
_db.ExecuteAsync(
|
||||
"UPDATE outbound SET Status = ?, AttemptCount = AttemptCount + 1, NextAttemptUnixMs = ? WHERE PointId = ? AND Status != ?",
|
||||
(int)OutboundStatus.InFlight, nextAttempt.ToUnixTimeMilliseconds(), pointId,
|
||||
(int)OutboundStatus.RejectedTerminal);
|
||||
|
||||
public Task MarkRejectedAsync(string pointId, string reason) =>
|
||||
_db.ExecuteAsync(
|
||||
"UPDATE outbound SET Status = ?, LastReason = ? WHERE PointId = ?",
|
||||
(int)OutboundStatus.RejectedTerminal, reason, pointId);
|
||||
|
||||
public async Task<int> PendingCountAsync() =>
|
||||
await _db.Table<OutboundMessage>()
|
||||
.Where(m => m.Status == (int)OutboundStatus.Pending || m.Status == (int)OutboundStatus.InFlight)
|
||||
.CountAsync();
|
||||
|
||||
public async Task<IReadOnlyList<OutboundMessage>> GetRejectedAsync() =>
|
||||
await _db.Table<OutboundMessage>().Where(m => m.Status == (int)OutboundStatus.RejectedTerminal).ToListAsync();
|
||||
}
|
||||
250
src/FieldLogger.Sync/PointRecord.cs
Normal file
250
src/FieldLogger.Sync/PointRecord.cs
Normal file
@@ -0,0 +1,250 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace FieldLogger.Sync;
|
||||
|
||||
// Wire shape of a Point (SRS §5 / telemetry-schema.md), as published to
|
||||
// ul/{orgId}/app/{clientId}/log/points. pointId (UUIDv7) is the idempotency key on every
|
||||
// path. This is the app→cloud contract payload; keep field names in sync with
|
||||
// telemetry-schema.md (breaking change → backend review).
|
||||
|
||||
/// <summary>Point origin (schema Identity group).</summary>
|
||||
public enum PointOrigin { APP, LOCATOR, MAGLINK }
|
||||
|
||||
/// <summary>How the record reached the cloud (schema Identity group).</summary>
|
||||
public enum UploadPath { APP_MQTT, DEVICE_MQTT, REST_BATCH }
|
||||
|
||||
/// <summary>What triggered the capture (schema Identity group).</summary>
|
||||
public enum CaptureTrigger { APP_UI, LOCATOR_BUTTON }
|
||||
|
||||
public sealed record PointRecord
|
||||
{
|
||||
/// <summary>Payload schema version — also carried as an MQTT5 user property.</summary>
|
||||
[JsonPropertyName("schemaVersion")] public string SchemaVersion { get; init; } = "1";
|
||||
|
||||
// ---- Identity & provenance ----
|
||||
[JsonPropertyName("pointId")] public required string PointId { get; init; } // UUIDv7
|
||||
[JsonPropertyName("ticketId")] public string? TicketId { get; init; }
|
||||
[JsonPropertyName("sessionId")] public string? SessionId { get; init; }
|
||||
[JsonPropertyName("pathId")] public string? PathId { get; init; }
|
||||
[JsonPropertyName("category")] public string Category { get; init; } = "LOCATE";
|
||||
[JsonPropertyName("origin")] public PointOrigin Origin { get; init; } = PointOrigin.APP;
|
||||
[JsonPropertyName("originClientId")] public string? OriginClientId { get; init; }
|
||||
[JsonPropertyName("uploadPath")] public UploadPath UploadPath { get; init; } = UploadPath.APP_MQTT;
|
||||
[JsonPropertyName("captureTrigger")] public CaptureTrigger CaptureTrigger { get; init; } = CaptureTrigger.LOCATOR_BUTTON;
|
||||
[JsonPropertyName("createdAt")] public DateTimeOffset CreatedAt { get; init; }
|
||||
[JsonPropertyName("author")] public string? Author { get; init; }
|
||||
[JsonPropertyName("appVersion")] public string? AppVersion { get; init; }
|
||||
|
||||
[JsonPropertyName("position")] public PositionGroup? Position { get; init; }
|
||||
[JsonPropertyName("gnss")] public GnssGroup? Gnss { get; init; }
|
||||
[JsonPropertyName("locate")] public LocateGroup? Locate { get; init; }
|
||||
[JsonPropertyName("attributes")] public AttributesGroup? Attributes { get; init; }
|
||||
[JsonPropertyName("quality")] public QualityGroup? Quality { get; init; }
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOpts = new()
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
Converters = { new JsonStringEnumConverter() },
|
||||
};
|
||||
|
||||
public byte[] ToJsonUtf8() => JsonSerializer.SerializeToUtf8Bytes(this, JsonOpts);
|
||||
public static PointRecord FromJsonUtf8(ReadOnlySpan<byte> utf8) =>
|
||||
JsonSerializer.Deserialize<PointRecord>(utf8, JsonOpts)
|
||||
?? throw new JsonException("null PointRecord");
|
||||
|
||||
/// <summary>
|
||||
/// Serializes a singleton v1 app-log batch matching telemetry-schema.md's frozen MQTT
|
||||
/// profile. Throws a reason-coded exception instead of silently queuing an unusable point.
|
||||
/// </summary>
|
||||
public byte[] ToAppLogPayloadUtf8(string? jobId = null, string? ticket = null)
|
||||
{
|
||||
string? resolvedJobId = string.IsNullOrWhiteSpace(jobId) ? TicketId : jobId;
|
||||
bool hasJob = !string.IsNullOrWhiteSpace(resolvedJobId);
|
||||
bool hasTicket = !string.IsNullOrWhiteSpace(ticket);
|
||||
if (hasJob == hasTicket)
|
||||
throw new PointNotPublishableException("JOB_IDENTITY_INVALID", "Exactly one of jobId or ticket is required.");
|
||||
if (!IsUuidV7(PointId))
|
||||
throw new PointNotPublishableException("POINT_ID_NOT_UUIDV7", "pointId must be a UUIDv7 value.");
|
||||
if (CreatedAt == default)
|
||||
throw new PointNotPublishableException("CREATED_AT_MISSING", "createdAt is required.");
|
||||
if (Position?.Lat is not double lat || Position.Lon is not double lng)
|
||||
throw new PointNotPublishableException("POSITION_MISSING", "Latitude and longitude are required.");
|
||||
|
||||
var wirePoint = new AppLogPointWire
|
||||
{
|
||||
PointId = PointId,
|
||||
CreatedAt = CreatedAt,
|
||||
Origin = "APP",
|
||||
UploadPath = "APP_MQTT",
|
||||
Lat = lat,
|
||||
Lng = lng,
|
||||
Alt = Position.EllipsoidalHeight,
|
||||
Ts = Position.PositionEpoch ?? Locate?.TelemetryEpoch ?? CreatedAt,
|
||||
Fix = NormalizeFix(Gnss?.FixType),
|
||||
HAcc = Gnss?.Hrms,
|
||||
VAcc = Gnss?.Vrms,
|
||||
Sats = Gnss?.SatsUsed,
|
||||
Hdop = Gnss?.Hdop,
|
||||
Depth = Locate?.Depth,
|
||||
FreqHz = Locate?.Frequency is double frequency ? checked((int)Math.Round(frequency)) : null,
|
||||
CurrentMa = Locate?.SignalCurrent,
|
||||
SignalDb = Locate?.SignalStrength,
|
||||
GainDb = Locate?.Gain,
|
||||
Mode = Locate?.LocateMode,
|
||||
PhaseDeg = Locate?.PhaseDegrees,
|
||||
CompassDeg = Locate?.CompassDegrees,
|
||||
DistortionPct = Locate?.DistortionPercent,
|
||||
Utility = NormalizeUtility(Attributes?.UtilityType),
|
||||
QualityFlag = Quality?.QualityFlag ?? "IN_SPEC",
|
||||
};
|
||||
var envelope = new AppLogPointsEnvelope
|
||||
{
|
||||
SchemaVersion = "1",
|
||||
JobId = hasJob ? resolvedJobId : null,
|
||||
Ticket = hasTicket ? ticket : null,
|
||||
Points = [wirePoint],
|
||||
};
|
||||
return JsonSerializer.SerializeToUtf8Bytes(envelope, AppLogJsonOptions);
|
||||
}
|
||||
|
||||
private static bool IsUuidV7(string value)
|
||||
{
|
||||
string text = value.ToLowerInvariant();
|
||||
return Guid.TryParseExact(text, "D", out _)
|
||||
&& text.Length == 36
|
||||
&& text[14] == '7'
|
||||
&& text[19] is '8' or '9' or 'a' or 'b';
|
||||
}
|
||||
|
||||
private static string? NormalizeFix(string? value) => value?.ToUpperInvariant() switch
|
||||
{
|
||||
null or "" => null,
|
||||
"AUTONOMOUS" => "AUTONOMOUS",
|
||||
"DGPS" => "DGPS",
|
||||
"FLOAT" or "RTK_FLOAT" or "FLOAT_RTK" => "FLOAT",
|
||||
"FIXED" or "RTK_FIXED" or "FIXED_RTK" => "FIXED",
|
||||
"NO_FIX" or "NONE" => "NO_FIX",
|
||||
_ => throw new PointNotPublishableException("FIX_TYPE_INVALID", $"Unsupported fix type '{value}'."),
|
||||
};
|
||||
|
||||
private static string? NormalizeUtility(string? value) => value?.ToUpperInvariant() switch
|
||||
{
|
||||
null or "" => null,
|
||||
"ELECTRIC" or "GAS" or "WATER" or "SEWER" or "TELECOM" or "CATV" or "FIBER" or "STEAM" or "UNKNOWN"
|
||||
=> value.ToUpperInvariant(),
|
||||
_ => throw new PointNotPublishableException("UTILITY_TYPE_INVALID", $"Unsupported utility type '{value}'."),
|
||||
};
|
||||
|
||||
private static readonly JsonSerializerOptions AppLogJsonOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
}
|
||||
|
||||
public sealed class PointNotPublishableException : InvalidOperationException
|
||||
{
|
||||
public string ReasonCode { get; }
|
||||
|
||||
public PointNotPublishableException(string reasonCode, string message) : base(message) =>
|
||||
ReasonCode = reasonCode;
|
||||
}
|
||||
|
||||
internal sealed record AppLogPointsEnvelope
|
||||
{
|
||||
public string SchemaVersion { get; init; } = "1";
|
||||
public string? JobId { get; init; }
|
||||
public string? Ticket { get; init; }
|
||||
public List<AppLogPointWire> Points { get; init; } = [];
|
||||
}
|
||||
|
||||
internal sealed record AppLogPointWire
|
||||
{
|
||||
public required string PointId { get; init; }
|
||||
public required DateTimeOffset CreatedAt { get; init; }
|
||||
public string Origin { get; init; } = "APP";
|
||||
public string UploadPath { get; init; } = "APP_MQTT";
|
||||
public required double Lat { get; init; }
|
||||
public required double Lng { get; init; }
|
||||
public double? Alt { get; init; }
|
||||
public required DateTimeOffset Ts { get; init; }
|
||||
public string? Fix { get; init; }
|
||||
public double? HAcc { get; init; }
|
||||
public double? VAcc { get; init; }
|
||||
public int? Sats { get; init; }
|
||||
public double? Hdop { get; init; }
|
||||
public double? Depth { get; init; }
|
||||
public int? FreqHz { get; init; }
|
||||
public double? CurrentMa { get; init; }
|
||||
public double? SignalDb { get; init; }
|
||||
public double? GainDb { get; init; }
|
||||
public string? Mode { get; init; }
|
||||
public double? PhaseDeg { get; init; }
|
||||
public double? CompassDeg { get; init; }
|
||||
public double? DistortionPct { get; init; }
|
||||
public string? Utility { get; init; }
|
||||
public string QualityFlag { get; init; } = "IN_SPEC";
|
||||
}
|
||||
|
||||
public sealed record PositionGroup
|
||||
{
|
||||
[JsonPropertyName("lat")] public double? Lat { get; init; }
|
||||
[JsonPropertyName("lon")] public double? Lon { get; init; }
|
||||
[JsonPropertyName("crsEpsg")] public int? CrsEpsg { get; init; }
|
||||
[JsonPropertyName("ellipsoidalHeight")] public double? EllipsoidalHeight { get; init; }
|
||||
[JsonPropertyName("orthometricHeight")] public double? OrthometricHeight { get; init; }
|
||||
[JsonPropertyName("geoidModel")] public string? GeoidModel { get; init; }
|
||||
[JsonPropertyName("antennaHeight")] public double? AntennaHeight { get; init; }
|
||||
[JsonPropertyName("positionEpoch")] public DateTimeOffset? PositionEpoch { get; init; }
|
||||
}
|
||||
|
||||
public sealed record GnssGroup
|
||||
{
|
||||
[JsonPropertyName("fixType")] public string? FixType { get; init; }
|
||||
[JsonPropertyName("satsUsed")] public int? SatsUsed { get; init; }
|
||||
[JsonPropertyName("hdop")] public double? Hdop { get; init; }
|
||||
[JsonPropertyName("hrms")] public double? Hrms { get; init; }
|
||||
[JsonPropertyName("vrms")] public double? Vrms { get; init; }
|
||||
[JsonPropertyName("correctionAge")] public double? CorrectionAge { get; init; }
|
||||
[JsonPropertyName("receiverModel")] public string? ReceiverModel { get; init; }
|
||||
[JsonPropertyName("receiverSerial")] public string? ReceiverSerial { get; init; }
|
||||
[JsonPropertyName("source")] public string? Source { get; init; } // MAGLINK | PHONE
|
||||
[JsonPropertyName("tiltAngle")] public double? TiltAngle { get; init; }
|
||||
[JsonPropertyName("clockSource")] public string? ClockSource { get; init; }
|
||||
}
|
||||
|
||||
public sealed record LocateGroup
|
||||
{
|
||||
[JsonPropertyName("depth")] public double? Depth { get; init; }
|
||||
[JsonPropertyName("depthUnits")] public string? DepthUnits { get; init; }
|
||||
[JsonPropertyName("signalCurrent")] public double? SignalCurrent { get; init; }
|
||||
[JsonPropertyName("signalStrength")] public double? SignalStrength { get; init; }
|
||||
[JsonPropertyName("frequency")] public double? Frequency { get; init; }
|
||||
[JsonPropertyName("locateMode")] public string? LocateMode { get; init; }
|
||||
[JsonPropertyName("gain")] public double? Gain { get; init; }
|
||||
[JsonPropertyName("signalDirection")] public double? SignalDirection { get; init; }
|
||||
[JsonPropertyName("phaseDegrees")] public double? PhaseDegrees { get; init; }
|
||||
[JsonPropertyName("compassDegrees")] public double? CompassDegrees { get; init; }
|
||||
[JsonPropertyName("distortionPercent")] public double? DistortionPercent { get; init; }
|
||||
[JsonPropertyName("warningFlags")] public int? WarningFlags { get; init; }
|
||||
[JsonPropertyName("locatorModel")] public string? LocatorModel { get; init; }
|
||||
[JsonPropertyName("locatorSerial")] public string? LocatorSerial { get; init; }
|
||||
[JsonPropertyName("telemetryEpoch")] public DateTimeOffset? TelemetryEpoch { get; init; }
|
||||
}
|
||||
|
||||
public sealed record AttributesGroup
|
||||
{
|
||||
[JsonPropertyName("utilityType")] public string? UtilityType { get; init; }
|
||||
[JsonPropertyName("owner")] public string? Owner { get; init; }
|
||||
[JsonPropertyName("markerColor")] public string? MarkerColor { get; init; }
|
||||
[JsonPropertyName("surfaceType")] public string? SurfaceType { get; init; }
|
||||
[JsonPropertyName("notes")] public string? Notes { get; init; }
|
||||
}
|
||||
|
||||
public sealed record QualityGroup
|
||||
{
|
||||
[JsonPropertyName("qualityFlag")] public string? QualityFlag { get; init; }
|
||||
[JsonPropertyName("gatePolicyId")] public string? GatePolicyId { get; init; }
|
||||
[JsonPropertyName("waiverId")] public string? WaiverId { get; init; }
|
||||
}
|
||||
Reference in New Issue
Block a user