feat(S2-a): ingest durable app MQTT points securely
This commit is contained in:
Brent Perteet
277
backend/src/ingest/app-log-ingest.service.ts
Normal file
277
backend/src/ingest/app-log-ingest.service.ts
Normal file
@@ -0,0 +1,277 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { GpsFixType, Prisma } from '@prisma/client';
|
||||
import { plainToInstance } from 'class-transformer';
|
||||
import { validate } from 'class-validator';
|
||||
import { toPointDto } from '../points/points.service';
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { RealtimeService } from '../realtime/realtime.service';
|
||||
import {
|
||||
AppFixType,
|
||||
AppLogAck,
|
||||
AppLogAckResult,
|
||||
AppLogPointDto,
|
||||
AppLogPointsMessageDto,
|
||||
} from './dto/app-messages.dto';
|
||||
import { MqttClientService } from './mqtt-client.service';
|
||||
|
||||
const SCHEMA_VERSION = '1';
|
||||
const UUID_V7 =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
|
||||
// Durable app-originated ingest for ul/{orgId}/app/{clientId}/log/points. The exact v1
|
||||
// publish/ack envelopes are frozen in meta/contracts/telemetry-schema.md.
|
||||
@Injectable()
|
||||
export class AppLogIngestService {
|
||||
private readonly logger = new Logger(AppLogIngestService.name);
|
||||
|
||||
constructor(
|
||||
private readonly prisma: PrismaService,
|
||||
private readonly realtime: RealtimeService,
|
||||
private readonly mqttClient: MqttClientService,
|
||||
) {}
|
||||
|
||||
async handle(
|
||||
orgId: string,
|
||||
clientId: string,
|
||||
rawPayload: string,
|
||||
): Promise<void> {
|
||||
const org = await this.prisma.organization.findUnique({
|
||||
where: { id: orgId },
|
||||
select: { id: true },
|
||||
});
|
||||
if (!org) {
|
||||
this.logger.warn(
|
||||
`app-log for unknown org "${orgId}" (client ${clientId}) — dropped`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(rawPayload);
|
||||
} catch {
|
||||
// There is no trustworthy pointId to acknowledge. Raw app payloads are redacted by the
|
||||
// router; retain/log on the app side rather than sending a null/ambiguous acknowledgement.
|
||||
this.logger.warn(
|
||||
`invalid JSON from app ${orgId}/${clientId} — no acknowledgement emitted`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const msg = plainToInstance(AppLogPointsMessageDto, parsed as object);
|
||||
const rawPoints = Array.isArray((parsed as { points?: unknown[] })?.points)
|
||||
? (parsed as { points: unknown[] }).points
|
||||
: [];
|
||||
const usableIds = rawPoints
|
||||
.map((value) =>
|
||||
typeof (value as { pointId?: unknown })?.pointId === 'string'
|
||||
? (value as { pointId: string }).pointId
|
||||
: null,
|
||||
)
|
||||
.filter((value): value is string => value !== null);
|
||||
|
||||
const hasJobId = typeof msg.jobId === 'string' && msg.jobId.length > 0;
|
||||
const hasTicket = typeof msg.ticket === 'string' && msg.ticket.length > 0;
|
||||
const envelopeValid =
|
||||
msg.schemaVersion === SCHEMA_VERSION &&
|
||||
rawPoints.length > 0 &&
|
||||
rawPoints.length <= 500 &&
|
||||
hasJobId !== hasTicket;
|
||||
if (!envelopeValid) {
|
||||
this.publishAck(
|
||||
orgId,
|
||||
clientId,
|
||||
usableIds.map((pointId) => ({
|
||||
pointId,
|
||||
outcome: 'REJECTED',
|
||||
reasonCode: 'ENVELOPE_VALIDATION_ERROR',
|
||||
})),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate independently so one malformed capture does not discard valid captures in the
|
||||
// same batch (LOG-7). Invalid UUID strings can still be named in a rejection; missing IDs
|
||||
// cannot be safely correlated and therefore remain unacknowledged.
|
||||
const results: AppLogAckResult[] = [];
|
||||
const candidates: AppLogPointDto[] = [];
|
||||
for (const rawPoint of rawPoints) {
|
||||
const point = plainToInstance(AppLogPointDto, rawPoint as object);
|
||||
const pointId = typeof point.pointId === 'string' ? point.pointId : null;
|
||||
const errors = await validate(point, { whitelist: true });
|
||||
if (!pointId) {
|
||||
continue;
|
||||
}
|
||||
if (errors.length > 0 || !UUID_V7.test(pointId)) {
|
||||
results.push({
|
||||
pointId,
|
||||
outcome: 'REJECTED',
|
||||
reasonCode: UUID_V7.test(pointId)
|
||||
? 'VALIDATION_ERROR'
|
||||
: 'POINT_ID_NOT_UUIDV7',
|
||||
});
|
||||
continue;
|
||||
}
|
||||
candidates.push(point);
|
||||
}
|
||||
|
||||
const job = await this.resolveJob(orgId, clientId, msg);
|
||||
if (!job) {
|
||||
results.push(
|
||||
...candidates.map((point) => ({
|
||||
pointId: point.pointId,
|
||||
outcome: 'REJECTED' as const,
|
||||
reasonCode: 'UNKNOWN_JOB_OR_WRONG_ORG',
|
||||
})),
|
||||
);
|
||||
this.publishAck(orgId, clientId, results);
|
||||
return;
|
||||
}
|
||||
|
||||
// Collapse duplicates inside this delivery, then use producer event time rather than
|
||||
// arrival/array order. The first valid occurrence wins; repeated IDs are safe to release.
|
||||
const byPointId = new Map<string, AppLogPointDto>();
|
||||
for (const point of candidates) {
|
||||
if (byPointId.has(point.pointId)) {
|
||||
results.push({
|
||||
pointId: point.pointId,
|
||||
outcome: 'DUPLICATE',
|
||||
reasonCode: 'DUPLICATE_IN_BATCH',
|
||||
});
|
||||
} else {
|
||||
byPointId.set(point.pointId, point);
|
||||
}
|
||||
}
|
||||
const ordered = [...byPointId.values()].sort(
|
||||
(a, b) => +new Date(a.createdAt) - +new Date(b.createdAt),
|
||||
);
|
||||
|
||||
const seen = ordered.length
|
||||
? await this.prisma.locatePoint.findMany({
|
||||
where: { pointId: { in: ordered.map((point) => point.pointId) } },
|
||||
select: { pointId: true },
|
||||
})
|
||||
: [];
|
||||
const seenIds = new Set(seen.map((row) => row.pointId));
|
||||
const fresh = ordered.filter((point) => !seenIds.has(point.pointId));
|
||||
for (const point of ordered.filter((candidate) =>
|
||||
seenIds.has(candidate.pointId),
|
||||
)) {
|
||||
results.push({ pointId: point.pointId, outcome: 'DUPLICATE' });
|
||||
}
|
||||
|
||||
let stored: Awaited<
|
||||
ReturnType<typeof this.prisma.locatePoint.createManyAndReturn>
|
||||
> = [];
|
||||
if (fresh.length > 0) {
|
||||
stored = await this.prisma.locatePoint.createManyAndReturn({
|
||||
data: fresh.map((point) => ({
|
||||
jobId: job.id,
|
||||
deviceId: null,
|
||||
pointId: point.pointId,
|
||||
origin: 'APP' as const,
|
||||
uploadPath: 'APP_MQTT' as const,
|
||||
originClientId: clientId,
|
||||
createdAt: new Date(point.createdAt),
|
||||
lat: point.lat,
|
||||
lng: point.lng,
|
||||
altitude: point.alt,
|
||||
utilityType: point.utility,
|
||||
sequence: point.seq,
|
||||
fixType: this.toStoredFix(point.fix),
|
||||
hAccuracy: point.hAcc,
|
||||
vAccuracy: point.vAcc,
|
||||
satellites: point.sats,
|
||||
hdop: point.hdop,
|
||||
depth: point.depth,
|
||||
frequencyHz: point.freqHz,
|
||||
currentMa: point.currentMa,
|
||||
signalDb: point.signalDb,
|
||||
gainDb: point.gainDb,
|
||||
locateMode: point.mode,
|
||||
phaseDeg: point.phaseDeg,
|
||||
compassDeg: point.compassDeg,
|
||||
distortionPct: point.distortionPct,
|
||||
recordedAt: new Date(point.ts),
|
||||
raw: point as unknown as Prisma.InputJsonValue,
|
||||
})),
|
||||
skipDuplicates: true,
|
||||
});
|
||||
}
|
||||
|
||||
const storedIds = new Set(stored.map((row) => row.pointId));
|
||||
for (const point of fresh) {
|
||||
results.push({
|
||||
pointId: point.pointId,
|
||||
outcome: storedIds.has(point.pointId) ? 'ACCEPTED' : 'DUPLICATE',
|
||||
});
|
||||
}
|
||||
|
||||
if (stored.length > 0) {
|
||||
this.realtime.publish(`job:${job.id}`, {
|
||||
type: 'points',
|
||||
jobId: job.id,
|
||||
points: stored.map(toPointDto),
|
||||
});
|
||||
}
|
||||
this.publishAck(orgId, clientId, results);
|
||||
}
|
||||
|
||||
private async resolveJob(
|
||||
orgId: string,
|
||||
clientId: string,
|
||||
msg: AppLogPointsMessageDto,
|
||||
) {
|
||||
if (msg.jobId) {
|
||||
const job = await this.prisma.job.findFirst({
|
||||
where: { id: msg.jobId, orgId },
|
||||
});
|
||||
if (!job)
|
||||
this.logger.warn(
|
||||
`app-log ${orgId}/${clientId} referenced a foreign/unknown job ${msg.jobId}`,
|
||||
);
|
||||
return job;
|
||||
}
|
||||
const existing = await this.prisma.job.findUnique({
|
||||
where: { orgId_ticketNumber: { orgId, ticketNumber: msg.ticket! } },
|
||||
});
|
||||
if (existing) return existing;
|
||||
return this.prisma.job.create({
|
||||
data: {
|
||||
orgId,
|
||||
ticketNumber: msg.ticket!,
|
||||
title: `Ticket ${msg.ticket} (app-created)`,
|
||||
status: 'IN_PROGRESS',
|
||||
source: 'WEB',
|
||||
startedAt: new Date(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private toStoredFix(fix?: AppFixType): GpsFixType | undefined {
|
||||
switch (fix) {
|
||||
case 'FLOAT':
|
||||
return GpsFixType.FLOAT_RTK;
|
||||
case 'FIXED':
|
||||
return GpsFixType.FIXED_RTK;
|
||||
case 'NO_FIX':
|
||||
return GpsFixType.NONE;
|
||||
case 'AUTONOMOUS':
|
||||
return GpsFixType.AUTONOMOUS;
|
||||
case 'DGPS':
|
||||
return GpsFixType.DGPS;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private publishAck(
|
||||
orgId: string,
|
||||
clientId: string,
|
||||
results: AppLogAckResult[],
|
||||
): void {
|
||||
if (results.length === 0) return;
|
||||
const ack: AppLogAck = { schemaVersion: SCHEMA_VERSION, results };
|
||||
this.mqttClient.publish(`ul/${orgId}/app/${clientId}/ack`, ack);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user