import dotenv from "dotenv"; dotenv.config(); import pg from "pg"; import { logger } from "./logger.js"; const STALE_THRESHOLD_MS = 2 * 60 * 1000; const CHECK_INTERVAL_MS = 30 * 1000; const DISK_CRITICAL_PERCENT = 90; async function checkDeviceStatuses(pool: pg.Pool) { const client = await pool.connect(); try { const staleThreshold = new Date(Date.now() - STALE_THRESHOLD_MS); const { rows: staleDevices } = await client.query( `UPDATE devices SET status = 'offline', updated_at = NOW() WHERE status = 'online' AND last_heartbeat_at < $1 AND is_disabled = false AND is_archived = false RETURNING id, name, organization_id`, [staleThreshold] ); for (const device of staleDevices) { const existing = await client.query( `SELECT id FROM alerts WHERE device_id = $1 AND alert_type = 'device_offline' AND is_resolved = false LIMIT 1`, [device.id] ); if (existing.rows.length === 0) { await client.query( `INSERT INTO alerts (organization_id, device_id, alert_type, severity, title, message) VALUES ($1, $2, 'device_offline', 'warning', $3, $4)`, [ device.organization_id, device.id, `Device "${device.name}" is offline`, `Device "${device.name}" has not sent a heartbeat in over ${STALE_THRESHOLD_MS / 1000} seconds.`, ] ); } } if (staleDevices.length > 0) { logger.info("Marked devices offline", { count: staleDevices.length }); } } finally { client.release(); } } async function checkDiskAlerts(pool: pg.Pool) { const client = await pool.connect(); try { const { rows: criticalDevices } = await client.query( `SELECT d.id, d.name, d.organization_id, m.disk_usage_percent FROM devices d JOIN LATERAL ( SELECT disk_usage_percent FROM metric_snapshots WHERE device_id = d.id ORDER BY reported_at DESC LIMIT 1 ) m ON true WHERE m.disk_usage_percent >= $1 AND d.status = 'online' AND d.is_disabled = false`, [DISK_CRITICAL_PERCENT] ); for (const device of criticalDevices) { const existing = await client.query( `SELECT id FROM alerts WHERE device_id = $1 AND alert_type = 'disk_critical' AND is_resolved = false LIMIT 1`, [device.id] ); if (existing.rows.length === 0) { await client.query( `INSERT INTO alerts (organization_id, device_id, alert_type, severity, title, message) VALUES ($1, $2, 'disk_critical', 'error', $3, $4)`, [ device.organization_id, device.id, `Critical disk usage on "${device.name}"`, `Disk usage is at ${device.disk_usage_percent}% on device "${device.name}".`, ] ); } } if (criticalDevices.length > 0) { logger.info("Created disk alerts", { count: criticalDevices.length }); } } finally { client.release(); } } async function checkAgentOutdated(pool: pg.Pool) { const client = await pool.connect(); try { const CURRENT_AGENT_VERSION = "0.1.0"; const { rows: outdatedDevices } = await client.query( `SELECT id, name, organization_id, agent_version FROM devices WHERE agent_version < $1 AND status = 'online' AND is_disabled = false`, [CURRENT_AGENT_VERSION] ); for (const device of outdatedDevices) { const existing = await client.query( `SELECT id FROM alerts WHERE device_id = $1 AND alert_type = 'agent_outdated' AND is_resolved = false LIMIT 1`, [device.id] ); if (existing.rows.length === 0) { await client.query( `INSERT INTO alerts (organization_id, device_id, alert_type, severity, title, message) VALUES ($1, $2, 'agent_outdated', 'info', $3, $4)`, [ device.organization_id, device.id, `Agent outdated on "${device.name}"`, `Device "${device.name}" is running agent v${device.agent_version}, current is v${CURRENT_AGENT_VERSION}.`, ] ); } } } finally { client.release(); } } async function dispatchJobs(pool: pg.Pool) { const client = await pool.connect(); try { const { rows: queuedJobs } = await client.query( `UPDATE jobs SET status = 'dispatched', dispatched_at = NOW() WHERE status = 'queued' AND (expires_at IS NULL OR expires_at > NOW()) RETURNING id, job_type` ); for (const job of queuedJobs) { await client.query( `UPDATE job_targets SET status = 'dispatched' WHERE job_id = $1 AND status = 'queued'`, [job.id] ); } const { rows: expiredJobs } = await client.query( `UPDATE jobs SET status = 'expired', completed_at = NOW() WHERE status IN ('queued', 'dispatched') AND expires_at IS NOT NULL AND expires_at < NOW() RETURNING id` ); for (const job of expiredJobs) { await client.query( `UPDATE job_targets SET status = 'expired', completed_at = NOW() WHERE job_id = $1 AND status IN ('queued', 'dispatched')`, [job.id] ); } if (queuedJobs.length > 0) { logger.info("Dispatched jobs", { count: queuedJobs.length }); } if (expiredJobs.length > 0) { logger.info("Expired jobs", { count: expiredJobs.length }); } } finally { client.release(); } } async function dispatchDeployments(pool: pg.Pool) { const client = await pool.connect(); try { const { rows: approvedDeployments } = await client.query( `SELECT id, name, job_type, parameters, organization_id, created_by_user_id FROM deployments WHERE status = 'approved' AND (scheduled_at IS NULL OR scheduled_at <= NOW())` ); for (const deployment of approvedDeployments) { await client.query( `UPDATE deployments SET status = 'running', started_at = NOW() WHERE id = $1`, [deployment.id] ); const { rows: targets } = await client.query( `SELECT id, device_id FROM deployment_targets WHERE deployment_id = $1 AND status = 'pending'`, [deployment.id] ); for (const target of targets) { const jobResult = await client.query( `INSERT INTO jobs (organization_id, created_by_user_id, job_type, parameters, status) VALUES ($1, $2, $3, $4, 'queued') RETURNING id`, [deployment.organization_id, deployment.created_by_user_id, deployment.job_type, JSON.stringify(deployment.parameters)] ); const jobId = jobResult.rows[0].id; await client.query( `INSERT INTO job_targets (job_id, device_id, status) VALUES ($1, $2, 'queued')`, [jobId, target.device_id] ); await client.query( `UPDATE deployment_targets SET job_id = $1, status = 'running', started_at = NOW() WHERE id = $2`, [jobId, target.id] ); } logger.info("Dispatched deployment", { deploymentId: deployment.id, name: deployment.name, targetCount: targets.length }); } const { rows: runningDeployments } = await client.query( `SELECT d.id, d.total_targets, (SELECT COUNT(*) FROM deployment_targets dt WHERE dt.deployment_id = d.id AND dt.status = 'succeeded') as completed, (SELECT COUNT(*) FROM deployment_targets dt WHERE dt.deployment_id = d.id AND dt.status = 'failed') as failed, (SELECT COUNT(*) FROM deployment_targets dt WHERE dt.deployment_id = d.id AND dt.status IN ('pending', 'running')) as active FROM deployments d WHERE d.status = 'running'` ); for (const d of runningDeployments) { if (Number(d.active) === 0) { const finalStatus = Number(d.failed) > 0 ? 'partially_completed' : 'completed'; await client.query( `UPDATE deployments SET status = $1, completed_at = NOW(), completed_targets = $2, failed_targets = $3 WHERE id = $4`, [finalStatus, Number(d.completed), Number(d.failed), d.id] ); await client.query( `UPDATE deployment_targets SET status = 'expired', completed_at = NOW() WHERE deployment_id = $1 AND status = 'pending'`, [d.id] ); logger.info("Deployment finished", { deploymentId: d.id, status: finalStatus }); } } } finally { client.release(); } } async function evaluateCompliance(pool: pg.Pool) { const client = await pool.connect(); try { const { rows: orgs } = await client.query( `SELECT DISTINCT organization_id FROM policies WHERE is_enabled = true` ); let totalEvaluated = 0; let totalNonCompliant = 0; for (const org of orgs) { const { rows: devices } = await client.query( `SELECT id FROM devices WHERE organization_id = $1 AND is_disabled = false AND is_archived = false`, [org.organization_id] ); const { rows: policies } = await client.query( `SELECT id, rules FROM policies WHERE organization_id = $1 AND is_enabled = true`, [org.organization_id] ); if (devices.length === 0 || policies.length === 0) continue; const deviceIds = devices.map((d: any) => d.id); const metricsMap = new Map(); const inventoryMap = new Map(); if (deviceIds.length > 0) { const { rows: metricsRows } = await client.query( `SELECT DISTINCT ON (device_id) device_id, cpu_usage_percent, memory_usage_percent, disk_usage_percent, load_average_1m, uptime_seconds FROM metric_snapshots WHERE device_id = ANY($1) ORDER BY device_id, reported_at DESC`, [deviceIds] ); for (const r of metricsRows) metricsMap.set(r.device_id, r); const { rows: invRows } = await client.query( `SELECT DISTINCT ON (device_id) device_id, packages, services FROM inventory_snapshots WHERE device_id = ANY($1) ORDER BY device_id, reported_at DESC`, [deviceIds] ); for (const r of invRows) inventoryMap.set(r.device_id, r); } const { rows: deviceRows } = await client.query( `SELECT id, os_name, os_version, platform, architecture, agent_version, cpu_cores, total_memory_bytes, total_disk_bytes, status, tags, kernel_version FROM devices WHERE id = ANY($1)`, [deviceIds] ); const deviceMap = new Map(); for (const d of deviceRows) deviceMap.set(d.id, d); const now = new Date(); for (const device of deviceRows) { const metrics = metricsMap.get(device.id) || null; const inventory = inventoryMap.get(device.id) || null; for (const policy of policies) { const rules = (policy.rules || []) as any[]; let allPassed = true; let hasUnknown = false; const ruleResults: Record = {}; for (const rule of rules) { const actual = getField(device, metrics, inventory, rule.type); if (actual === undefined) { ruleResults[rule.type] = { condition: rule.condition, expected: rule.value, actual: null, passed: false, reason: "field_not_found", }; hasUnknown = true; allPassed = false; continue; } const passed = evaluateRule(actual, rule.condition, rule.value); ruleResults[rule.type] = { condition: rule.condition, expected: rule.value, actual, passed, }; if (!passed) allPassed = false; } let status: string; if (hasUnknown && !allPassed) status = "unknown"; else if (allPassed) status = "compliant"; else status = "non_compliant"; await client.query( `INSERT INTO compliance_results (device_id, policy_id, status, details, evaluated_at) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (device_id, policy_id) DO UPDATE SET status = EXCLUDED.status, details = EXCLUDED.details, evaluated_at = EXCLUDED.evaluated_at`, [device.id, policy.id, status, JSON.stringify(ruleResults), now] ); totalEvaluated++; if (status === "non_compliant") totalNonCompliant++; } } } if (totalEvaluated > 0) { console.log( `Compliance: ${totalEvaluated} evaluations, ${totalNonCompliant} non-compliant.` ); } } finally { client.release(); } } function getField( device: any, metrics: any, inventory: any, fieldType: string ): unknown { switch (fieldType) { case "os_version": return device.os_version; case "os_name": return device.os_name; case "platform": return device.platform; case "architecture": return device.architecture; case "agent_version": return device.agent_version; case "cpu_cores": return device.cpu_cores; case "memory_total": return device.total_memory_bytes; case "disk_total": return device.total_disk_bytes; case "status": return device.status; case "tags": return device.tags || []; case "kernel_version": return device.kernel_version; case "cpu_usage": return metrics?.cpu_usage_percent; case "memory_usage": return metrics?.memory_usage_percent; case "disk_usage": return metrics?.disk_usage_percent; case "load_average": return metrics?.load_average_1m; case "uptime": return metrics?.uptime_seconds; case "package_count": return inventory?.packages?.length ?? 0; case "has_package": return inventory?.packages?.map((p: any) => p.name) ?? []; case "service_count": return inventory?.services?.length ?? 0; default: return undefined; } } function compareVersions(a: string, b: string): number { const pa = a.split(".").map(Number); const pb = b.split(".").map(Number); for (let i = 0; i < Math.max(pa.length, pb.length); i++) { const na = pa[i] || 0; const nb = pb[i] || 0; if (na > nb) return 1; if (na < nb) return -1; } return 0; } function evaluateRule(actual: unknown, condition: string, expected: any): boolean { if (actual === undefined || actual === null) { return condition === "not_equals" && expected !== null; } switch (condition) { case "equals": return String(actual) === String(expected); case "not_equals": return String(actual) !== String(expected); case "greater_than": return Number(actual) > Number(expected); case "less_than": return Number(actual) < Number(expected); case "greater_than_or_equal": return Number(actual) >= Number(expected); case "less_than_or_equal": return Number(actual) <= Number(expected); case "contains": if (Array.isArray(actual)) return actual.includes(String(expected)); return String(actual).includes(String(expected)); case "not_contains": if (Array.isArray(actual)) return !actual.includes(String(expected)); return !String(actual).includes(String(expected)); case "in": if (Array.isArray(expected)) return expected.map(String).includes(String(actual)); return String(actual) === String(expected); case "not_in": if (Array.isArray(expected)) return !expected.map(String).includes(String(actual)); return String(actual) !== String(expected); case "version_gte": return compareVersions(String(actual), String(expected)) >= 0; case "version_lte": return compareVersions(String(actual), String(expected)) <= 0; case "version_gt": return compareVersions(String(actual), String(expected)) > 0; case "version_lt": return compareVersions(String(actual), String(expected)) < 0; case "is_true": return actual === true || actual === "true"; case "is_false": return actual === false || actual === "false"; case "is_empty": if (Array.isArray(actual)) return actual.length === 0; if (typeof actual === "string") return actual.length === 0; return actual === null || actual === undefined; case "is_not_empty": if (Array.isArray(actual)) return actual.length > 0; if (typeof actual === "string") return actual.length > 0; return actual !== null && actual !== undefined; default: return false; } } async function runChecks(pool: pg.Pool) { try { await checkDeviceStatuses(pool); await checkDiskAlerts(pool); await checkAgentOutdated(pool); await dispatchJobs(pool); await dispatchDeployments(pool); await evaluateCompliance(pool); } catch (err) { logger.error("Worker check error", {}, err as Error); } } async function main() { logger.info("Fleet Forge Worker starting", { staleThreshold: `${STALE_THRESHOLD_MS / 1000}s`, diskCriticalThreshold: `${DISK_CRITICAL_PERCENT}%`, checkInterval: `${CHECK_INTERVAL_MS / 1000}s`, }); const pool = new pg.Pool({ connectionString: process.env.DATABASE_URL, }); await pool.query("SELECT 1"); logger.info("Connected to PostgreSQL"); await runChecks(pool); setInterval(() => runChecks(pool), CHECK_INTERVAL_MS); const shutdown = async () => { logger.info("Worker shutting down"); await pool.end(); process.exit(0); }; process.on("SIGINT", shutdown); process.on("SIGTERM", shutdown); } main().catch((err) => { logger.error("Worker failed to start", {}, err as Error); process.exit(1); });