feat: Fleet Forge UEM - device management, agent, remote desktop (VNC), TLS reverse proxy
This commit is contained in:
Pepe44DEV
@@ -0,0 +1,535 @@
|
||||
import dotenv from "dotenv";
|
||||
dotenv.config();
|
||||
|
||||
import pg from "pg";
|
||||
import { logger } from "./logger.js";
|
||||
|
||||
const STALE_THRESHOLD_MS = 2 * 60 * 1000;
|
||||
const CHECK_INTERVAL_MS = 30 * 1000;
|
||||
const DISK_CRITICAL_PERCENT = 90;
|
||||
|
||||
async function checkDeviceStatuses(pool: pg.Pool) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const staleThreshold = new Date(Date.now() - STALE_THRESHOLD_MS);
|
||||
|
||||
const { rows: staleDevices } = await client.query(
|
||||
`UPDATE devices SET status = 'offline', updated_at = NOW()
|
||||
WHERE status = 'online'
|
||||
AND last_heartbeat_at < $1
|
||||
AND is_disabled = false
|
||||
AND is_archived = false
|
||||
RETURNING id, name, organization_id`,
|
||||
[staleThreshold]
|
||||
);
|
||||
|
||||
for (const device of staleDevices) {
|
||||
const existing = await client.query(
|
||||
`SELECT id FROM alerts
|
||||
WHERE device_id = $1
|
||||
AND alert_type = 'device_offline'
|
||||
AND is_resolved = false
|
||||
LIMIT 1`,
|
||||
[device.id]
|
||||
);
|
||||
|
||||
if (existing.rows.length === 0) {
|
||||
await client.query(
|
||||
`INSERT INTO alerts (organization_id, device_id, alert_type, severity, title, message)
|
||||
VALUES ($1, $2, 'device_offline', 'warning', $3, $4)`,
|
||||
[
|
||||
device.organization_id,
|
||||
device.id,
|
||||
`Device "${device.name}" is offline`,
|
||||
`Device "${device.name}" has not sent a heartbeat in over ${STALE_THRESHOLD_MS / 1000} seconds.`,
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (staleDevices.length > 0) {
|
||||
logger.info("Marked devices offline", { count: staleDevices.length });
|
||||
}
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function checkDiskAlerts(pool: pg.Pool) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const { rows: criticalDevices } = await client.query(
|
||||
`SELECT d.id, d.name, d.organization_id, m.disk_usage_percent
|
||||
FROM devices d
|
||||
JOIN LATERAL (
|
||||
SELECT disk_usage_percent
|
||||
FROM metric_snapshots
|
||||
WHERE device_id = d.id
|
||||
ORDER BY reported_at DESC
|
||||
LIMIT 1
|
||||
) m ON true
|
||||
WHERE m.disk_usage_percent >= $1
|
||||
AND d.status = 'online'
|
||||
AND d.is_disabled = false`,
|
||||
[DISK_CRITICAL_PERCENT]
|
||||
);
|
||||
|
||||
for (const device of criticalDevices) {
|
||||
const existing = await client.query(
|
||||
`SELECT id FROM alerts
|
||||
WHERE device_id = $1
|
||||
AND alert_type = 'disk_critical'
|
||||
AND is_resolved = false
|
||||
LIMIT 1`,
|
||||
[device.id]
|
||||
);
|
||||
|
||||
if (existing.rows.length === 0) {
|
||||
await client.query(
|
||||
`INSERT INTO alerts (organization_id, device_id, alert_type, severity, title, message)
|
||||
VALUES ($1, $2, 'disk_critical', 'error', $3, $4)`,
|
||||
[
|
||||
device.organization_id,
|
||||
device.id,
|
||||
`Critical disk usage on "${device.name}"`,
|
||||
`Disk usage is at ${device.disk_usage_percent}% on device "${device.name}".`,
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (criticalDevices.length > 0) {
|
||||
logger.info("Created disk alerts", { count: criticalDevices.length });
|
||||
}
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAgentOutdated(pool: pg.Pool) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const CURRENT_AGENT_VERSION = "0.1.0";
|
||||
|
||||
const { rows: outdatedDevices } = await client.query(
|
||||
`SELECT id, name, organization_id, agent_version
|
||||
FROM devices
|
||||
WHERE agent_version < $1
|
||||
AND status = 'online'
|
||||
AND is_disabled = false`,
|
||||
[CURRENT_AGENT_VERSION]
|
||||
);
|
||||
|
||||
for (const device of outdatedDevices) {
|
||||
const existing = await client.query(
|
||||
`SELECT id FROM alerts
|
||||
WHERE device_id = $1
|
||||
AND alert_type = 'agent_outdated'
|
||||
AND is_resolved = false
|
||||
LIMIT 1`,
|
||||
[device.id]
|
||||
);
|
||||
|
||||
if (existing.rows.length === 0) {
|
||||
await client.query(
|
||||
`INSERT INTO alerts (organization_id, device_id, alert_type, severity, title, message)
|
||||
VALUES ($1, $2, 'agent_outdated', 'info', $3, $4)`,
|
||||
[
|
||||
device.organization_id,
|
||||
device.id,
|
||||
`Agent outdated on "${device.name}"`,
|
||||
`Device "${device.name}" is running agent v${device.agent_version}, current is v${CURRENT_AGENT_VERSION}.`,
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function dispatchJobs(pool: pg.Pool) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const { rows: queuedJobs } = await client.query(
|
||||
`UPDATE jobs SET status = 'dispatched', dispatched_at = NOW()
|
||||
WHERE status = 'queued'
|
||||
AND (expires_at IS NULL OR expires_at > NOW())
|
||||
RETURNING id, job_type`
|
||||
);
|
||||
|
||||
for (const job of queuedJobs) {
|
||||
await client.query(
|
||||
`UPDATE job_targets SET status = 'dispatched'
|
||||
WHERE job_id = $1 AND status = 'queued'`,
|
||||
[job.id]
|
||||
);
|
||||
}
|
||||
|
||||
const { rows: expiredJobs } = await client.query(
|
||||
`UPDATE jobs SET status = 'expired', completed_at = NOW()
|
||||
WHERE status IN ('queued', 'dispatched')
|
||||
AND expires_at IS NOT NULL
|
||||
AND expires_at < NOW()
|
||||
RETURNING id`
|
||||
);
|
||||
|
||||
for (const job of expiredJobs) {
|
||||
await client.query(
|
||||
`UPDATE job_targets SET status = 'expired', completed_at = NOW()
|
||||
WHERE job_id = $1 AND status IN ('queued', 'dispatched')`,
|
||||
[job.id]
|
||||
);
|
||||
}
|
||||
|
||||
if (queuedJobs.length > 0) {
|
||||
logger.info("Dispatched jobs", { count: queuedJobs.length });
|
||||
}
|
||||
if (expiredJobs.length > 0) {
|
||||
logger.info("Expired jobs", { count: expiredJobs.length });
|
||||
}
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function dispatchDeployments(pool: pg.Pool) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const { rows: approvedDeployments } = await client.query(
|
||||
`SELECT id, name, job_type, parameters, organization_id, created_by_user_id
|
||||
FROM deployments
|
||||
WHERE status = 'approved'
|
||||
AND (scheduled_at IS NULL OR scheduled_at <= NOW())`
|
||||
);
|
||||
|
||||
for (const deployment of approvedDeployments) {
|
||||
await client.query(
|
||||
`UPDATE deployments SET status = 'running', started_at = NOW()
|
||||
WHERE id = $1`,
|
||||
[deployment.id]
|
||||
);
|
||||
|
||||
const { rows: targets } = await client.query(
|
||||
`SELECT id, device_id FROM deployment_targets
|
||||
WHERE deployment_id = $1 AND status = 'pending'`,
|
||||
[deployment.id]
|
||||
);
|
||||
|
||||
for (const target of targets) {
|
||||
const jobResult = await client.query(
|
||||
`INSERT INTO jobs (organization_id, created_by_user_id, job_type, parameters, status)
|
||||
VALUES ($1, $2, $3, $4, 'queued')
|
||||
RETURNING id`,
|
||||
[deployment.organization_id, deployment.created_by_user_id, deployment.job_type, JSON.stringify(deployment.parameters)]
|
||||
);
|
||||
|
||||
const jobId = jobResult.rows[0].id;
|
||||
|
||||
await client.query(
|
||||
`INSERT INTO job_targets (job_id, device_id, status)
|
||||
VALUES ($1, $2, 'queued')`,
|
||||
[jobId, target.device_id]
|
||||
);
|
||||
|
||||
await client.query(
|
||||
`UPDATE deployment_targets SET job_id = $1, status = 'running', started_at = NOW()
|
||||
WHERE id = $2`,
|
||||
[jobId, target.id]
|
||||
);
|
||||
}
|
||||
|
||||
logger.info("Dispatched deployment", { deploymentId: deployment.id, name: deployment.name, targetCount: targets.length });
|
||||
}
|
||||
|
||||
const { rows: runningDeployments } = await client.query(
|
||||
`SELECT d.id, d.total_targets,
|
||||
(SELECT COUNT(*) FROM deployment_targets dt WHERE dt.deployment_id = d.id AND dt.status = 'succeeded') as completed,
|
||||
(SELECT COUNT(*) FROM deployment_targets dt WHERE dt.deployment_id = d.id AND dt.status = 'failed') as failed,
|
||||
(SELECT COUNT(*) FROM deployment_targets dt WHERE dt.deployment_id = d.id AND dt.status IN ('pending', 'running')) as active
|
||||
FROM deployments d
|
||||
WHERE d.status = 'running'`
|
||||
);
|
||||
|
||||
for (const d of runningDeployments) {
|
||||
if (Number(d.active) === 0) {
|
||||
const finalStatus = Number(d.failed) > 0 ? 'partially_completed' : 'completed';
|
||||
await client.query(
|
||||
`UPDATE deployments SET status = $1, completed_at = NOW(), completed_targets = $2, failed_targets = $3
|
||||
WHERE id = $4`,
|
||||
[finalStatus, Number(d.completed), Number(d.failed), d.id]
|
||||
);
|
||||
|
||||
await client.query(
|
||||
`UPDATE deployment_targets SET status = 'expired', completed_at = NOW()
|
||||
WHERE deployment_id = $1 AND status = 'pending'`,
|
||||
[d.id]
|
||||
);
|
||||
|
||||
logger.info("Deployment finished", { deploymentId: d.id, status: finalStatus });
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function evaluateCompliance(pool: pg.Pool) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const { rows: orgs } = await client.query(
|
||||
`SELECT DISTINCT organization_id FROM policies WHERE is_enabled = true`
|
||||
);
|
||||
|
||||
let totalEvaluated = 0;
|
||||
let totalNonCompliant = 0;
|
||||
|
||||
for (const org of orgs) {
|
||||
const { rows: devices } = await client.query(
|
||||
`SELECT id FROM devices
|
||||
WHERE organization_id = $1
|
||||
AND is_disabled = false AND is_archived = false`,
|
||||
[org.organization_id]
|
||||
);
|
||||
|
||||
const { rows: policies } = await client.query(
|
||||
`SELECT id, rules FROM policies
|
||||
WHERE organization_id = $1 AND is_enabled = true`,
|
||||
[org.organization_id]
|
||||
);
|
||||
|
||||
if (devices.length === 0 || policies.length === 0) continue;
|
||||
|
||||
const deviceIds = devices.map((d: any) => d.id);
|
||||
|
||||
const metricsMap = new Map<string, any>();
|
||||
const inventoryMap = new Map<string, any>();
|
||||
|
||||
if (deviceIds.length > 0) {
|
||||
const { rows: metricsRows } = await client.query(
|
||||
`SELECT DISTINCT ON (device_id) device_id,
|
||||
cpu_usage_percent, memory_usage_percent, disk_usage_percent,
|
||||
load_average_1m, uptime_seconds
|
||||
FROM metric_snapshots WHERE device_id = ANY($1)
|
||||
ORDER BY device_id, reported_at DESC`,
|
||||
[deviceIds]
|
||||
);
|
||||
for (const r of metricsRows) metricsMap.set(r.device_id, r);
|
||||
|
||||
const { rows: invRows } = await client.query(
|
||||
`SELECT DISTINCT ON (device_id) device_id, packages, services
|
||||
FROM inventory_snapshots WHERE device_id = ANY($1)
|
||||
ORDER BY device_id, reported_at DESC`,
|
||||
[deviceIds]
|
||||
);
|
||||
for (const r of invRows) inventoryMap.set(r.device_id, r);
|
||||
}
|
||||
|
||||
const { rows: deviceRows } = await client.query(
|
||||
`SELECT id, os_name, os_version, platform, architecture, agent_version,
|
||||
cpu_cores, total_memory_bytes, total_disk_bytes, status,
|
||||
tags, kernel_version
|
||||
FROM devices WHERE id = ANY($1)`,
|
||||
[deviceIds]
|
||||
);
|
||||
|
||||
const deviceMap = new Map<string, any>();
|
||||
for (const d of deviceRows) deviceMap.set(d.id, d);
|
||||
|
||||
const now = new Date();
|
||||
|
||||
for (const device of deviceRows) {
|
||||
const metrics = metricsMap.get(device.id) || null;
|
||||
const inventory = inventoryMap.get(device.id) || null;
|
||||
|
||||
for (const policy of policies) {
|
||||
const rules = (policy.rules || []) as any[];
|
||||
|
||||
let allPassed = true;
|
||||
let hasUnknown = false;
|
||||
const ruleResults: Record<string, unknown> = {};
|
||||
|
||||
for (const rule of rules) {
|
||||
const actual = getField(device, metrics, inventory, rule.type);
|
||||
|
||||
if (actual === undefined) {
|
||||
ruleResults[rule.type] = {
|
||||
condition: rule.condition,
|
||||
expected: rule.value,
|
||||
actual: null,
|
||||
passed: false,
|
||||
reason: "field_not_found",
|
||||
};
|
||||
hasUnknown = true;
|
||||
allPassed = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
const passed = evaluateRule(actual, rule.condition, rule.value);
|
||||
ruleResults[rule.type] = {
|
||||
condition: rule.condition,
|
||||
expected: rule.value,
|
||||
actual,
|
||||
passed,
|
||||
};
|
||||
if (!passed) allPassed = false;
|
||||
}
|
||||
|
||||
let status: string;
|
||||
if (hasUnknown && !allPassed) status = "unknown";
|
||||
else if (allPassed) status = "compliant";
|
||||
else status = "non_compliant";
|
||||
|
||||
await client.query(
|
||||
`INSERT INTO compliance_results (device_id, policy_id, status, details, evaluated_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (device_id, policy_id) DO UPDATE SET
|
||||
status = EXCLUDED.status, details = EXCLUDED.details, evaluated_at = EXCLUDED.evaluated_at`,
|
||||
[device.id, policy.id, status, JSON.stringify(ruleResults), now]
|
||||
);
|
||||
|
||||
totalEvaluated++;
|
||||
if (status === "non_compliant") totalNonCompliant++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (totalEvaluated > 0) {
|
||||
console.log(
|
||||
`Compliance: ${totalEvaluated} evaluations, ${totalNonCompliant} non-compliant.`
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
function getField(
|
||||
device: any,
|
||||
metrics: any,
|
||||
inventory: any,
|
||||
fieldType: string
|
||||
): unknown {
|
||||
switch (fieldType) {
|
||||
case "os_version": return device.os_version;
|
||||
case "os_name": return device.os_name;
|
||||
case "platform": return device.platform;
|
||||
case "architecture": return device.architecture;
|
||||
case "agent_version": return device.agent_version;
|
||||
case "cpu_cores": return device.cpu_cores;
|
||||
case "memory_total": return device.total_memory_bytes;
|
||||
case "disk_total": return device.total_disk_bytes;
|
||||
case "status": return device.status;
|
||||
case "tags": return device.tags || [];
|
||||
case "kernel_version": return device.kernel_version;
|
||||
case "cpu_usage": return metrics?.cpu_usage_percent;
|
||||
case "memory_usage": return metrics?.memory_usage_percent;
|
||||
case "disk_usage": return metrics?.disk_usage_percent;
|
||||
case "load_average": return metrics?.load_average_1m;
|
||||
case "uptime": return metrics?.uptime_seconds;
|
||||
case "package_count": return inventory?.packages?.length ?? 0;
|
||||
case "has_package": return inventory?.packages?.map((p: any) => p.name) ?? [];
|
||||
case "service_count": return inventory?.services?.length ?? 0;
|
||||
default: return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function compareVersions(a: string, b: string): number {
|
||||
const pa = a.split(".").map(Number);
|
||||
const pb = b.split(".").map(Number);
|
||||
for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
|
||||
const na = pa[i] || 0;
|
||||
const nb = pb[i] || 0;
|
||||
if (na > nb) return 1;
|
||||
if (na < nb) return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function evaluateRule(actual: unknown, condition: string, expected: any): boolean {
|
||||
if (actual === undefined || actual === null) {
|
||||
return condition === "not_equals" && expected !== null;
|
||||
}
|
||||
|
||||
switch (condition) {
|
||||
case "equals": return String(actual) === String(expected);
|
||||
case "not_equals": return String(actual) !== String(expected);
|
||||
case "greater_than": return Number(actual) > Number(expected);
|
||||
case "less_than": return Number(actual) < Number(expected);
|
||||
case "greater_than_or_equal": return Number(actual) >= Number(expected);
|
||||
case "less_than_or_equal": return Number(actual) <= Number(expected);
|
||||
case "contains":
|
||||
if (Array.isArray(actual)) return actual.includes(String(expected));
|
||||
return String(actual).includes(String(expected));
|
||||
case "not_contains":
|
||||
if (Array.isArray(actual)) return !actual.includes(String(expected));
|
||||
return !String(actual).includes(String(expected));
|
||||
case "in":
|
||||
if (Array.isArray(expected)) return expected.map(String).includes(String(actual));
|
||||
return String(actual) === String(expected);
|
||||
case "not_in":
|
||||
if (Array.isArray(expected)) return !expected.map(String).includes(String(actual));
|
||||
return String(actual) !== String(expected);
|
||||
case "version_gte": return compareVersions(String(actual), String(expected)) >= 0;
|
||||
case "version_lte": return compareVersions(String(actual), String(expected)) <= 0;
|
||||
case "version_gt": return compareVersions(String(actual), String(expected)) > 0;
|
||||
case "version_lt": return compareVersions(String(actual), String(expected)) < 0;
|
||||
case "is_true": return actual === true || actual === "true";
|
||||
case "is_false": return actual === false || actual === "false";
|
||||
case "is_empty":
|
||||
if (Array.isArray(actual)) return actual.length === 0;
|
||||
if (typeof actual === "string") return actual.length === 0;
|
||||
return actual === null || actual === undefined;
|
||||
case "is_not_empty":
|
||||
if (Array.isArray(actual)) return actual.length > 0;
|
||||
if (typeof actual === "string") return actual.length > 0;
|
||||
return actual !== null && actual !== undefined;
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function runChecks(pool: pg.Pool) {
|
||||
try {
|
||||
await checkDeviceStatuses(pool);
|
||||
await checkDiskAlerts(pool);
|
||||
await checkAgentOutdated(pool);
|
||||
await dispatchJobs(pool);
|
||||
await dispatchDeployments(pool);
|
||||
await evaluateCompliance(pool);
|
||||
} catch (err) {
|
||||
logger.error("Worker check error", {}, err as Error);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
logger.info("Fleet Forge Worker starting", {
|
||||
staleThreshold: `${STALE_THRESHOLD_MS / 1000}s`,
|
||||
diskCriticalThreshold: `${DISK_CRITICAL_PERCENT}%`,
|
||||
checkInterval: `${CHECK_INTERVAL_MS / 1000}s`,
|
||||
});
|
||||
|
||||
const pool = new pg.Pool({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
});
|
||||
|
||||
await pool.query("SELECT 1");
|
||||
logger.info("Connected to PostgreSQL");
|
||||
|
||||
await runChecks(pool);
|
||||
|
||||
setInterval(() => runChecks(pool), CHECK_INTERVAL_MS);
|
||||
|
||||
const shutdown = async () => {
|
||||
logger.info("Worker shutting down");
|
||||
await pool.end();
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on("SIGINT", shutdown);
|
||||
process.on("SIGTERM", shutdown);
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
logger.error("Worker failed to start", {}, err as Error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
type LogLevel = "debug" | "info" | "warn" | "error";
|
||||
|
||||
interface LogEntry {
|
||||
timestamp: string;
|
||||
level: LogLevel;
|
||||
message: string;
|
||||
context?: Record<string, unknown>;
|
||||
error?: {
|
||||
name: string;
|
||||
message: string;
|
||||
stack?: string;
|
||||
};
|
||||
}
|
||||
|
||||
function formatEntry(entry: LogEntry): string {
|
||||
return JSON.stringify(entry);
|
||||
}
|
||||
|
||||
function createLogger(defaultContext?: Record<string, unknown>) {
|
||||
const log = (level: LogLevel, message: string, context?: Record<string, unknown>, error?: Error) => {
|
||||
const entry: LogEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
level,
|
||||
message,
|
||||
...((defaultContext || context) && { context: { ...defaultContext, ...context } }),
|
||||
};
|
||||
|
||||
if (error) {
|
||||
entry.error = {
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
stack: error.stack,
|
||||
};
|
||||
}
|
||||
|
||||
const line = formatEntry(entry);
|
||||
|
||||
switch (level) {
|
||||
case "error":
|
||||
console.error(line);
|
||||
break;
|
||||
case "warn":
|
||||
console.warn(line);
|
||||
break;
|
||||
default:
|
||||
console.log(line);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
debug: (message: string, context?: Record<string, unknown>) =>
|
||||
log("debug", message, context),
|
||||
info: (message: string, context?: Record<string, unknown>) =>
|
||||
log("info", message, context),
|
||||
warn: (message: string, context?: Record<string, unknown>) =>
|
||||
log("warn", message, context),
|
||||
error: (message: string, context?: Record<string, unknown>, error?: Error) =>
|
||||
log("error", message, context, error),
|
||||
child: (childContext: Record<string, unknown>) =>
|
||||
createLogger({ ...defaultContext, ...childContext }),
|
||||
};
|
||||
}
|
||||
|
||||
export const logger = createLogger({ service: "fleetforge-worker" });
|
||||
export { createLogger };
|
||||
Reference in New Issue
Block a user