1import { getPodHealthInfo, withPortForward, getConsensusSyncStatus, fmtPodDownErr } from './ethStatusCoreAI.mts'
2import { allPromCalls } from '../../sharedF/moDashF/promF/allPromCallsF.mts'
3import { chalkGreen, chalkRed } from '../../serverF/libChalkF.mts'
4import { getOfflineReason } from '../k8sF/ctxF/klusterReachabilityAI.mts'
5
6type EthStatusCfg = {
7  cluster_name: string
8  consClient: {fuzzyPodName: string, portNo: number}
9}
10
11type SyncLagResult = {
12  cfg: EthStatusCfg
13  isSynced?: boolean
14  headSlot?: number
15  syncDistance?: number
16  error?: string
17  offline?: string
18}
19
20const checkEthSyncLags = async ({ethStatusCfgs}: {ethStatusCfgs: EthStatusCfg[]}): Promise<SyncLagResult[]> => {
21  return await allPromCalls(ethStatusCfgs, async (cfg): Promise<SyncLagResult> => {
22    const offlineReason = getOfflineReason(cfg.cluster_name)
23    if (offlineReason) return {cfg, offline: offlineReason}
24    let podHealth: any
25    try {
26      podHealth = await getPodHealthInfo({cluster_name: cfg.cluster_name, fuzzyPodName: cfg.consClient.fuzzyPodName})
27      if (!podHealth.isRunning || !podHealth.podName) return {cfg, error: `${cfg.consClient.fuzzyPodName} ${podHealth.status}`}
28      const status = await withPortForward({
29        cluster_name: cfg.cluster_name, podName: podHealth.podName, portNo: cfg.consClient.portNo,
30        checkerFn: getConsensusSyncStatus,
31      })
32      return {cfg, ...status.syncStatus}
33    } catch (err) {
34      return {cfg, error: fmtPodDownErr({podHealth, errMsg: (err as Error).message})}
35    }
36  })
37}
38
39const formatLag = (r: SyncLagResult) => {
40  const name = `${r.cfg.cluster_name} ${r.cfg.consClient.fuzzyPodName}`
41  if (r.offline) return `${name}: ${chalkRed(`offline (${r.offline})`)}`
42  if (r.error) return `${name}: ${chalkRed(r.error)}`
43  if (r.isSynced) return `${name}: ${chalkGreen('synced')}`
44  return `${name}: ${chalkRed(`-${r.syncDistance} slots`)}`
45}
46
47export const mkEthSyncMonitorJob = ({ethStatusCfgs, maxLagSlots = 64, intervalSec = 60 * 30}: {
48  ethStatusCfgs: EthStatusCfg[], maxLagSlots?: number, intervalSec?: number,
49}) => ({
50  name: 'ethSyncMonitor',
51  intervalSec,
52  runFnc: async () => {
53    const results = await checkEthSyncLags({ethStatusCfgs})
54    const behind = results.filter((r: SyncLagResult) => r.syncDistance != null && r.syncDistance > maxLagSlots)
55    const down = results.filter((r: SyncLagResult) => r.error)
56    if (behind.length || down.length) {
57      throw new Error(`eth sync: ${[...down, ...behind].map(formatLag).join(', ')}`)
58    }
59  },
60})
61
62export const mkSyncLagCheck = ({maxLagSlots = 64}: {maxLagSlots?: number} = {}) => {
63  return async ({ethStatusCfgs}: {ethStatusCfgs?: EthStatusCfg[]} = {}) => {
64    if (!ethStatusCfgs?.length) return
65    const results = await checkEthSyncLags({ethStatusCfgs})
66    const hasIssues = results.some((r) => r.error || (r.syncDistance != null && r.syncDistance > maxLagSlots))
67    const summary = results.map(formatLag).join('\n  ')
68    console.log(`\n${hasIssues ? chalkRed('Sync Lag') : chalkGreen('Sync')}\n  ${summary}`)
69  }
70}
71