1// Generic test runner core - can be used by any app with eptNextjsApp or similar
2import * as _ from 'lodash-es'
3import { chalkRed, chalkGreen, chalkGray, chalkCyan, chalkYellow } from '../../serverF/libChalkF.mts'
4import { appCfgCtx, getAppCfg} from '../k8sF/ctxF/appCfgCtxF.mts'
5import { startDevServer } from '../../nextF/deployF/startDevServerAI.mts'
6import { testS3EndpointCtx } from '../../serverF/testS3EndpointCtxF.mts'
7import { parseCli, cliHasArg } from '../../serverF/cliArgAI.mts'
8import { recordTestResult, printTestHistory, getLastPassingSha } from './recordTestResultAI.mts'
9import { toPtRelPath } from '../../serverF/pathF/absPathToPtPathF.mts'
10import { getTimingHint } from './actionResultsAI.mts'
11import { shouldSkipTests, printSkipMsg } from './testMemoAI.mts'
12import { ptenvLocal, ptenvTestprod, ptenvEnum, type Ptenv } from '../../sharedF/ptenvF.mts'
13import { checkPodsHealthy } from '../k8sF/checkPodsHealthyAI.mts'
14import { assertTruthy } from '../../sharedF/assertsF/assertTruthyAI.mts'
15import { throwIf } from '../../sharedF/throwErrorF/throwIfF.mts'
16import { assertIncludes } from '../../sharedF/assertsF/assertIncludesF.mts'
17import { assertDefined } from '../../sharedF/assertsF/assertDefinedF.mts'
18import { silenceEtherealCtx } from '../../serverF/emailF/silenceEtherealCtxAI.mts'
19import { setEnvConf } from '../../serverF/envConfF.mts'
20import { bisectWorktreePrefix, bisectHintCmd } from '../constantsF.mts'
21import { gqlFetch, getTestFragment, sumAiUsage, type TestResultItem } from './runTestsHelperAI.mts'
22import { portForwardMgr } from '../../serverF/kubePortFwdF/portForwardMgrAI.mts'
23
24
25const schemaErrPatternsAI = [
26  /column "[\w_]+" (of relation "[\w_]+" )?does not exist/i,
27  /relation "[\w_]+" does not exist/i,
28]
29const hasSchemaErr = (results: Record<string, SuiteResult>) => {
30  for (const r of Object.values(results)) {
31    if (r.passed) continue
32    const txt = (r.error?.message || '') + (r.message || '')
33    if (schemaErrPatternsAI.some(p => p.test(txt))) return true
34  }
35  return false
36}
37
38export type SuiteResult = {
39  passed: boolean
40  message: string
41  testResults: TestResultItem[]
42  skipped?: boolean
43  error?: Error
44  durationSec?: number
45}
46
47type StopServerFnc = (() => Promise<void>) | { stop: () => Promise<void>; getOutput: () => string }
48
49export type SuiteRunnerCtx = {
50  ptenv: Ptenv
51  singleTestName?: string  // partial name match for --test=suite:name_fragment
52  limit: number
53  verbose: boolean
54  stopServerRef: { current: StopServerFnc | null }
55  appCfg: any
56  baseUrl: string | undefined
57  mockEth: boolean  // Use mock payments instead of real blockchain txs (default: true for ptenv=local)
58  topSuiteName?: string  // top-level suite name for --test= hint in testline output
59  isTestdeploy?: boolean  // true when invoked via testdeploy action
60}
61
62export type SuiteRunner = (ctx: SuiteRunnerCtx) => Promise<SuiteResult>
63
64export type SuiteDep = 'server' | 'db'
65
66export type SuiteConfig = {
67  name: string
68  runner: SuiteRunner
69  deps?: readonly SuiteDep[]  // empty/undefined = suite manages everything; 'server' = start dev server; 'db' = DB wrapper (implies server)
70  timeoutMs?: number          // per-suite timeout, defaults to 15min
71}
72
73export const defaultTestSuiteTimeoutMs = 15 * 60 * 1000 // 15 min
74
75export const combineSuites = (runners: SuiteRunner[]): SuiteRunner => async (ctx) => {
76  const allResults: SuiteResult['testResults'] = []
77  let allPassed = true
78  for (const runner of runners) {
79    const result = await runner(ctx)
80    if (result.skipped) continue
81    allResults.push(...result.testResults)
82    if (!result.passed) allPassed = false
83  }
84  return { passed: allPassed, message: allPassed ? 'all passed' : 'some failed', testResults: allResults }
85}
86
87export type RunTestsCoreConfig = {
88  allSuites: string[]
89  localAppCfg: any
90  suiteConfigs: SuiteConfig[]
91  wrapDbEnv?: (opts: { appConfig: any; qsName?: string; reuseConn: boolean; assumeDaemon?: boolean; wrappedFnc: () => Promise<void> }) => Promise<void>
92  preDeployedTests?: () => Promise<void>
93  getFlags?: () => Record<string, boolean>
94  historyFlags?: string[]
95}
96
97type SingleTest = { suite: string; nameFragment: string }
98
99const parseSingleTest = (testArg: string | undefined): SingleTest | null => {
100  if (!testArg) return null
101  // Format: suite:name_fragment (e.g., "donate:sender_addr")
102  const colonIdx = testArg.indexOf(':')
103  throwIf(() => colonIdx === -1, {testArg})
104  const suite = testArg.slice(0, colonIdx)
105  const nameFragment = testArg.slice(colonIdx + 1)
106  assertTruthy(suite, {testArg})
107  assertTruthy(nameFragment, {testArg})
108  return { suite, nameFragment }
109}
110
111const runSuite = async (name: string, fn: () => Promise<SuiteResult>, results: Record<string, SuiteResult>, timeoutMs: number) => {
112  const startTime = Date.now()
113  try {
114    const abortSignal = AbortSignal.timeout(timeoutMs)
115    const timeoutPromise = new Promise<SuiteResult>((_, reject) => {
116      abortSignal.addEventListener('abort', () => reject(new Error(`Suite timeout after ${Math.round(timeoutMs/1000)}s`)))
117    })
118    results[name] = await Promise.race([fn(), timeoutPromise])
119  } catch (err: any) {
120    const debugH = err.uniqDebugH ? JSON.stringify(err.uniqDebugH) : ''
121    const message = debugH ? `${err.message} ${debugH}` : err.message
122    results[name] = { passed: false, message, error: err, testResults: [] }
123  }
124  results[name].durationSec = Math.round((Date.now() - startTime) / 1000)
125  return results[name]
126}
127
128export const baseRuntestsCli = {
129  suite:            { type: 'csv' as const, desc: 'suites to run (comma-separated)' },
130  history:          { flag: true as const, desc: 'show test history' },
131  'no-memo':        { flag: true as const, desc: 'skip memoization, always run tests' },
132  skipExtDep:       { flag: true as const, hidden: true as const, desc: 'skip extDep tests (ptm bisect)' },
133}
134
135export const mkRuntestsCli = (allSuites: string[]) => ({
136  ...baseRuntestsCli,
137  suite:            { type: 'csv' as const, default: allSuites, desc: 'suites to run (comma-separated)' },
138  ptenv:            { type: 'string' as const, default: ptenvLocal, enum: ptenvEnum, desc: 'testlocal|testprod' },
139  limit:            { type: 'int' as const, default: 10 },
140  verbose:          { flag: true as const, desc: 'show all test output' },
141  test:             { type: 'string' as const, hint: 'name', desc: 'run single test by name, e.g. donate:sender_addr' },
142})
143
144// Eth-related flags - only include in apps that have eth payment tests (e.g. donateapp)
145// mockEth defaults to true for local, false for deployed. --real-eth forces real payments.
146export const ethRuntestsCli = {
147  'real-eth':       { flag: true as const, desc: 'force real ETH payments (default: mock for local, real for deployed)' },
148}
149
150export type RunTestsCoreResult = {
151  allPassed: boolean
152  ranSuites: Record<string, SuiteResult>
153  skippedSuites: Record<string, SuiteResult>
154  totalTests: number
155  durationSec?: number
156}
157
158export type RunTestsOpts = {
159  ptenv: Ptenv
160  noMemo: boolean
161  fastFail: boolean
162  singleTest: SingleTest | null
163  testsuites: string[]
164  limit: number
165  verbose: boolean
166  mockEth: boolean
167  isTestdeploy?: boolean
168}
169
170// Core test runner - purely programmatic, no CLI reading. Callers parse CLI and pass opts.
171export const runTestsCore = async ({
172  config,
173  opts,
174}: {
175  config: RunTestsCoreConfig
176  opts: RunTestsOpts
177}): Promise<RunTestsCoreResult> => {
178  const coreStartTime = Date.now()
179  const { allSuites, localAppCfg, suiteConfigs, wrapDbEnv, getFlags } = config
180  const { ptenv, noMemo, fastFail, singleTest, limit, verbose, mockEth, isTestdeploy } = opts
181  let { testsuites } = opts
182
183  // Ensure envConf (including secretsMapping) is available for test helpers like genEmailOtp
184  const appCfgEnvConf = getAppCfg()?.envConf || localAppCfg?.envConf
185  if (appCfgEnvConf) setEnvConf(appCfgEnvConf)
186
187  const ep = getAppCfg()?.importMetaUrl || ''
188  if (!noMemo && !singleTest) {
189    try {
190      const { skip, reason, lastRecord } = await shouldSkipTests({ ep, ptenv })
191      if (skip) {
192        printSkipMsg(ptenv, reason!, lastRecord)
193        return { allPassed: true, ranSuites: {}, skippedSuites: {}, totalTests: 0 }
194      }
195    } catch { /* memo check failed, continue with tests */ } // catch:userapproved
196  }
197
198  if (singleTest) {
199    assertIncludes(allSuites, singleTest.suite, {suite: singleTest.suite, available: allSuites})
200    testsuites = [singleTest.suite]
201  }
202
203  if (ptenv === ptenvTestprod) {
204    const appCfg = getAppCfg() || localAppCfg
205    const { name, cluster_name } = appCfg
206    if (name && cluster_name) {
207      const health = await checkPodsHealthy({ name, cluster_name })
208      if (!health.healthy) {
209        console.log(chalkRed(`Pods not healthy: ${health.message}. Run: ptnode <ep> apply`))
210        return { allPassed: false, ranSuites: {}, skippedSuites: {}, totalTests: 0 }
211      }
212    }
213  }
214
215  const etaHint = singleTest ? '' : ` # ${getTimingHint(ep, 'runtests', {ptenv, suites: testsuites})}`
216  console.log(`${chalkCyan('runtests')} ${chalkGray(`--ptenv=${ptenv} --suite=${testsuites.join(',')}${etaHint}`)}`)
217
218  const results: Record<string, SuiteResult> = {}
219  let allPassed = true
220  let exitEarly = false
221  const stopServerRef: { current: StopServerFnc | null } = { current: null }
222  const singleTestName = singleTest?.nameFragment
223
224    const getBaseUrlAndEnsureServer = async () => {
225      const appCfg = getAppCfg() || localAppCfg
226      const testPort = appCfg.testSvcPortNo ?? (appCfg.svcPortNo || 3000)
227      const baseUrl = ptenv === ptenvLocal ? `http://127.0.0.1:${testPort}` : appCfg.deployedBaseUrl
228    if (ptenv === ptenvLocal && !stopServerRef.current) {
229      appCfgCtx.enterWith(appCfg)
230      stopServerRef.current = await startDevServer({envH: {}})
231      await gqlFetch({baseUrl: baseUrl!, query: '{ gqDeployInfo }', skipTestIp: true})
232    }
233    return { baseUrl, appCfg }
234  }
235
236  // Validate suiteConfigs - fail fast on circular import issues
237  for (let i = 0; i < suiteConfigs.length; i++) {
238    const sc = suiteConfigs[i]
239    assertTruthy(sc?.name && sc?.runner, {suiteConfigsIdx: i, sc})
240  }
241
242  // Build lookup for suite configs
243  const suiteConfigMap = Object.fromEntries(suiteConfigs.map(sc => [sc.name, sc]))
244
245  // Run suites in order
246  const runSuiteWithDeps = async (suiteName: string, suiteConfig: SuiteConfig): Promise<boolean> => {
247    const { runner, deps = [], timeoutMs = defaultTestSuiteTimeoutMs } = suiteConfig
248    // Merge localAppCfg (from app's appCfgF.mjs) with context appCfg (from entrypoint chain)
249    // localAppCfg has test-specific config like healthgqlTestFnc; context has runtime config
250    const appCfg = {...localAppCfg, ...getAppCfg()}
251    appCfgCtx.enterWith(appCfg) // ensure getAppCfg() in suite runners sees merged config
252    const testPort = appCfg.testSvcPortNo ?? (appCfg.svcPortNo || 3000)
253    const defaultBaseUrl = ptenv === ptenvLocal ? `http://127.0.0.1:${testPort}` : appCfg.deployedBaseUrl
254    if (ptenv === ptenvLocal) testS3EndpointCtx.enterWith(defaultBaseUrl) // signed-URL test fetches target the isolated test server
255
256    const doRun = async (baseUrl: string | undefined, cfg: typeof appCfg) => {
257      const ctx: SuiteRunnerCtx = { ptenv, singleTestName, limit, verbose, stopServerRef, appCfg: cfg, baseUrl, mockEth, topSuiteName: suiteName, isTestdeploy }
258      const result = await runSuite(suiteName, () => runner(ctx), results, timeoutMs)
259      return result.passed !== false
260    }
261
262    if (deps.includes('db') && wrapDbEnv) {
263      let passed = false
264      await wrapDbEnv({ 
265        appConfig: appCfg, 
266        qsName: 'defaultdb_qs', 
267        reuseConn: false,
268        assumeDaemon: true,
269        wrappedFnc: async () => {
270          const { baseUrl } = await getBaseUrlAndEnsureServer()
271          passed = await doRun(baseUrl, appCfg)
272        }
273      })
274      return passed
275    }
276    
277    if (deps.includes('server')) {
278      const { baseUrl, appCfg: serverCfg } = await getBaseUrlAndEnsureServer()
279      return doRun(baseUrl, serverCfg)
280    }
281    
282    return doRun(defaultBaseUrl, appCfg)
283  }
284
285  try {
286    for (const suiteName of testsuites) {
287      if (exitEarly) break
288      const suiteConfig = suiteConfigMap[suiteName]
289      assertDefined(suiteConfig, {suiteName, available: Object.keys(suiteConfigMap)})
290      
291      try {
292        allPassed = await runSuiteWithDeps(suiteName, suiteConfig) && allPassed
293      } catch (err: any) {
294        results[suiteName] = { passed: false, message: err.message, testResults: [] }
295        allPassed = false
296        if (fastFail) exitEarly = true
297      }
298    }
299  } finally {
300    portForwardMgr.teardown()
301  }
302
303  const stopFnc = stopServerRef.current
304  if (stopFnc) await (typeof stopFnc === 'function' ? stopFnc() : stopFnc.stop())
305
306  const ranSuites = _.pickBy(results, r => !r.skipped) as Record<string, SuiteResult>
307  const skippedSuites = _.pickBy(results, r => r.skipped) as Record<string, SuiteResult>
308  const passedCount = _.values(ranSuites).filter(r => r.passed).length
309  const totalTests = _.sumBy(_.values(ranSuites), r => r.testResults?.length || 0)
310
311  console.log(chalkCyan('\n' + '='.repeat(40)))
312  for (const [suite, result] of _.toPairs(ranSuites)) {
313    console.log(`${result.passed ? chalkGreen('PASS') : chalkRed('FAIL')} [${suite}] ${result.message}`)
314  }
315  for (const [suite, result] of _.toPairs(skippedSuites)) {
316    console.log(`${chalkYellow('SKIP')} [${suite}] ${result.message}`)
317  }
318  console.log(allPassed ? chalkGreen(`\nAll ${_.size(ranSuites)} suites passed!`) : chalkRed(`\n${passedCount}/${_.size(ranSuites)} suites passed`))
319
320  // Aggregate and print total gas spent across all tests
321  const allTestResults = _.values(ranSuites).flatMap(r => r.testResults || [])
322  const totalGasEth = _.sumBy(allTestResults, r => parseFloat(r.ethGasSpent || '0'))
323  if (totalGasEth > 0) console.log(chalkGray(`runtests gas: ${totalGasEth.toFixed(10)} ETH`))
324  const aiTot = sumAiUsage(allTestResults)
325  if (aiTot.tok > 0) console.log(chalkGray(`runtests ai: ${aiTot.tok} tok, $${aiTot.usd.toFixed(4)}`))
326
327  if (!allPassed && hasSchemaErr(ranSuites)) {
328    console.log(chalkYellow(`HINT: DB schema error detected. Run: ptnode <db_sync.mjs> migrate`))
329  }
330  if (!allPassed) {
331    const lastGoodSha = getLastPassingSha({ep})
332    const epRel = toPtRelPath(ep)
333    const firstFail = _.toPairs(ranSuites).find(([, r]) => !r.passed)
334    const firstFailTest = firstFail ? firstFail[1].testResults.find(t => !t.passed) : undefined
335    const testArg = firstFail && firstFailTest ? `${firstFail[0]}:${getTestFragment(firstFailTest.name)}` : undefined
336    if (lastGoodSha && !process.cwd().includes(bisectWorktreePrefix)) console.log(chalkYellow(`HINT: To bisect: ${bisectHintCmd(epRel, lastGoodSha, testArg)}`))
337  }
338
339  const durationSec = Math.round((Date.now() - coreStartTime) / 1000)
340  const flags = getFlags?.()
341  recordTestResult({ ep, ptenv, ranSuites, durationSec, flags })
342
343  return { allPassed, ranSuites, skippedSuites, totalTests, durationSec }
344}
345
346// Build RunTestsOpts from parsed CLI. Accepts any cli that extends mkRuntestsCli (with or without ethRuntestsCli)
347export const optsFromCli = (cli: ReturnType<typeof parseCli<ReturnType<typeof mkRuntestsCli>>> & Partial<{['real-eth']: boolean}>, allSuites: string[]): RunTestsOpts => {
348  const ptenv = cli.ptenv as Ptenv
349  assertIncludes(ptenvEnum, ptenv, {ptenv})
350  const verbose = cli.verbose ?? false
351  const singleTest = parseSingleTest(cli.test)
352  // mockEth: default true for local, false for deployed. --real-eth forces real payments.
353  const mockEth = cli['real-eth'] ? false : ptenv === ptenvLocal
354  return {
355    ptenv,
356    noMemo: cli['no-memo'] ?? false,
357    fastFail: !verbose,
358    singleTest,
359    testsuites: cli.suite || allSuites,
360    limit: cli.limit ?? 10,
361    verbose,
362    mockEth,
363  }
364}
365
366// Default opts for programmatic callers (like testdeploy) that don't read CLI
367export const defaultRunTestsOpts = (allSuites: string[], ptenv: Ptenv): RunTestsOpts => ({
368  ptenv,
369  noMemo: false,
370  fastFail: true,
371  singleTest: null,
372  testsuites: allSuites,
373  limit: 10,
374  verbose: false,
375  mockEth: ptenv === ptenvLocal,
376})
377
378// CLI wrapper factory for runtests action - handles --history flag
379export const mkRuntests = (config: RunTestsCoreConfig) => {
380  const runtests = async () => {
381    const runtestsCli = mkRuntestsCli(config.allSuites)
382    const cli = parseCli(runtestsCli)
383
384    if (cli.history) {
385      const ep = getAppCfg()?.importMetaUrl
386      printTestHistory({ ep, suites: cli.suite, flagCols: config.historyFlags })
387      return
388    }
389
390    const opts = optsFromCli(cli, config.allSuites)
391    const result = await silenceEtherealCtx.run(true, () => runTestsCore({ config, opts }))
392    if (!result.allPassed) process.exit(1)
393  }
394
395  runtests.cliSchema = mkRuntestsCli(config.allSuites)
396  return runtests
397}
398