1import { create } from 'kubo-rpc-client'
2import { CID } from 'multiformats/cid'
3import fs from 'fs/promises'
4import path from 'path'
5import os from 'os'
6import * as _ from 'lodash-es'
7import { getEnvConf } from '../../serverF/envConfF.mts'
8import { toPtRelPath } from '../../serverF/pathF/absPathToPtPathF.mts'
9import { chainPromiseCalls } from '../../sharedF/moDashF/promF/moPromisesF.mts'
10import { runMemoTempfileWithHit } from '../../serverF/ptDiskF/memoTmpfileF.mts'
11import { calcHash } from '../../serverF/calcHashF.mts'
12import { ipfsApiUrl as hetzn1IpfsApiUrl, ipfsGatewayUrl as hetzn1IpfsGatewayUrl } from '../../klustersF/hetzn1/epSyncIpfs.mjs'
13
14const getIpfsApiUrl = () => getEnvConf().ipfsApiUrl || (process.env.OCWEB === '1' && 'http://ipfs:5001') || hetzn1IpfsApiUrl
15const getIpfsGatewayUrl = () => getEnvConf().ipfsGatewayUrl || hetzn1IpfsGatewayUrl
16
17export const ipfsGatewayLink = (cid: string, gatewayUrl?: string) => `${gatewayUrl || getIpfsGatewayUrl()}/${cid}`
18
19const getFilesRecursive = async (dir: string, baseDir = dir): Promise<{fullPath: string, relativePath: string}[]> => {
20  const entries = await fs.readdir(dir, {withFileTypes: true})
21  const results = await Promise.all(entries.map(async (entry) => {
22    const fullPath = path.join(dir, entry.name)
23    if (entry.isDirectory()) return getFilesRecursive(fullPath, baseDir)
24    const relativePath = path.relative(baseDir, fullPath)
25    return [{fullPath, relativePath}]
26  }))
27  return _.flatten(results)
28}
29
30export const ipfsUploadDir = async ({dirPath, apiUrl = getIpfsApiUrl()}: {dirPath: string, apiUrl?: string}) => {
31  const client = create({url: apiUrl})
32  const logPath = path.join(os.tmpdir(), `ipfs-upload-${Date.now()}.log`)
33  const logLines: string[] = []
34  const stats = {cached: 0, reuploaded: 0, new: 0}
35  
36  console.log(`Uploading ${toPtRelPath(dirPath)} to IPFS...`)
37  
38  const allFiles = await getFilesRecursive(dirPath)
39  const files = _.filter(allFiles, f => path.basename(f.fullPath) !== '.DS_Store')
40  
41  const mfsRoot = `/ens-site-${Date.now()}`
42  
43  const fileCids: {relativePath: string, cid: string}[] = []
44  for (let index = 0; index < files.length; index++) {
45    const {fullPath, relativePath} = files[index]
46    const content = await fs.readFile(fullPath)
47    const contentHash = calcHash(content.toString('base64'), 'sha256')
48    
49    const {cacheHit, value: cachedCid} = await runMemoTempfileWithHit({cacheKeyA: ['ipfsUploadDir', contentHash]}, async () => {
50      const result = await client.add(content, {pin: false, cidVersion: 1})
51      return result.cid.toString()
52    })
53    
54    let cid = cachedCid, status = 'new'
55    if (cacheHit) {
56      const cidObj = CID.parse(cachedCid)
57      const existsOnNode = await client.block.stat(cidObj).then(() => true).catch(() => false)
58      if (existsOnNode) {
59        status = 'cached'
60        stats.cached++
61      } else {
62        const result = await client.add(content, {pin: false, cidVersion: 1})
63        cid = result.cid.toString()
64        status = 're-uploaded'
65        stats.reuploaded++
66      }
67    } else {
68      stats.new++
69    }
70    logLines.push(`[${index + 1}/${files.length}] ${relativePath} → ${cid} (${status})`)
71    fileCids.push({relativePath, cid})
72  }
73  
74  await fs.writeFile(logPath, logLines.join('\n'))
75  const statParts = []
76  if (stats.cached) statParts.push(`${stats.cached} cached`)
77  if (stats.reuploaded) statParts.push(`${stats.reuploaded} re-uploaded`)
78  if (stats.new) statParts.push(`${stats.new} new`)
79  console.log(`  ${statParts.join(', ')} (log: ${logPath})`)
80  
81  for (const {relativePath, cid} of fileCids) {
82    const mfsPath = `${mfsRoot}/${relativePath}`
83    const mfsDir = path.dirname(mfsPath)
84    await client.files.mkdir(mfsDir, {parents: true}).catch(() => {})
85    await client.files.cp(`/ipfs/${cid}`, mfsPath)
86  }
87  
88  const stat = await client.files.stat(mfsRoot)
89  const rootCid = CID.parse(stat.cid.toString()).toV1().toString()
90  
91  let pinned = false
92  try {
93    for await (const _ of client.pin.ls({paths: [rootCid]})) { pinned = true; break }
94  } catch {}
95  if (pinned) {
96    console.log(`Already pinned: ${rootCid}`)
97  } else {
98    await client.pin.add(rootCid)
99    console.log(`Pinned: ${rootCid}`)
100  }
101  try { for await (const _ of client.routing.provide(CID.parse(rootCid))) {} } catch {} // catch:userapproved
102  await client.files.rm(mfsRoot, {recursive: true}).catch(() => {})
103  return {rootCid, fileCids}
104}
105