# loader.mjs -rw-r--r-- 1.9 KiB View raw
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
import assert from 'node:assert'
import { openAsBlob } from 'node:fs'

// Provides TextDecoderStream for Bun.
function textDecoderStream() {
  if ('TextDecoderStream' in global) {
    return new TextDecoderStream()
  }

  let decoder = new TextDecoder()
  return new TransformStream({
    transform(chunk, controller) {
      let text = decoder.decode(chunk, { stream: true })
      if (text.length > 0) {
        controller.enqueue(text)
      }
    },
    flush(controller) {
      let text = decoder.decode()
      if (text.length > 0) {
        controller.enqueue(text)
      }
    },
  })
}

async function main() {
  let blob = await openAsBlob(process.argv[2])

  // Anything past the final newline in a string chunk is part of an incomplete line and
  // will get saved here.
  let leftover = ''
  let eventsFound = 0

  let rd = blob.stream().pipeThrough(textDecoderStream()).getReader()
  while (true) {
    let { chunk, done } = await rd.read()
    if (chunk === undefined) {
      break
    }

    if (leftover.length !== 0) {
      chunk = leftover + chunk
    }

    let offset = 0
    while (true) {
      let candidatePos = chunk.indexOf('model_version', offset)
      if (candidatePos === -1) {
        break
      }

      let lineEnd = chunk.indexOf('\n', candidatePos)
      if (lineEnd === -1) {
        break
      }

      let lineStart = chunk.lastIndexOf('\n', candidatePos)
      lineStart = lineStart !== -1 ? lineStart + 1 : 0

      let line = chunk.substring(lineStart, lineEnd)
      let event = JSON.parse(line)
      console.log(event)

      eventsFound++
      offset = lineEnd + 1
    }

    let nl = chunk.lastIndexOf('\n')
    if (nl === -1) {
      leftover = chunk
    } else {
      leftover = chunk.substring(nl + 1)
    }

    if (done) {
      break
    }
  }

  // Nothing should be left over past the final newline.
  assert(leftover === '')

  console.log({ eventsFound })
}

await main()