For very large JSON/GeoJSON files, I'd like to create a primitive key/value store that keeps track of the starting positions and lengths of each JSON record in the file. This way, I could look up individual records at a later stage without reading the whole file into memory (Using the fd.read API). Somewhat similar to a super simple database, but read-only and without the extra overhead.

The issue I'm facing is that I don't know how I could determine the starting position and byte length of each JSON record / GeoJSON feature in the original file.

Here's some pseudo-code showcasing what I'm trying to achieve, note that the geojsonStream.parse callback doesn't receive the startByte and length arguments in reality though.

Thanks for your help, also happy about any feedback outlining why this might be a bad idea :)

import geojsonStream from 'geojson-stream'
import { open } from 'fs/promises'
import { Buffer } from 'buffer'

function getFeaturePositionsInFile(fd) {
  return new Promise((resolve,reject) => {
    const featurePositionsInFile = []
    const stream = fd
      .createReadStream()
      .pipe(geojsonStream.parse((building, index, startByte, length) => {
          // The startByte and length callback arguments are not real unfortunately :(
          featurePositionsInFile.push({
            index,
            startPosition,
            length
          })
      }))
     stream.on('end', () => resolve(featurePositionsInFile))
     stream.on('error', () => reject)
  })
}

function readSingleFeatureFromFile(fd, startPosition, length) {
  return new Promise((resolve, reject) => {
    try {
      const buff = Buffer.alloc(length)
      const offset = 0
      const { buffer } = await fd.read(buff, offset, length, startPosition)
      const singleFeature = JSON.parse(buffer.toString())
      resolve(singleFeature)
    } catch (e) {
      reject(e)
    }
  })
}

const fd = await open('buildings.geojson')
const featurePositionsInFile = await getFeaturePositionsInFile(fd)
const featureIndexToRead = 0
const { startPosition, length } = featurePositionsInFile[featureIndexToRead]
const singleFeature = await readSingleFeatureFromFile(fd, startPosition, length)