Read files exported by Prometheus

Question

Periodic exports to a distributed block store have been configured on Prometheus. The export is in custom Prometheus/TSDB format.

I want to read these exports (without Prometheus/Thanos) and then process the aggregations in the backend.

Unable to find any libraries that help with reading the files directly. Any library in Python, Node, Go, or even command line tool would work for me.

Any pointers would be helpful.

Answer 1

You should have a folder with this structure:

the estructure of the files

I'm using Go version 1.19

The code:

go.mod

module tsdb

go 1.19

require (
    github.com/go-kit/kit v0.12.0
    github.com/prometheus/prometheus v0.35.0
)

main.go

package main

import (
    log "github.com/go-kit/kit/log"
    "fmt"

    "github.com/prometheus/prometheus/model/labels"
    "github.com/prometheus/prometheus/storage"
    "github.com/prometheus/prometheus/tsdb"
    "github.com/prometheus/prometheus/tsdb/chunkenc"
)


func main() {
  
  //
  // FIRST WAY TO QUERY USING LabelValues(name)
  //
  var logger log.Logger
  r, _ := tsdb.OpenBlock(logger, "01GQFJ9DSFCCF3138X611DDE4R", chunkenc.NewPool())
  labelNames, _ := r.LabelNames()

  res, _ := r.Index()
  fmt.Println(res.LabelNames())

  for _, label := range labelNames {

      values, _ := res.LabelValues(label)

      fmt.Printf("LABEL:   %v | VALUES:   %v  \n\n", label, values)
  }
  
  
  //
  // SECOND WAY TO QUERY USING Select([]matcher)
  //
  s, _ := tsdb.NewBlockQuerier(r, 0, 10)
  match := labels.MustNewMatcher(labels.MatchEqual, "app", "prometheus")

  var sto storage.SelectHints
  sto.Start = r.MinTime() // 1674475200587 <- minTime that is in the meta.json file
  sto.End = r.MaxTime()   // 1674482400000 <- maxTime that is in the meta.json file

  ss := s.Select(false, &sto, match)

  for ss.Next() {
      series := ss.At()
      fmt.Println("series:", series.Labels().String())

      it := series.Iterator()
      for it.Next() {
          i, v := it.At()
          fmt.Println("sample", i, v)
      }

      fmt.Println("it.Err():", it.Err())
  }

  fmt.Println("\n\n\nTOTAL NUMSERIES: ", r.Meta().Stats.NumSeries)
}

func noErr(err error) {
    if err != nil {
        panic(err)
    }
}

For more inf: see doc https://pkg.go.dev/github.com/prometheus/[email protected]/tsdb#section-readme