create packetSource of gopacket with s3 file

Question

when I get a pcap file from s3 client, I need to generate a packetSource of gopacket to read packets in it. But I only found OpenOfflineFile function in gopacket document, how could I do to generate packetSource with []byte(which is read from s3 file).

I have read source code of OpenOfflineFile function in gopacket, but I still get confused because I'm unfamiliar with uintptr, can I just generate a unitptr with []byte and then use it to generate packetSource.

func openOffline(file string) (handle *Handle, err error) {
    err = LoadWinPCAP()
    if err != nil {
        return nil, err
    }

    buf := make([]byte, errorBufferSize)
    f, err := syscall.BytePtrFromString(file)
    if err != nil {
        return nil, err
    }

    var cptr uintptr
    if pcapOpenOfflineWithTstampPrecisionPtr == 0 {
        cptr, _, _ = syscall.Syscall(pcapOpenOfflinePtr, 2, uintptr(unsafe.Pointer(f)), uintptr(unsafe.Pointer(&buf[0])), 0)
    } else {
        cptr, _, _ = syscall.Syscall(pcapOpenOfflineWithTstampPrecisionPtr, 3, uintptr(unsafe.Pointer(f)), uintptr(pcapTstampPrecisionNano), uintptr(unsafe.Pointer(&buf[0])))
    }

    if cptr == 0 {
        return nil, errors.New(byteSliceToString(buf))
    }

    h := &Handle{cptr: pcapTPtr(cptr)}
    return h, nil
}

Answer 1

Try github.com/google/gopacket/pcapgo

If the file format is supported by the package github.com/google/gopacket/pcapgo, considering using it because it makes it easy:

package main

import (
    "bytes"
    "io"
    "log"
    "os"

    "github.com/google/gopacket"
    "github.com/google/gopacket/layers"
    "github.com/google/gopacket/pcapgo"
)

func main() {
    f, err := os.Open("test.pcap")
    if err != nil {
        panic(err)
    }
    // As described in the question, buf is read from S3 file. In order to
    // make this demo simple and executable, we read it from a local file.
    buf, err := io.ReadAll(f)
    if err != nil {
        panic(err)
    }

    // Convert []byte into a reader. The S3 client should give us a reader
    // that we can use directly in the place of the fileReader. Try the best
    // to avoid reading the response as []byte and then convert it into a reader.
    fileReader := bytes.NewReader(buf)

    r, err := pcapgo.NewReader(fileReader)
    if err != nil {
        panic(err)
    }
    source := gopacket.NewPacketSource(r, layers.LayerTypeEthernet)

    for packet := range source.Packets() {
        log.Printf("%v", packet)
    }
}

Use os.Pipe with github.com/google/gopacket/pcap

If the file format is not supported by github.com/google/gopacket/pcapgo and we have to go with github.com/google/gopacket/pcap, a workaround is to create a pipe, and pass the r file to pcap.OpenOfflineFile:

package main

import (
    "bytes"
    "io"
    "log"
    "os"

    "github.com/google/gopacket"
    "github.com/google/gopacket/layers"
    "github.com/google/gopacket/pcap"
)

func main() {
    f, err := os.Open("test.pcap")
    if err != nil {
        panic(err)
    }
    // As described in the question, buf is read from S3 file. In order to
    // make this demo simple and executable, we read it from a local file.
    buf, err := io.ReadAll(f)
    if err != nil {
        panic(err)
    }

    r, w, err := os.Pipe()
    if err != nil {
        panic(err)
    }

    go func() {
        // Convert []byte into a reader. The S3 client should give us a reader
        // that we can use directly in the place of the fileReader. Try the best
        // to avoid reading the response as []byte and then convert it into a reader.
        fileReader := bytes.NewReader(buf)
        _, err := io.Copy(w, fileReader)
        defer w.Close()
        if err != nil {
            panic(err)
        }
    }()

    handle, err := pcap.OpenOfflineFile(r)
    if err != nil {
        panic(err)
    }
    source := gopacket.NewPacketSource(handle, layers.LayerTypeEthernet)

    for packet := range source.Packets() {
        log.Printf("%v", packet)
    }
}

Notes:

1. This is tested on Linux only. But it should work on Windows.
2. github.com/google/gopacket/pcap is a wrapper of libpcap (or winpcap or npcap on Windows). That's why it's a little complicated to work with []byte or io.Reader.
3. When you download a file from S3, the client should give you a reader. You can use the reader directly (see the comments in my demo). Avoid reading from the reader yourself.