trying to record two channels from multi channel audio device into a wav file using the gstreamer c api without gst_parse_launch

Question

I'm trying to record two channels from my Clarret 8pre sound card as stereo to a wav file using the gstreamer api without using gst_launch on MacOS using cpp, but i use the c api of gstreamer.

i was able to perform that from the terminal using:

gst-launch-1.0 osxaudiosrc device=97 ! audioconvert ! deinterleave name=d interleave name=i ! audioconvert ! wavenc ! filesink location=stereo.wav d.src_4 ! queue ! i.sink_0 d.src_5 ! queue ! i.sink_1

but as i learned the gstreamer api is a bit different.

so in general i first connect and link osxaudiosource device=97 ! audioconvert ! audioresample ! capsfilter ! deinterleave and separately i connect interleave ! encoder ! sink

then in the pad-added callback to the deinterleave element, i pass the interleave element as a data, i wait for src_4 and src_5 from deinterleave and connect them to new sinks sink_0 and sink_1 created by gst_element_request_pad_simple(sink, "sink_%u");

the capsfilter is probably not needed, i was trying to see how to overcome the warning the i get which causes the pipeline to not record anything.

WARN              interleave interleave.c:319:gst_interleave_set_channel_positions:<interleave> Invalid channel positions, using NONE

i'm really new to gstreamer, i was able to create a function to record a single channel, now i'm trying to record stereo and encounter this issue, any information regarding it would be greatly appreciated.

this is the full code:

#include <gst/gst.h>

static void on_pad_added(GstElement *element, GstPad *pad, gpointer data);

int main(int argc, char *argv[])
{
    setenv("GST_DEBUG", "3", true);

    // return my_sound_dev::gst_main(reinterpret_cast<GstMainFunc>(my_main), argc, argv);
   GMainLoop *loop;
    GstElement *pipeline, *src, *convert, *deinterleave, *interleave, *resample, *encoder, *sink;
    GstPad *pad, *sinkpad;
    GstCaps *caps;

    gst_init(&argc, &argv);

    src = gst_element_factory_make("osxaudiosrc", "source");
    convert = gst_element_factory_make("audioconvert", "convert");
    resample = gst_element_factory_make("audioresample", "resample");
    deinterleave = gst_element_factory_make("deinterleave", "deinterleave"); 
    GstElement *capsfilter = gst_element_factory_make("capsfilter", "filter");
    caps = gst_caps_from_string("audio/x-raw,channels=20,layout=(string)interleaved");
    //caps = gst_caps_from_string("audio/x-raw,channels=2,channel-mask=(bitmask)0x3,layout=(string)interleaved");
    g_object_set (G_OBJECT (capsfilter), "caps", caps, NULL);

    interleave = gst_element_factory_make("interleave", "interleave");
    encoder = gst_element_factory_make("wavenc", "encoder");
    sink = gst_element_factory_make("filesink", "sink");

    g_object_set(src, "device", 97, NULL);
    pipeline = gst_pipeline_new("audio-pipeline");

    gst_bin_add_many(GST_BIN(pipeline), src, deinterleave, interleave, convert, resample, encoder, sink, capsfilter, NULL);

    g_object_set(sink, "location", "/Users/ufk/stereo.wav", NULL);

    if (!gst_element_link_many(src, convert, resample, capsfilter, deinterleave, NULL))
    {
        g_error("Failed to link elements");
        return -1;
    }

    if(!gst_element_link(interleave, encoder)){
        g_error("Failed to link interleave and encoder");
        return -1;
    }


    if(!gst_element_link(encoder, sink)){
        g_error("Failed to link encoder and sink");
        return -1;
    }

    loop = g_main_loop_new(NULL, FALSE);

    g_signal_connect(deinterleave, "pad-added", G_CALLBACK(on_pad_added), interleave);

    if(gst_element_set_state(pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE){
        g_error("Could not set pipeline to playing state");
        return -1;
    }

    g_main_loop_run(loop);

    gst_element_set_state(pipeline, GST_STATE_NULL);
    gst_object_unref(GST_OBJECT(pipeline));
    g_main_loop_unref(loop);


    return 0;
}

void print_pads(GstElement* element) {
    GValue item = G_VALUE_INIT;
    gboolean done = FALSE;

    GstIterator* it = gst_element_iterate_pads(element);

    while (!done) {
        switch (gst_iterator_next(it, &item)) {
        case GST_ITERATOR_OK: {
                GstPad *pad = GST_PAD(g_value_get_object(&item));
                g_print ("pad name: %s\n", GST_PAD_NAME(pad));
                g_value_reset(&item);
                break;
        }
        case GST_ITERATOR_RESYNC:
            gst_iterator_resync(it);
            break;
        case GST_ITERATOR_ERROR:
        case GST_ITERATOR_DONE:
          done = TRUE;
            break;
        }
    }
    g_value_unset(&item);
    gst_iterator_free(it);
}

void on_pad_added(GstElement *element, GstPad *pad, gpointer data)
{
    char *name;
    GstPad *sinkpad;
    GstElement *queue = gst_element_factory_make("queue", "queue");
    GstElement *sink = (GstElement *)data;

    name = gst_pad_get_name(pad);

    if(g_strcmp0(name, "src_4") == 0){
        sinkpad = gst_element_request_pad_simple(sink, "sink_%u");
    if (!sinkpad)
    {
        g_error("could not create sink for src_4");
    }

        if (gst_pad_is_linked(sinkpad))
        {
            MoLog::fatal() << "sinkpad already linked";
        }
        if (gst_pad_is_linked(pad))
        {
            MoLog::fatal() << "new_pad already linked";
        }
        if (!gst_pad_can_link(pad, sinkpad))
        {
            MoLog::error() << "can't link new_pad to sinkpad";

            GstCaps *new_pad_caps = gst_pad_get_current_caps(pad);
            GstCaps *sinkpad_caps = gst_pad_get_pad_template_caps(sinkpad);
            if (!gst_caps_can_intersect(new_pad_caps, sinkpad_caps)) {
                MoLog::fatal() << "Caps are not compatible";
            } else
            {
                MoLog::info() << "caps can intersect";
            }
            gst_caps_unref(new_pad_caps);
            gst_caps_unref(sinkpad_caps);
        }
        GstPadLinkReturn ret = gst_pad_link(pad, sinkpad);
        if (ret != GST_PAD_LINK_OK){
            print_pads(sink);
            g_error("Error linking pad src_4: %d",ret);
        } else
        {
            g_print("src_4 linked successfully!");
        }

        gst_object_unref(sinkpad);
    }

    else if(g_strcmp0(name, "src_5") == 0){
        sinkpad = gst_element_request_pad_simple(sink, "sink_%u");
        if (!sinkpad)
        {
            g_error("could not create sink for src_5");
        }

        if ((gst_pad_link(pad, sinkpad)) != GST_PAD_LINK_OK){
            g_error("Error linking pad src_5");
        } else
        {
            g_print("successfully linked src_5\n");
        }

        gst_object_unref(sinkpad);
    }

g_free(name);

}

i also have a print_pads(GstElement* element) for debugging to understand when it creates the pads, but i'm quite lost from here. i used queue in the gst-launch but not here.. i'm not sure if i should and if so what's the flow

thanks

Answer 1

so.. it was actually simpler than i thought, it appears that the command like of gst_parse_launch is not that different from the flow of the C implementation, i just needed to better understand what's going on.

first i created a capsfilter for 44100 and S16LE

GstCaps* caps = gst_caps_new_simple("audio/x-raw",
                                         "format", G_TYPE_STRING, "S16LE",
                                         "rate", G_TYPE_INT, bitrate,
                                         NULL);
        GstCaps* fixed = gst_caps_fixate (caps);
        g_object_set (G_OBJECT (capsfilter), "caps", fixed, NULL);

later i want to record it to a wav file using libsndfile instead of filesink so i prepared that before hand.

and my inital flow is this:

 if (!gst_element_link_many(source, convert, resample, capsfilter, deinterleave, NULL)) {
        gst_object_unref(pipeline);
        MoLog::fatal() << "Elements could not be linked.";
    }

it's osxaudiosrc ! audioconvert !audioresample! capsfilter! deinterleave

created to queue elements and two audioresample elements to be used after the queue:

queue1 = gst_element_factory_make("queue", "queue1");
    queue2 = gst_element_factory_make("queue", "queue2");
    resample_after_queue1 = gst_element_factory_make("audioresample", "resample-after-queue1");
    resample_after_queue2 = gst_element_factory_make("audioresample", "resample-after-queue2");

i request the sink pad for each queue:

audio_connected_queue1_sink_pad = gst_element_get_static_pad(queue1, "sink");
        if (!audio_connected_queue1_sink_pad)
        {
            MoLog::fatal() << "interleave sink pad could not be fetched";
            return;
        }

        audio_connected_queue2_sink_pad = gst_element_get_static_pad(queue2, "sink");
        if (!audio_connected_queue2_sink_pad)
        {
            MoLog::fatal() << "interleave sink pad could not be fetched";
            return;
        }

and connect each queue to it's resample and to the interleave and the interleave to the sink

    if (!gst_element_link_many(queue1, resample_after_queue1, interleave, NULL)) {
        gst_object_unref(pipeline);
        MoLog::fatal() << "Elements could not be linked.";
    }

    if (!gst_element_link_many(queue2, resample_after_queue2, interleave, NULL)) {
        gst_object_unref(pipeline);
        MoLog::fatal() << "Elements could not be linked.";
    }

    if (!gst_element_link(interleave, sink))
    {
        gst_object_unref(pipeline);
        MoLog::fatal() << "could not link interleave and sink.";
    }

connnected to the pad-added signal of the deinterleave element:

        g_signal_connect(deinterleave, "pad-added", G_CALLBACK(sound_card_new_pad_handler), this);

i created a common function to connect each channel to a queue

void SoundCardData::attach_pad(const int channelId, GstPad *nextPad) const
    {
        const std::string srcPadName = "src_" + std::to_string(channelId);

        GstPad *srcPad = gst_element_get_static_pad (deinterleave, srcPadName.c_str());
        if (!srcPad)
        {
            MoLog::fatal() << "could not get deinterlaeve pad " << srcPadName;
        }
        if (gst_pad_is_linked(nextPad))
        {
            MoLog::fatal() << "pad already linked";
        }
        if (gst_pad_is_linked(srcPad))
        {
            MoLog::fatal() << "new_pad already linked";
        }
        if (!gst_pad_can_link(srcPad, nextPad))
        {
            MoLog::error() << "can't link new_pad to sinkpad";

            GstCaps *new_pad_caps = gst_pad_get_current_caps(srcPad);
            GstCaps *sinkpad_caps = gst_pad_get_pad_template_caps(nextPad);
            if (!gst_caps_can_intersect(new_pad_caps, sinkpad_caps)) {
                MoLog::fatal() << "Caps are not compatible";
            } else
            {
                MoLog::info() << "caps can intersect";
            }
            gst_caps_unref(new_pad_caps);
            gst_caps_unref(sinkpad_caps);
        }
        if (gst_pad_link(srcPad, nextPad) != GST_PAD_LINK_OK) {
            g_warning("Failed to link deinterleave pad to encoder");
        }
        MoLog::debug() << "attached pad " << srcPadName << " succesfully!";

    }

and in my case i wait for all the pads to be added by counting how much pads where added and compare it the channels in the sound card:

 static void sound_card_new_pad_handler(GstElement *src, GstPad *new_pad, gpointer user_data)
    {
        auto data = static_cast<SoundCardData*>(user_data);
        gchar *name = gst_pad_get_name(new_pad);
        // MoLog::debug() << "new_pad_handler executed for " << name;
        data->increasePadChannels();
        g_free(name);
    }

and i use this function to like the src pads from interleave to each queue:

attach_pad(channelId1, audio_connected_queue1_sink_pad);
attach_pad(channelId2, audio_connected_queue2_sink_pad);
   

and voila! works like charm.