Boost spirit parser fails with incomplete type error

Question

I've been trying all sorts of things but still not quite understanding why the following fails with 'incomplete type' error

#define BOOST_PHOENIX_LIMIT 30
#define SPIRIT_ARGUMENTS_LIMIT 30

#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <ctime>
#include <chrono>
#include <string>
#include <iomanip>

namespace qi = boost::spirit::qi;
namespace phi = boost::phoenix;
using namespace std::chrono_literals;
using namespace qi::labels;

using It = std::string::const_iterator;

#define PRICE_MULT 10000

class ImbalanceMsg
{

public:
    ImbalanceMsg(){}

    ImbalanceMsg(timespec ts,
                 uint8_t msgtype,
                 uint64_t seq_num,
                 std::string symbol,
                 uint64_t symbol_seqnum,
                 uint64_t ref_price,
                 uint32_t paired_qty,
                 uint32_t total_imb_qty,
                 uint32_t mkt_imb_qty,
                 uint32_t auction_time,
                 char     auction_type,
                 char     imb_side,
                 uint64_t cont_clear_price,
                 uint64_t auction_int_clear_price,
                 uint64_t ssr_filling_price,
                 uint64_t ind_match_price,
                 uint64_t upper_collar,
                 uint64_t lower_collar,
                 uint32_t auction_status,
                 uint32_t freeze_status,
                 uint32_t num_ext
    ) :

            m_ref_price{ref_price},
            m_paired_qty{paired_qty},
            m_total_imb_qty{total_imb_qty},
            m_mkt_imb_qty{mkt_imb_qty},
            m_auction_time{auction_time},
            m_auction_type{auction_type},
            m_imb_side{imb_side},
            m_cont_clear_price{cont_clear_price},
            m_auction_int_clear_price{auction_int_clear_price},
            m_ssr_filling_price{ssr_filling_price},
            m_ind_match_price{ind_match_price},
            m_upper_collar{upper_collar},
            m_lower_collar{lower_collar},
            m_auction_status{auction_status},
            m_freeze_status{freeze_status},
            m_num_ext{num_ext}
    {}

    // auto msg = parse( "105,42982201,15:00:05.553620224,AAPL,1192101,157.56,0,200,0,1600,C,S,0,0,0,157.57,159140000,155980000,0,0,0" );


    std::string m_symbol;
    uint64_t m_symbol_seqnum;
    uint64_t m_ref_price;
    uint32_t m_paired_qty;
    uint32_t m_total_imb_qty;
    uint32_t m_mkt_imb_qty;
    uint32_t m_auction_time;
    char     m_auction_type;
    char     m_imb_side;
    uint64_t m_cont_clear_price;
    uint64_t m_auction_int_clear_price;
    uint64_t m_ssr_filling_price;
    uint64_t m_ind_match_price;
    uint64_t m_upper_collar;
    uint64_t m_lower_collar;
    uint32_t  m_auction_status;
    uint32_t  m_freeze_status;
    uint32_t m_num_ext;

};




int main() {

    std::string s = "AAPL,1192101,157.56,0,200,0,1600,C,S,0,0,0,157.57,159140000,155980000,0,0,0";

    timespec ts;
    uint8_t msgtype = 105;
    uint64_t seq_num = 42982201;

    qi::uint_parser<uint32_t, 10, 1, 6> int_part;
    qi::uint_parser<uint8_t , 10, 1, 1> m_digit;

    qi::rule<std::string::iterator, uint64_t()>
        m_fixed_point = int_part[qi::_val =  qi::_1 * PRICE_MULT] >>
              -("." >> -(m_digit[qi::_val += qi::_1 * 1000])
                    >> -(m_digit[qi::_val += qi::_1 * 100])
                    >> -(m_digit[qi::_val += qi::_1 * 10])
                    >> -(m_digit[qi::_val += qi::_1 ])
              );

    qi::rule<std::string::iterator, ImbalanceMsg()>
        m_wire_msg = ( qi::as_string[*qi::alpha]   >> "," // symbol
                                                   >> qi::ulong_    >> "," // symbol seq num
                                                   >> m_fixed_point >> "," // ref price
                                                   >> qi::uint_     >> "," // paired_qty
                                                   >> qi::uint_     >> "," // total_imb_qty
                                                   >> qi::uint_     >> "," // mkt_imb_qty
                                                   >> qi::uint_     >> "," // auction_time
                                                   >> qi::char_     >> "," // auction type
                                                   >> qi::char_     >> "," // imb side
                                                   >> m_fixed_point >> "," // cont_clear_price
                                                   >> m_fixed_point >> "," // auction_int_clear_price
                                                   >> m_fixed_point >> "," // ssr_filling_price
                                                   >> m_fixed_point >> "," // ind_match_price
                                                   >> m_fixed_point >> "," // upper_collar
                                                   >> m_fixed_point >> "," // lower_collar
                                                   >> qi::ushort_   >> "," // auction status
                                                   >> qi::ushort_   >> "," // freeze status
                                                   >> qi::ushort_
         )[qi::_val = phi::construct<ImbalanceMsg>(ts, msgtype, seq_num,
                                                      qi::_1, //symbol
                                                      qi::_2, //market_id
                                                      qi::_3, //system_id
                                                      qi::_4, //exchange_code
                                                      qi::_5, //security_type
                                                      qi::_6, //lot_size
                                                      qi::_7, // prev_close_price
                                                      qi::_8, // prev_close_volume
                                                      qi::_9, // price_resolution
                                                      qi::_10, // round_lot
                                                      qi::_11, // mpv
                                                      qi::_12,
                                                      qi::_13,
                                                      qi::_14,
                                                      qi::_15,
                                                      qi::_16,
                                                      qi::_17,
                                                      qi::_18
    )];

    ImbalanceMsg m;
    bool ok = parse( s.begin(), s.end(), m_wire_msg, m );
    std::cout << "ok=" << ok << std::endl;


}

And I don't get the same issues with smaller classes with fewer attributes compared to ImbalanceMsg.

I have several other types of message classes with similar code but all of them compiled fine.

Can someone offer any pointers?

Answer 1

Just having a placeholder makes it so that the semantic action expression (inside []) can be compiled.

So far so good.

But to actually compile the rule (from the template expression that includes your semantic action) you need to actually have Fusion support for large sequences as well:

#define BOOST_MAX_FUSION_VECTOR_SIZE 30

Note, besides that you're passing 3 arguments too many (_1 through _18 makes 18, but you already pass the hardcoded (ts, msgtype, seq_num) so 15 would be enough).

But as in my previous answer (Compile error due boost spirit placeholder limit not more than 10) I'd advise against this. This is only going to make your code unreadable, extremely expensive to compile and generally not lead to useful benefits.

Instead, consider using qi::_0 which just passes the whole synthesized attribute sequence, at once.

Or don't use semantic actions at all;

Look Ma, No Hands!

I frequently state I prefer to avoid Semantic Actions. The reasons why: Boost Spirit: "Semantic actions are evil"?

In this case I'd suggest simple Fusion sequence adaptation:

struct ImbalanceMsg {
    timespec    ts;
    uint8_t     msgtype;
    uint64_t    seq_num;

    std::string symbol;
    uint64_t    symbol_seqnum;
    uint64_t    ref_price;
    uint32_t    paired_qty;
    uint32_t    total_imb_qty;
    uint32_t    mkt_imb_qty;
    uint32_t    auction_time;
    char        auction_type;
    char        imb_side;
    uint64_t    cont_clear_price;
    uint64_t    auction_int_clear_price;
    uint64_t    ssr_filling_price;
    uint64_t    ind_match_price;
    uint64_t    upper_collar;
    uint64_t    lower_collar;
    uint32_t    auction_status;
    uint32_t    freeze_status;
    uint32_t    num_ext;
};

BOOST_FUSION_ADAPT_STRUCT(ImbalanceMsg, 
    ts, msgtype, seq_num, symbol, symbol_seqnum, ref_price,
    paired_qty, total_imb_qty, mkt_imb_qty, auction_time, auction_type, imb_side,
    cont_clear_price, auction_int_clear_price, ssr_filling_price, ind_match_price, upper_collar,
    lower_collar, auction_status, freeze_status, num_ext)

Now you can simply parse all members:

qi::rule<std::string::const_iterator, ImbalanceMsg()> m_wire_msg 
    = qi::attr(ts) >> qi::attr(msgtype) >> qi::attr(seq_num)
    >> qi::as_string[*qi::alpha] >> "," // symbol
    >> qi::ulong_                >> "," // symbol seq num
    >> m_fixed_point             >> "," // ref price
    >> qi::uint_                 >> "," // paired_qty
    >> qi::uint_                 >> "," // total_imb_qty
    >> qi::uint_                 >> "," // mkt_imb_qty
    >> qi::uint_                 >> "," // auction_time
    >> qi::char_                 >> "," // auction type
    >> qi::char_                 >> "," // imb side
    >> m_fixed_point             >> "," // cont_clear_price
    >> m_fixed_point             >> "," // auction_int_clear_price
    >> m_fixed_point             >> "," // ssr_filling_price
    >> m_fixed_point             >> "," // ind_match_price
    >> m_fixed_point             >> "," // upper_collar
    >> m_fixed_point             >> "," // lower_collar
    >> qi::ushort_               >> "," // auction status
    >> qi::ushort_               >> "," // freeze status
    >> qi::ushort_;

And attribute propagation is completely automagic:

Live On Coliru

#define BOOST_MAX_FUSION_VECTOR_SIZE 30
#define BOOST_PHOENIX_LIMIT 30
#define SPIRIT_ARGUMENTS_LIMIT 30
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix.hpp>
#include <boost/fusion/include/io.hpp>
#include <ctime>

namespace qi = boost::spirit::qi;

#define PRICE_MULT 10000

struct ImbalanceMsg {
    timespec    ts;
    uint8_t     msgtype;
    uint64_t    seq_num;

    std::string symbol;
    uint64_t    symbol_seqnum;
    uint64_t    ref_price;
    uint32_t    paired_qty;
    uint32_t    total_imb_qty;
    uint32_t    mkt_imb_qty;
    uint32_t    auction_time;
    char        auction_type;
    char        imb_side;
    uint64_t    cont_clear_price;
    uint64_t    auction_int_clear_price;
    uint64_t    ssr_filling_price;
    uint64_t    ind_match_price;
    uint64_t    upper_collar;
    uint64_t    lower_collar;
    uint32_t    auction_status;
    uint32_t    freeze_status;
    uint32_t    num_ext;
};

BOOST_FUSION_ADAPT_STRUCT(ImbalanceMsg, 
    ts, msgtype, seq_num, symbol, symbol_seqnum, ref_price,
    paired_qty, total_imb_qty, mkt_imb_qty, auction_time, auction_type, imb_side,
    cont_clear_price, auction_int_clear_price, ssr_filling_price, ind_match_price, upper_collar,
    lower_collar, auction_status, freeze_status, num_ext)

static inline std::ostream& operator<<(std::ostream& os, timespec) { return os << "(timespec:TODO)"; }

struct SimpleReal : qi::real_policies<double> {
    template <typename It> static bool parse_exp(It, It)         { return false; }
    template <typename It> static bool parse_exp_n(It, It, int)  { return false; }
    template <typename It> static bool parse_nan(It, It, double) { return false; }
    template <typename It> static bool parse_inf(It, It, double) { return false; }
};

int main() {

    qi::rule<std::string::const_iterator, uint64_t()> m_fixed_point 
        = qi::real_parser<double, SimpleReal>{} [ qi::_val = PRICE_MULT*qi::_1 ];

    timespec ts;
    uint8_t msgtype = 105;
    uint64_t seq_num = 42982201;

    qi::rule<std::string::const_iterator, ImbalanceMsg()> m_wire_msg 
        = qi::attr(ts) >> qi::attr(msgtype) >> qi::attr(seq_num)
        >> qi::as_string[*qi::alpha] >> "," // symbol
        >> qi::ulong_                >> "," // symbol seq num
        >> m_fixed_point             >> "," // ref price
        >> qi::uint_                 >> "," // paired_qty
        >> qi::uint_                 >> "," // total_imb_qty
        >> qi::uint_                 >> "," // mkt_imb_qty
        >> qi::uint_                 >> "," // auction_time
        >> qi::char_                 >> "," // auction type
        >> qi::char_                 >> "," // imb side
        >> m_fixed_point             >> "," // cont_clear_price
        >> m_fixed_point             >> "," // auction_int_clear_price
        >> m_fixed_point             >> "," // ssr_filling_price
        >> m_fixed_point             >> "," // ind_match_price
        >> m_fixed_point             >> "," // upper_collar
        >> m_fixed_point             >> "," // lower_collar
        >> qi::ushort_               >> "," // auction status
        >> qi::ushort_               >> "," // freeze status
        >> qi::ushort_;

    ImbalanceMsg m;
    std::string const s = "AAPL,1192101,157.56,0,200,0,1600,C,S,0,0,0,157.57,159140000,155980000,0,0,0";

    bool ok = parse( s.begin(), s.end(), m_wire_msg, m );

    std::cout << "ok=" << ok << "\n";

    using boost::fusion::operator<<;
    std::cout << m << "\n";
}

Prints

ok=1
((timespec:TODO) i 42982201 AAPL 1192101 1575600 0 200 0 1600 C S 0 0 0 1575700 1591400000000 1559800000000 0 0 0)

Using qi::_0

You can read some more about this approach here: Passing each element of a parsed sequence to a function that returns a rule's attribute type

template <typename T> struct Factory {
    template <typename Seq>
    T operator()(Seq const& seq) const { return my_apply(Construct{}, seq); }
  private:
    struct Construct {
        template <typename... I> T operator()(I... initializers) const 
            { return T { initializers... }; }
    };
};

Surprisingly boost::fusion::apply still doesn't exist, so I'll add the my_apply implementation from that answer (c++14).

Now you can use that in the semantic action:

...
>> qi::ushort_) [ qi::_val = phi::bind(Factory<ImbalanceMsg>{}, qi::_0) ];

This has the benefit of not requiring Boost Fusion Adaptation, and still not requiring a zillion placeholders.

BONUS

I'd also do the fixed point real parsing simpler:

qi::rule<std::string::const_iterator, uint64_t()> m_fixed_point 
    = qi::real_parser<double, SimpleReal>{} [ qi::_val = PRICE_MULT*qi::_1 ];

Where SimpleReal is the real policy that doesn't allow for special representations (like scientific notation).