I have a (large) C++ project built with CMake, and I am trying to use pybind11 on it. The targets include:
- to build and run an executable (like a normal C++ project);
- to call some C++ methods through python.
Therefore, I tried cmake_example (https://github.com/pybind/cmake_example), which is a simple demo provided by pybind11. I can successfully call the C++ method through python (target 2) but have trouble in building an executable (target 1). To ensure that everything is clear, I've included the reproduction code below.
Project structure: It is just the structure of cmake_example.
├── CMakeLists.txt
├── pybind11
├── setup.py
└── src
└── main.cpp
src/main.cpp: It is basically the same as in cmake_example except a trivial main function is added.
#include <pybind11/pybind11.h>
#include <iostream>
#define STRINGIFY(x) #x
#define MACRO_STRINGIFY(x) STRINGIFY(x)
int add(int i, int j) {
return i + j;
}
int main(int argc, char **argv) {
std::cout << "Hello World!!!\n";
return 0;
}
namespace py = pybind11;
PYBIND11_MODULE(cmake_example, m) {
m.doc() = R"pbdoc(
Pybind11 example plugin
-----------------------
.. currentmodule:: cmake_example
.. autosummary::
:toctree: _generate
add
subtract
)pbdoc";
m.def("add", &add, R"pbdoc(
Add two numbers
Some other explanation about the add function.
)pbdoc");
m.def("subtract", [](int i, int j) { return i - j; }, R"pbdoc(
Subtract two numbers
Some other explanation about the subtract function.
)pbdoc");
#ifdef VERSION_INFO
m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO);
#else
m.attr("__version__") = "dev";
#endif
}
CMakeLists.txt: It is basically the same as in cmake_example except add_executable is added to build executable.
cmake_minimum_required(VERSION 3.4...3.18)
project(cmake_example)
set(CMAKE_CXX_STANDARD 17)
add_subdirectory(pybind11)
pybind11_add_module(cmake_example src/main.cpp)
# EXAMPLE_VERSION_INFO is defined by setup.py and passed into the C++ code as a
# define (VERSION_INFO) here.
target_compile_definitions(cmake_example
PRIVATE VERSION_INFO=${EXAMPLE_VERSION_INFO})
add_executable(exec_example src/main.cpp)
setup.py: It is the same as in cmake_example, but I've still included the code here.
import os
import re
import subprocess
import sys
from pathlib import Path
from setuptools import Extension, setup
from setuptools.command.build_ext import build_ext
# Convert distutils Windows platform specifiers to CMake -A arguments
PLAT_TO_CMAKE = {
"win32": "Win32",
"win-amd64": "x64",
"win-arm32": "ARM",
"win-arm64": "ARM64",
}
# A CMakeExtension needs a sourcedir instead of a file list.
# The name must be the _single_ output extension from the CMake build.
# If you need multiple extensions, see scikit-build.
class CMakeExtension(Extension):
def __init__(self, name: str, sourcedir: str = "") -> None:
super().__init__(name, sources=[])
self.sourcedir = os.fspath(Path(sourcedir).resolve())
class CMakeBuild(build_ext):
def build_extension(self, ext: CMakeExtension) -> None:
# Must be in this form due to bug in .resolve() only fixed in Python 3.10+
ext_fullpath = Path.cwd() / self.get_ext_fullpath(ext.name)
extdir = ext_fullpath.parent.resolve()
# Using this requires trailing slash for auto-detection & inclusion of
# auxiliary "native" libs
debug = int(os.environ.get("DEBUG", 0)) if self.debug is None else self.debug
cfg = "Debug" if debug else "Release"
# CMake lets you override the generator - we need to check this.
# Can be set with Conda-Build, for example.
cmake_generator = os.environ.get("CMAKE_GENERATOR", "")
# Set Python_EXECUTABLE instead if you use PYBIND11_FINDPYTHON
# EXAMPLE_VERSION_INFO shows you how to pass a value into the C++ code
# from Python.
cmake_args = [
f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}{os.sep}",
f"-DPYTHON_EXECUTABLE={sys.executable}",
f"-DCMAKE_BUILD_TYPE={cfg}", # not used on MSVC, but no harm
]
build_args = []
# Adding CMake arguments set as environment variable
# (needed e.g. to build for ARM OSx on conda-forge)
if "CMAKE_ARGS" in os.environ:
cmake_args += [item for item in os.environ["CMAKE_ARGS"].split(" ") if item]
# In this example, we pass in the version to C++. You might not need to.
cmake_args += [f"-DEXAMPLE_VERSION_INFO={self.distribution.get_version()}"]
if self.compiler.compiler_type != "msvc":
# Using Ninja-build since it a) is available as a wheel and b)
# multithreads automatically. MSVC would require all variables be
# exported for Ninja to pick it up, which is a little tricky to do.
# Users can override the generator with CMAKE_GENERATOR in CMake
# 3.15+.
if not cmake_generator or cmake_generator == "Ninja":
try:
import ninja
ninja_executable_path = Path(ninja.BIN_DIR) / "ninja"
cmake_args += [
"-GNinja",
f"-DCMAKE_MAKE_PROGRAM:FILEPATH={ninja_executable_path}",
]
except ImportError:
pass
else:
# Single config generators are handled "normally"
single_config = any(x in cmake_generator for x in {"NMake", "Ninja"})
# CMake allows an arch-in-generator style for backward compatibility
contains_arch = any(x in cmake_generator for x in {"ARM", "Win64"})
# Specify the arch if using MSVC generator, but only if it doesn't
# contain a backward-compatibility arch spec already in the
# generator name.
if not single_config and not contains_arch:
cmake_args += ["-A", PLAT_TO_CMAKE[self.plat_name]]
# Multi-config generators have a different way to specify configs
if not single_config:
cmake_args += [
f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"
]
build_args += ["--config", cfg]
if sys.platform.startswith("darwin"):
# Cross-compile support for macOS - respect ARCHFLAGS if set
archs = re.findall(r"-arch (\S+)", os.environ.get("ARCHFLAGS", ""))
if archs:
cmake_args += ["-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs))]
# Set CMAKE_BUILD_PARALLEL_LEVEL to control the parallel build level
# across all generators.
if "CMAKE_BUILD_PARALLEL_LEVEL" not in os.environ:
# self.parallel is a Python 3 only way to set parallel jobs by hand
# using -j in the build_ext call, not supported by pip or PyPA-build.
if hasattr(self, "parallel") and self.parallel:
# CMake 3.12+ only.
build_args += [f"-j{self.parallel}"]
build_temp = Path(self.build_temp) / ext.name
if not build_temp.exists():
build_temp.mkdir(parents=True)
subprocess.run(
["cmake", ext.sourcedir, *cmake_args], cwd=build_temp, check=True
)
subprocess.run(
["cmake", "--build", ".", *build_args], cwd=build_temp, check=True
)
# The information here can also be placed in setup.cfg - better separation of
# logic and declaration, and simpler if you include description/version in a file.
setup(
name="cmake_example",
version="0.0.1",
author="Dean Moldovan",
author_email="[email protected]",
description="A test project using pybind11 and CMake",
long_description="",
ext_modules=[CMakeExtension("cmake_example")],
cmdclass={"build_ext": CMakeBuild},
zip_safe=False,
extras_require={"test": ["pytest>=6.0"]},
python_requires=">=3.7",
)
I followed their instructions of installation (i.e., pip install ./cmake_example), then I can call add method through python (e.g., import cmake_example cmake_example.add(1, 2)). Now the problem is that I also need the project to be built and run regularly. So I built the project via
mkdir build
cd build
cmake ..
make
but I got
/home/jjt/work/BN/cmake_example/src/main.cpp:2:10: fatal error: pybind11/pybind11.h: No such file or directory
2 | #include <pybind11/pybind11.h>
| ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
make[2]: *** [CMakeFiles/exec_example.dir/build.make:76: CMakeFiles/exec_example.dir/src/main.cpp.o] Error 1
make[1]: *** [CMakeFiles/Makefile2:128: CMakeFiles/exec_example.dir/all] Error 2
make: *** [Makefile:91: all] Error 2
pybind11 is a submodule in cmake_example, and the path of pybind11.h is pybind11/include/pybind11/pybind11.h.
Tried make VERBOSE=1 and got:
/usr/bin/c++ -std=gnu++17 -MD -MT CMakeFiles/exec_example.dir/src/main.cpp.o -MF CMakeFiles/exec_example.dir/src/main.cpp.o.d -o CMakeFiles/exec_example.dir/src/main.cpp.o -c /home/jjt/work/BN/cmake_example/src/main.cpp
/home/jjt/work/BN/cmake_example/src/main.cpp:2:10: fatal error: pybind11/pybind11.h: No such file or directory
2 | #include <pybind11/pybind11.h>
| ^~~~~~~~~~~~~~~~~~~~~
compilation terminated.
make[2]: *** [CMakeFiles/exec_example.dir/build.make:76: CMakeFiles/exec_example.dir/src/main.cpp.o] Error 1
make[2]: Leaving directory '/home/jjt/work/BN/cmake_example/build'
make[1]: *** [CMakeFiles/Makefile2:128: CMakeFiles/exec_example.dir/all] Error 2
make[1]: Leaving directory '/home/jjt/work/BN/cmake_example/build'
make: *** [Makefile:91: all] Error 2