How to set cdll search path at build time?

Question

I have a shred library libcustom.so in a non standard folder, and a python package where I use ctypes.cdll.LoadLibrary("libcustom.so").

How can I set libcustom.so path at build time (something similar to rpath) ?

env LD_LIBRARY_PATH=/path/to/custom/lib python3 -c "import mypackage"

This works fine, but I don't want to use global LD_LIBRARY_PATH, and I don't want to set library path at run time.

python3 -c "import mypackage"

Results in an error:

OSError: libcustum.so: cannot open shared object file: No such file or directory

Answer 1

The only solution I have found (there must be better ones) is to create a python module to encapsulate the dlopen call. It is thus possible to insert an rpath at compilation time to resolve dynamic links at runtime.

/* _custom.c */
#include <Python.h>
#include <dlfcn.h>

static struct PyModuleDef _custommodule = {
    PyModuleDef_HEAD_INIT,
    .m_name = "_custom",
    .m_doc = "Convenience module, libcustom.so wrapper",
    .m_size = -1
};

PyMODINIT_FUNC
PyInit__custom(void)
{
    PyObject *self = NULL, *ctypes = NULL, *cdll = NULL, *lib = NULL;

    /* We must import the library a first time here to allow the use
       of rpath. Otherwise `ctypes.cdll.LoadLibrary` uses dlopen from
       an internal python module, and the internal module's rpath will
       be used for link resolution instead of our. */
    if (! dlopen("libcustom.so", RTLD_NOW | RTLD_LOCAL)) {
        PyErr_SetString(PyExc_ImportError, dlerror());
        return NULL;
    }
    if (! (self = PyModule_Create(&_custommodule))) goto fail;
    if (! (ctypes = PyImport_ImportModule("ctypes"))) goto fail;
    if (! (cdll = PyDict_GetItemString(PyModule_GetDict(ctypes), (char*) "cdll"))) goto fail;
    if (! (lib = PyObject_CallMethod(cdll, "LoadLibrary", "s", "libcustom.so"))) goto fail;
    if (PyModule_AddObject(self, "_lib", lib) < 0) goto fail;

    Py_DECREF(ctypes);
    Py_DECREF(cdll);
    return self;

 fail:
    PyErr_SetString(PyExc_ImportError, "Internal error");
    Py_XDECREF(self);
    Py_XDECREF(ctypes);
    Py_XDECREF(cdll);
    Py_XDECREF(lib);
    return NULL;
}

We then add this extension to the python package:

# setup.py
setup(
    ext_modules=[Extension("mypackage._custom", ["_custom.c"])],
    ...
)

Then, when building the package, we can insert the rpath:

python3 setup.py build_ext --rpath /path/to/custom/lib
python3 setup.py install

It only remains to replace ctypes.cdll.LoadLibrary("libcustom.so") by importlib.import_module("mypackage._custom")._lib.

Answer 2

Listing [Python.Docs]: ctypes - A foreign function library for Python.

You can pass the full .dll name to CDLL (or cdll.LoadLibrary).

code00.py:

#!/usr/bin/env python

import ctypes as ct
import os
import sys
from custom_dll_path import CUSTOM_DLL_PATH


DLL_NAME = "dll00.{:s}".format("dll" if sys.platform[:3].lower() == "win" else "so")


def main(*argv):
    if argv:
        dll_name = DLL_NAME
    else:
        dll_name = os.path.join(CUSTOM_DLL_PATH, DLL_NAME)
    print("Attempting to load: {:s}".format(dll_name))
    dll00 = ct.CDLL(dll_name)


if __name__ == "__main__":
    print("Python {:s} {:03d}bit on {:s}\n".format(" ".join(elem.strip() for elem in sys.version.split("\n")),
                                                   64 if sys.maxsize > 0x100000000 else 32, sys.platform))
    rc = main(*sys.argv[1:])
    print("\nDone.")
    sys.exit(rc)

setup.py:

#!/usr/bin/env python

import sys
from setuptools import Command, setup


class SetDllPath(Command):
    user_options = [
        ("path=", "p", "Dll path"),
    ]

    def initialize_options(self):
        self.path = ""

    def finalize_options(self):
        pass

    def run(self):
        with open("custom_dll_path.py", "w") as f:
            f.write("CUSTOM_DLL_PATH = \"{:s}\"\n".format(self.path))


setup(
    name="custom_project",
    cmdclass={
        "set_dll_path": SetDllPath,
    },
    #  Other stuff ...
)

Output:

[cfati@cfati-5510-0:/mnt/e/Work/Dev/StackOverflow/q070640586]> ~/sopr.sh
### Set shorter prompt to better fit when pasted in StackOverflow (or other) pages ###

[064bit prompt]> ls
code00.py  setup.py
[064bit prompt]> echo "int f() { return 0; }" > dll00.c
[064bit prompt]> gcc -fPIC -shared -o dll00.so dll00.c
[064bit prompt]> ls
code00.py  dll00.c  dll00.so  setup.py
[064bit prompt]>
[064bit prompt]> python3.9 setup.py set_dll_path -p $(pwd)
running set_dll_path
[064bit prompt]> ls
code00.py  custom_dll_path.py  dll00.c  dll00.so  setup.py
[064bit prompt]> cat custom_dll_path.py
CUSTOM_DLL_PATH = "/mnt/e/Work/Dev/StackOverflow/q070640586"
[064bit prompt]>
[064bit prompt]> python3.9 code00.py dummy args to fail
Python 3.9.9 (main, Nov 16 2021, 03:08:02) [GCC 9.3.0] 064bit on linux

Attempting to load: dll00.so
Traceback (most recent call last):
  File "/mnt/e/Work/Dev/StackOverflow/q070640586/code00.py", line 24, in <module>
    rc = main(*sys.argv[1:])
  File "/mnt/e/Work/Dev/StackOverflow/q070640586/code00.py", line 18, in main
    dll00 = ct.CDLL(dll_name)
  File "/usr/lib/python3.9/ctypes/__init__.py", line 374, in __init__
    self._handle = _dlopen(self._name, mode)
OSError: dll00.so: cannot open shared object file: No such file or directory
[064bit prompt]>
[064bit prompt]> python3.9 code00.py
Python 3.9.9 (main, Nov 16 2021, 03:08:02) [GCC 9.3.0] 064bit on linux

Attempting to load: /mnt/e/Work/Dev/StackOverflow/q070640586/dll00.so

Done.