aio_write fails on OSX: Inappropriate ioctl for device

Question

Code for the aio write and read:

#include "io_engine.h"

namespace demo {

Status PosixAIOEngine::AsyncWrite(int fd, uint64_t offset, const char* buffer, uint64_t size,
                                  char** cb) {
  // TODO Check io_depth
  requests_.push_back({.aio_req_ = {}, .type_ = kAsyncWrite});
  auto& request = requests_.back();
  memset(&request.aio_req_, 0, sizeof(struct aiocb));
  request.aio_req_.aio_offset = (off_t)offset;
  request.aio_req_.aio_buf = (void*)buffer;
  request.aio_req_.aio_nbytes = size;
  request.aio_req_.aio_fildes = fd;

  int ret = aio_write(&request.aio_req_);
  if (ret == -1) {
    auto msg = "aio_write failed, err msg: " + std::string(strerror(errno));
    LOG(ERROR, msg);
    return Status::IOError(msg);
  }

  return Status::OK();
}

Status PosixAIOEngine::AsyncRead(int fd, uint64_t offset, char* buffer, uint64_t size, char** cb) {
  requests_.push_back({.aio_req_ = {}, .type_ = kAsyncRead});
  auto& request = requests_.back();
  memset(&request.aio_req_, 0, sizeof(struct aiocb));
  request.aio_req_.aio_offset = (off_t)offset;
  request.aio_req_.aio_buf = (void*)buffer;
  request.aio_req_.aio_nbytes = size;
  request.aio_req_.aio_fildes = fd;

  int ret = aio_read(&request.aio_req_);
  if (ret == -1) {
    LOG(ERROR, "aio_read failed, err: {}, offset: {}, sz: {}", strerror(errno), offset, size);
    requests_.pop_back();
    return Status::IOError("aio read failed!");
  }
  return Status::OK();
}

// TODO Probably we should check the event by their submission order?
uint32_t PosixAIOEngine::Poll() {
  uint32_t cnt = 0;
  auto it = requests_.begin();
  while (it != requests_.end()) {
    auto& req = *it;
    int ret = aio_error(&req.aio_req_);
    if (ret == 0) {
      LOG(INFO, IORequest::GetTypeName(req.type_) +
                    ", callback offset: " + std::to_string(req.aio_req_.aio_offset));
      uint64_t finish_code = aio_return(&req.aio_req_);
      if (finish_code != 0) {
        LOG(ERROR, "return code error : {}, offset: {}", strerror(errno), req.aio_req_.aio_offset);
      }
      it = requests_.erase(it);
      cnt++;
    } else {
      ++it;
    }
  }
  return cnt;
}

}

code of how I test it:

#include "io_engine.h"

#include <fcntl.h>
#include <gtest/gtest.h>
#include <unistd.h>

#include <memory>

#include "utils.h"

namespace demo {
class IOEngineTest : public ::testing::Test {
 public:
  std::string file_prefix = "io_engine_test_file_";
  std::string test_file_;
  int write_fd_{};
  int read_fd_{};
  uint64_t file_size_ = 10 << 20;
  std::unique_ptr<IOEngine> io_engine_;

  void SetUp() override {
    test_file_ = CreateRandomFile(file_size_);
    io_engine_ = std::make_unique<PosixAIOEngine>();
    write_fd_ = open(test_file_.c_str(), O_RDWR | O_CREAT, 0777);
    if (write_fd_ == -1) {
      LOG(ERROR, "Open writable file failed, error: {}", std::strerror(errno));
      abort();
    }
    read_fd_ = open(test_file_.c_str(), O_RDONLY);
    if (read_fd_ == -1) {
      LOG(ERROR, "Open readonly file failed, error: {}", std::strerror(errno));
      abort();
    }
  }

  void TearDown() override {
    int total = FileUtils::DeleteFilesByPrefix(".", file_prefix);
    LOG(INFO, "{} files was deleted", total);
  }

  std::string CreateRandomFile(uint32_t sz) {
    auto filename = FileUtils::GenerateRandomFile(file_prefix, sz);
    filenames_.emplace_back(filename);
    return filename;
  }

 private:
  std::vector<std::string> filenames_;
};

TEST_F(IOEngineTest, MultipleAIOWriteTest) {
  uint64_t page_size = 4UL << 10;
  uint32_t total_requests = 6;

  // Write Request
  for (int i = 0; i < total_requests; ++i) {
    std::string data = std::to_string(i);
    auto s = io_engine_->AsyncWrite(write_fd_, i * page_size, data.c_str(), data.size(), nullptr);
    EXPECT_TRUE(s.ok());
  }

  // POLL result out
  uint32_t cnt = 0;
  while (cnt < total_requests) {
    cnt += io_engine_->Poll();
  }
  EXPECT_EQ(cnt, total_requests);
  EXPECT_EQ(io_engine_->GetInFlightRequests(), 0);

  // Read Request
  char* buf = nullptr;
  posix_memalign((void**)&buf, page_size, total_requests * page_size);
  for (int i = 0; i < total_requests; ++i) {
    auto s = io_engine_->AsyncRead(read_fd_, i * page_size, buf, page_size, nullptr);
    ASSERT_TRUE(s.ok());
  }
  EXPECT_EQ(io_engine_->GetInFlightRequests(), total_requests);

  // POLL result out
  cnt = 0;
  while (cnt < total_requests) {
    cnt += io_engine_->Poll();
  }

  EXPECT_EQ(cnt, total_requests);
  for (int i = 0; i < total_requests; ++i) {
    ASSERT_EQ(buf[i], static_cast<char>(i));
  }
  delete buf;
}
}

The io submits are ok, but when I poll the result, the returned finish code is not zero and thus there is an error: "Inappropriate ioctl for device".

Not sure what happens, but IIRC, posix aio library is a user-space aysnc IO engine that should be able to work on any system.

The log message:

[2024-03-11 T11:01:36.786][132731804][info][utils.h:35] Generate random file, filename: io_engine_test_file_4245618977
[2024-03-11 T11:01:36.786][132731804][info][io_engine_test.cc:34] write fd: 4, read fd: 5
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncWrite, callback offset: 4096
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 4096, fd : 4
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncWrite, callback offset: 8192
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 8192, fd : 4
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncWrite, callback offset: 12288
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 12288, fd : 4
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncWrite, callback offset: 0
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 0, fd : 4
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncWrite, callback offset: 16384
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 16384, fd : 4
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncWrite, callback offset: 20480
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 20480, fd : 4
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncRead, callback offset: 4096
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 4096, fd : 5
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncRead, callback offset: 12288
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 12288, fd : 5
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncRead, callback offset: 16384
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 16384, fd : 5
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncRead, callback offset: 20480
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 20480, fd : 5
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncRead, callback offset: 0
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 0, fd : 5
[2024-03-11 T11:01:36.787][132731804][info][io_engine.cc:62] kAsyncRead, callback offset: 8192
[2024-03-11 T11:01:36.787][132731804][error][io_engine.cc:66] return code error : Inappropriate ioctl for device, offset: 8192, fd : 5