I notice a fast unpickle method in gork. Here is the code:
@contextlib.contextmanager
def copy_to_shm(file: str):
if file.startswith("/dev/shm/"):
# Nothing to do, the file is already in shared memory.
yield file
return
tmp_dir = "/dev/shm/"
fd, tmp_path = tempfile.mkstemp(dir=tmp_dir)
try:
shutil.copyfile(file, tmp_path)
yield tmp_path
finally:
os.remove(tmp_path)
os.close(fd)
def fast_unpickle(path: str) -> Any:
with copy_to_shm(path) as tmp_path:
with open(tmp_path, "rb") as f:
return pickle.load(f)
Is it helpful to copy the file into /dev/shm/ before opening the file?
What size of files will benefit from it?