How to avoid zombies finished child processes when parent is working in loop

Question

I am searching a good solution for avoiding a ZOMBIE child processes when parent is still working. For example parent is working in loop and is checking some conditions. If conditions are acomplished, parent process run subprocess.

Now I wrote simple python scripts:

File main.py:

#!/usr/bin/env python3
# coding=utf-8

import os
import psutil
import subprocess
import time
import signal
import setproctitle


def main():

    python_venv = 'python3'
    print(f"{__file__}  - Run proc 1")
    a1 = [python_venv, 'proc1.py']
    string_do_wywolania_part2 = ['arg1'  , 'arg2']
    a2 = a1 + string_do_wywolania_part2
    p = subprocess.Popen(a2, preexec_fn=os.setpgrp)

    print(f"{__file__}  - Run proc 2")
    a1 = [python_venv, 'proc2.py']
    string_do_wywolania_part2 = ['arg1'  , 'arg2']
    a2 = a1 + string_do_wywolania_part2
    p = subprocess.Popen(a2, preexec_fn=os.setpgrp)

    pid_id = os.getpid()
    ppid_id = os.getppid()
    print(f"{__file__} - pid_id  : {pid_id} - ppid_id  : {ppid_id}")

    parent = psutil.Process(pid_id)
    print(f"{__file__} : parent  : {parent}")

    t = 6
    print(f"{__file__} - I am waiting {t} sec.")
    time.sleep(t)



if __name__ == "__main__":

    "call sub processes"
    setproctitle.setproctitle('python3 - main')
    main()

    while True:

        k = 1
        pid_id = os.getpid()
        parent = psutil.Process(pid_id)
        print()
        for child in parent.children(recursive=True):  # or parent.children() for recursive=False
            print(f'{__file__} - {child}  = {child.status()}')
            k +=1
        time.sleep(5)


    print(f'{__file__} : Finished main')

File proc1.py

#!/usr/bin/env python3
# coding=utf-8
import os
import sys
import time
import setproctitle


if __name__ == "__main__":
    setproctitle.setproctitle('python3 - proc1')
    pid = os.getpid()
    ppid = os.getppid()
    print(f"{__file__} - Process 1 is running")
    print(f"{__file__} - pid  : {pid} - ppid  : {ppid}")
    time.sleep(5)
    print(f"{__file__} : Finished process 1  : {pid}")
    sys.exit(0)

proc2.file:

#!/usr/bin/env python3
# coding=utf-8

import os
import sys
import time
import subprocess
import signal
import setproctitle


if __name__ == "__main__":
    setproctitle.setproctitle('python3 - proc2')
    print(f"{__file__} - Process 2 is running")
    try:
        pid = os.getpid()
        ppid = os.getppid()
        print(f"{__file__} - pid  : {pid} - ppid  : {ppid}")
        print(f"{__file__}  - Run proc 3")

        python_venv = 'python3'
        a1 = [python_venv, 'proc3.py']
        string_do_wywolania_part2 = ['arg1'  , 'arg2']
        a2 = a1 + string_do_wywolania_part2
        p = subprocess.Popen(a2, preexec_fn=os.setpgrp)
        time.sleep(15)
    except Exception as e:
        print(e)

    time.sleep(7)
    print(f"{__file__} : Finished process 2  : {pid}")
    sys.exit(0)

and the last one proc3.py

#!/usr/bin/env python3
# coding=utf-8

import os
import sys
import time
import setproctitle


if __name__ == "__main__":
    ""
    setproctitle.setproctitle('python3 - proc3')
    pid = os.getpid()
    ppid = os.getppid()
    print(f"{__file__} - Process 3 is running")
    print(f"{__file__} - pid  : {pid} -  ppid  : {ppid}")
    time.sleep(10)
    print(f"{__file__} : Finished process 3  : {pid}")

    sys.exit(0)

When I am running python3 main.py i am getting displays like bellow:

main.py  - Run proc 1
main.py  - Run proc 2
main.py - pid_id  : 5696 - ppid_id  : 4763
main.py : parent  : psutil.Process(pid=5696, name='python3 - main', status='running', started='14:26:46')
main.py - I am waiting 6 sec.
proc1.py - Process 1 is running
proc1.py - pid  : 5697 - ppid  : 5696
proc2.py - Process 2 is running
proc2.py - pid  : 5698 - ppid  : 5696
proc2.py  - Run proc 3
proc3.py - Process 3 is running
proc3.py - pid  : 5699 -  ppid  : 5698
proc1.py : Finished process 1  : 5697

main.py - psutil.Process(pid=5697, name='python3 - proc1', status='zombie', started='14:26:46')  = zombie
main.py - psutil.Process(pid=5698, name='python3 - proc2', status='sleeping', started='14:26:46')  = sleeping
main.py - psutil.Process(pid=5699, name='python3 - proc3', status='sleeping', started='14:26:46')  = sleeping
proc3.py : Finished process 3  : 5699

main.py - psutil.Process(pid=5697, name='python3 - proc1', status='zombie', started='14:26:46')  = zombie
main.py - psutil.Process(pid=5698, name='python3 - proc2', status='sleeping', started='14:26:46')  = sleeping
main.py - psutil.Process(pid=5699, name='python3 - proc3', status='zombie', started='14:26:46')  = zombie

main.py - psutil.Process(pid=5697, name='python3 - proc1', status='zombie', started='14:26:46')  = zombie
main.py - psutil.Process(pid=5698, name='python3 - proc2', status='sleeping', started='14:26:46')  = sleeping
main.py - psutil.Process(pid=5699, name='python3 - proc3', status='zombie', started='14:26:46')  = zombie

main.py - psutil.Process(pid=5697, name='python3 - proc1', status='zombie', started='14:26:46')  = zombie
main.py - psutil.Process(pid=5698, name='python3 - proc2', status='sleeping', started='14:26:46')  = sleeping
main.py - psutil.Process(pid=5699, name='python3 - proc3', status='zombie', started='14:26:46')  = zombie
proc2.py : Finished process 2  : 5698

main.py - psutil.Process(pid=5697, name='python3 - proc1', status='zombie', started='14:26:46')  = zombie
main.py - psutil.Process(pid=5698, name='python3 - proc2', status='zombie', started='14:26:46')  = zombie

As you see, proc3 first is zombie but when parent process for him was finished, proc 3 disappear. But still I have proc1 and proc2 as zombies.

How to call child's sub processes and avoid them to belong to zombie after finish? I read info about call, popen and preexec_fn and that I should del subprocess.popen object, but I do not know which is right one.

Can you tell me what should I do?

EDITED:

while True:
     if some_condition:
        p = subprocess.Popen()

        # what to do to avoid a zombie childs

     for proc in processes_list:
        if proc.poll() == 0:
            print(f'--> I am terminate {proc} --> {proc.pid}')
            proc.terminate()
            processes_list.remove(proc)

Answer 1

To sum up:

A parent process must wait on the exit status of the child. If the parent exits before waiting on it's children then you end up with zombie processes.

You have already considered having a list of Popen objects called processes_list

You can use Popen.wait with a timeout like:

for proc in processes_list:
    try:
        # use Popen.wait or Popen.communicate if using pipes.
        proc.wait(timeout=1)
        # proc.returncode contains the exit code of the child
        # no need to track anymore finished
        processes_list.remove(proc)
    except subprocess.TimeoutExpired as to:
        pass

If you use Popen.poll as in your edited code then you should check for None which means that the child is not finished yet, so don't remove it from the list. You where checking for 0 but what if the return code is something else?

while True:

    if some_condition:
        p = subprocess.Popen()
        processes_list.append(p)


    for proc in processes_list:
        # you can save the return value, although proc.returncode should have it as well
        p = proc.poll()
        # None means child is not finished yet, will poll again next round
        if p is not None:
            # again proc.returncode also contains the return code
            print(f'--> I am terminate {proc} --> {proc.pid}')
            processes_list.remove(proc)

Finally on the proc2 file the same applies you must wait for the child process to finish and then let the parent itself exit. This can be achived with Popen.wait:

#!/usr/bin/env python3
# coding=utf-8

import os
import sys
import time
import subprocess
import signal
import setproctitle


if __name__ == "__main__":
    setproctitle.setproctitle('python3 - proc2')
    print(f"{__file__} - Process 2 is running")
    try:
        pid = os.getpid()
        ppid = os.getppid()
        print(f"{__file__} - pid  : {pid} - ppid  : {ppid}")
        print(f"{__file__}  - Run proc 3")

        python_venv = 'python3'
        a1 = [python_venv, 'proc3.py']
        string_do_wywolania_part2 = ['arg1'  , 'arg2']
        a2 = a1 + string_do_wywolania_part2
        p = subprocess.Popen(a2, preexec_fn=os.setpgrp)
        # no need to sleep as this does not guarantee that the child is done
        #time.sleep(15)
        # this will block until the child is done.
        # if you use poll here it will most probably result in a zombie. 
        p.wait()
        

        
    except Exception as e:
        print(e)

    time.sleep(7)
    print(f"{__file__} : Finished process 2  : {pid}")
    sys.exit(0)