13964 2020-06-29 06:19:03 +0000 bitbake stuck doing nothing after hitting disk-full condition 2022-09-13 00:56:47 +0000 1 1 7 Build System, Metadata & Runtime BitBake bitbake unspecified x86 Multiple RESOLVED WORKSFORME Medium+ normal 4.1 M2 1 ydirson unassigned akuster808 poky.bs.watcher poky.watcher randy.macleod --- --- Don't know oldest_to_newest 87660 0 ydirson 2020-06-29 06:19:03 +0000 A dunfell image CI build stopped doing anything with the following, with processes still running, although not consuming CPU - looks like they're mostly waiting for some event that won't come. Possibly linked to https://bugzilla.yoctoproject.org/show_bug.cgi?id=13843 ? NOTE: recipe nativesdk-gettext-0.20.1-r0: task do_install: Started WARNING: The free space of TOP/build/tmp (/dev/sda3) is running low (0.965GB left) ERROR: No new tasks can be executed since the disk space monitor action is "STOPTASKS"! WARNING: The free space of /home/gitlab-runner/sstate-cache-4.99multilib (/dev/sda3) is running low (0.965GB left) ERROR: No new tasks can be executed since the disk space monitor action is "STOPTASKS"! WARNING: The free space of /home/gitlab-runner/downloads-dunfell (/dev/sda3) is running low (0.965GB left) ERROR: No new tasks can be executed since the disk space monitor action is "STOPTASKS"! NOTE: recipe nativesdk-perl-5.30.1-r0: task do_install: Succeeded NOTE: recipe nativesdk-python3-3.8.2-r0: task do_install: Succeeded NOTE: recipe nativesdk-qtbase-5.14.2+gitAUTOINC+3a6d8df521-r0: task do_configure: Succeeded NOTE: recipe nativesdk-gettext-0.20.1-r0: task do_install: Succeeded ERROR: Execution of event handler 'runqueue_stats' failed Traceback (most recent call last): File "TOP/poky/meta/classes/buildstats.bbclass", line 212, in runqueue_stats(e=<bb.event.HeartbeatEvent object at 0x7f1196fffc88>): done = isinstance(e, bb.event.BuildCompleted) > system_stats.sample(e, force=done) if done: File "TOP/poky/meta/lib/buildstats.py", line 151, in SystemStats.sample(event=<bb.event.HeartbeatEvent object at 0x7f1196fffc88>, force=False): data + > b'\n') self.last_proc = now OSError: [Errno 28] No space left on device systemd(1)-+-Cooker(1601)-+-Worker(1699)---{python3}(1700) | |-pseudo(31632)---Worker (Fakeroo(31731)---{python3}(537) | `-python3(1602) $ gdb -p 1601 (gdb) py-bt Traceback (most recent call first): <built-in method waitpid of module object at remote 0x7f11a3ce57c8> File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 29, in poll pid, sts = os.waitpid(self.pid, flag) File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 51, in wait return self.poll(os.WNOHANG if timeout == 0.0 else 0) File "/usr/lib/python3.5/multiprocessing/process.py", line 121, in join res = self._popen.wait(timeout) File "/usr/lib/python3.5/multiprocessing/util.py", line 311, in _exit_function p.join() File "/usr/lib/python3.5/multiprocessing/process.py", line 252, in _bootstrap util._exit_function() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 74, in _launch code = process_obj._bootstrap() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 20, in __init__ self._launch(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 267, in _Popen return Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 212, in _Popen return _default_context.get_context().Process._Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/process.py", line 105, in start self._popen = self._Popen(self) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/server/process.py", line 471, in _startServer server.start() File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/daemonize.py", line 87, in createDaemon function() File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/server/process.py", line 399, in __init__ bb.daemonize.createDaemon(self._startServer, logfile) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/main.py", line 438, in setup_bitbake server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/main.py", line 366, in bitbake_main server_connection, ui_module = setup_bitbake(configParams, configuration) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/bin/bitbake", line 36, in <module> cookerdata.CookerConfiguration())) $ gdb -p 1699 (gdb) info thr Id Target Id Frame * 1 Thread 0x7fddd7354700 (LWP 1699) "Worker" 0x00007fddd611a603 in select () from /lib/x86_64-linux-gnu/libc.so.6 2 Thread 0x7fddd3865700 (LWP 1700) "python3" 0x00007fddd6f36720 in do_futex_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 (gdb) py-bt Traceback (most recent call first): <built-in method select of module object at remote 0x7fddd5de34f8> File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/bin/bitbake-worker", line 369, in serve (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/bin/bitbake-worker", line 494, in <module> worker.serve() (gdb) thr 2 [Switching to thread 2 (Thread 0x7fddd3865700 (LWP 1700))] #0 0x00007fddd6f36720 in do_futex_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 (gdb) py-bt Traceback (most recent call first): <built-in method acquire of _thread.lock object at remote 0x7fddd5f07e68> File "/usr/lib/python3.5/threading.py", line 297, in wait gotit = waiter.acquire(True, timeout) File "/usr/lib/python3.5/queue.py", line 173, in get self.not_empty.wait(remaining) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/bin/bitbake-worker", line 95, in worker_flush worker_queue_int = worker_queue_int + worker_queue.get(True, 1) File "/usr/lib/python3.5/threading.py", line 862, in run self._target(*self._args, **self._kwargs) File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner self.run() File "/usr/lib/python3.5/threading.py", line 882, in _bootstrap self._bootstrap_inner() $ gdb -p 31731 (gdb) info thr Id Target Id Frame * 1 Thread 0x7fe490dde700 (LWP 31731) "Worker (Fakeroo" 0x00007fe48f94c603 in select () from /lib/x86_64-linux-gnu/libc.so.6 2 Thread 0x7fe48d097700 (LWP 537) "python3" 0x00007fe490768720 in do_futex_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 (gdb) py-bt Traceback (most recent call first): <built-in method select of module object at remote 0x7fe48f6154f8> File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/bin/bitbake-worker", line 369, in serve (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/bin/bitbake-worker", line 494, in <module> worker.serve() (gdb) thr 2 [Switching to thread 2 (Thread 0x7fe48d097700 (LWP 537))] #0 0x00007fe490768720 in do_futex_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 (gdb) py-bt Traceback (most recent call first): <built-in method acquire of _thread.lock object at remote 0x7fe48c314da0> File "/usr/lib/python3.5/threading.py", line 297, in wait gotit = waiter.acquire(True, timeout) File "/usr/lib/python3.5/queue.py", line 173, in get self.not_empty.wait(remaining) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/bin/bitbake-worker", line 95, in worker_flush worker_queue_int = worker_queue_int + worker_queue.get(True, 1) File "/usr/lib/python3.5/threading.py", line 862, in run self._target(*self._args, **self._kwargs) File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner self.run() File "/usr/lib/python3.5/threading.py", line 882, in _bootstrap self._bootstrap_inner() $ gdb -p 1602 (gdb) info thr Id Target Id Frame * 1 Thread 0x7f11a5218700 (LWP 1602) "python3" 0x00007f11a3fe62e3 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) py-bt Traceback (most recent call first): <built-in method poll of select.epoll object at remote 0x7f11a12a3ae0> File "/usr/lib/python3.5/selectors.py", line 445, in select fd_event_list = self._epoll.poll(timeout, max_ev) File "/usr/lib/python3.5/asyncio/base_events.py", line 1388, in _run_once event_list = self._selector.select(timeout) File "/usr/lib/python3.5/asyncio/base_events.py", line 421, in run_forever self._run_once() File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/hashserv/server.py", line 438, in serve_forever self.loop.run_forever() File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run self._target(*self._args, **self._kwargs) File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap self.run() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 74, in _launch code = process_obj._bootstrap() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 20, in __init__ self._launch(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 267, in _Popen return Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 212, in _Popen return _default_context.get_context().Process._Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/process.py", line 105, in start self._popen = self._Popen(self) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/cooker.py", line 394, in handlePRServ self.hashserv.process.start() File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/cooker.py", line 1635, in pre_serve self.handlePRServ() File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/server/process.py", line 118, in main self.cooker.pre_serve() File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/server/process.py", line 113, in run ret = self.main() File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap self.run() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 74, in _launch code = process_obj._bootstrap() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 20, in __init__ self._launch(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 267, in _Popen return Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 212, in _Popen return _default_context.get_context().Process._Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/process.py", line 105, in start self._popen = self._Popen(self) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/server/process.py", line 471, in _startServer server.start() File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/daemonize.py", line 87, in createDaemon function() File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/server/process.py", line 399, in __init__ bb.daemonize.createDaemon(self._startServer, logfile) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/main.py", line 438, in setup_bitbake server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/lib/bb/main.py", line 366, in bitbake_main server_connection, ui_module = setup_bitbake(configParams, configuration) File "/home/gitlab-runner/builds/1b4b81df/0/blade-group/dev/clients/boxes/meta-shadow-os/poky/bitbake/bin/bitbake", line 36, in <module> cookerdata.CookerConfiguration()))systemd(1)-+-Cooker(1601)-+-Worker(1699)---{python3}(1700) | |-pseudo(31632)---Worker (Fakeroo(31731)---{python3}(537) | `-python3(1602) $ gdb -p 1601 (gdb) py-bt Traceback (most recent call first): <built-in method waitpid of module object at remote 0x7f11a3ce57c8> File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 29, in poll pid, sts = os.waitpid(self.pid, flag) File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 51, in wait return self.poll(os.WNOHANG if timeout == 0.0 else 0) File "/usr/lib/python3.5/multiprocessing/process.py", line 121, in join res = self._popen.wait(timeout) File "/usr/lib/python3.5/multiprocessing/util.py", line 311, in _exit_function p.join() File "/usr/lib/python3.5/multiprocessing/process.py", line 252, in _bootstrap util._exit_function() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 74, in _launch code = process_obj._bootstrap() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 20, in __init__ self._launch(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 267, in _Popen return Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 212, in _Popen return _default_context.get_context().Process._Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/process.py", line 105, in start self._popen = self._Popen(self) File "TOP/poky/bitbake/lib/bb/server/process.py", line 471, in _startServer server.start() File "TOP/poky/bitbake/lib/bb/daemonize.py", line 87, in createDaemon function() File "TOP/poky/bitbake/lib/bb/server/process.py", line 399, in __init__ bb.daemonize.createDaemon(self._startServer, logfile) File "TOP/poky/bitbake/lib/bb/main.py", line 438, in setup_bitbake server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset) File "TOP/poky/bitbake/lib/bb/main.py", line 366, in bitbake_main server_connection, ui_module = setup_bitbake(configParams, configuration) File "TOP/poky/bitbake/bin/bitbake", line 36, in <module> cookerdata.CookerConfiguration())) $ gdb -p 1699 (gdb) info thr Id Target Id Frame * 1 Thread 0x7fddd7354700 (LWP 1699) "Worker" 0x00007fddd611a603 in select () from /lib/x86_64-linux-gnu/libc.so.6 2 Thread 0x7fddd3865700 (LWP 1700) "python3" 0x00007fddd6f36720 in do_futex_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 (gdb) py-bt Traceback (most recent call first): <built-in method select of module object at remote 0x7fddd5de34f8> File "TOP/poky/bitbake/bin/bitbake-worker", line 369, in serve (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1) File "TOP/poky/bitbake/bin/bitbake-worker", line 494, in <module> worker.serve() (gdb) thr 2 [Switching to thread 2 (Thread 0x7fddd3865700 (LWP 1700))] #0 0x00007fddd6f36720 in do_futex_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 (gdb) py-bt Traceback (most recent call first): <built-in method acquire of _thread.lock object at remote 0x7fddd5f07e68> File "/usr/lib/python3.5/threading.py", line 297, in wait gotit = waiter.acquire(True, timeout) File "/usr/lib/python3.5/queue.py", line 173, in get self.not_empty.wait(remaining) File "TOP/poky/bitbake/bin/bitbake-worker", line 95, in worker_flush worker_queue_int = worker_queue_int + worker_queue.get(True, 1) File "/usr/lib/python3.5/threading.py", line 862, in run self._target(*self._args, **self._kwargs) File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner self.run() File "/usr/lib/python3.5/threading.py", line 882, in _bootstrap self._bootstrap_inner() $ gdb -p 31731 (gdb) info thr Id Target Id Frame * 1 Thread 0x7fe490dde700 (LWP 31731) "Worker (Fakeroo" 0x00007fe48f94c603 in select () from /lib/x86_64-linux-gnu/libc.so.6 2 Thread 0x7fe48d097700 (LWP 537) "python3" 0x00007fe490768720 in do_futex_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 (gdb) py-bt Traceback (most recent call first): <built-in method select of module object at remote 0x7fe48f6154f8> File "TOP/poky/bitbake/bin/bitbake-worker", line 369, in serve (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1) File "TOP/poky/bitbake/bin/bitbake-worker", line 494, in <module> worker.serve() (gdb) thr 2 [Switching to thread 2 (Thread 0x7fe48d097700 (LWP 537))] #0 0x00007fe490768720 in do_futex_wait () from /lib/x86_64-linux-gnu/libpthread.so.0 (gdb) py-bt Traceback (most recent call first): <built-in method acquire of _thread.lock object at remote 0x7fe48c314da0> File "/usr/lib/python3.5/threading.py", line 297, in wait gotit = waiter.acquire(True, timeout) File "/usr/lib/python3.5/queue.py", line 173, in get self.not_empty.wait(remaining) File "TOP/poky/bitbake/bin/bitbake-worker", line 95, in worker_flush worker_queue_int = worker_queue_int + worker_queue.get(True, 1) File "/usr/lib/python3.5/threading.py", line 862, in run self._target(*self._args, **self._kwargs) File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner self.run() File "/usr/lib/python3.5/threading.py", line 882, in _bootstrap self._bootstrap_inner() $ gdb -p 1602 (gdb) info thr Id Target Id Frame * 1 Thread 0x7f11a5218700 (LWP 1602) "python3" 0x00007f11a3fe62e3 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) py-bt Traceback (most recent call first): <built-in method poll of select.epoll object at remote 0x7f11a12a3ae0> File "/usr/lib/python3.5/selectors.py", line 445, in select fd_event_list = self._epoll.poll(timeout, max_ev) File "/usr/lib/python3.5/asyncio/base_events.py", line 1388, in _run_once event_list = self._selector.select(timeout) File "/usr/lib/python3.5/asyncio/base_events.py", line 421, in run_forever self._run_once() File "TOP/poky/bitbake/lib/hashserv/server.py", line 438, in serve_forever self.loop.run_forever() File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run self._target(*self._args, **self._kwargs) File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap self.run() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 74, in _launch code = process_obj._bootstrap() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 20, in __init__ self._launch(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 267, in _Popen return Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 212, in _Popen return _default_context.get_context().Process._Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/process.py", line 105, in start self._popen = self._Popen(self) File "TOP/poky/bitbake/lib/bb/cooker.py", line 394, in handlePRServ self.hashserv.process.start() File "TOP/poky/bitbake/lib/bb/cooker.py", line 1635, in pre_serve self.handlePRServ() File "TOP/poky/bitbake/lib/bb/server/process.py", line 118, in main self.cooker.pre_serve() File "TOP/poky/bitbake/lib/bb/server/process.py", line 113, in run ret = self.main() File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap self.run() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 74, in _launch code = process_obj._bootstrap() File "/usr/lib/python3.5/multiprocessing/popen_fork.py", line 20, in __init__ self._launch(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 267, in _Popen return Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/context.py", line 212, in _Popen return _default_context.get_context().Process._Popen(process_obj) File "/usr/lib/python3.5/multiprocessing/process.py", line 105, in start self._popen = self._Popen(self) File "TOP/poky/bitbake/lib/bb/server/process.py", line 471, in _startServer server.start() File "TOP/poky/bitbake/lib/bb/daemonize.py", line 87, in createDaemon function() File "TOP/poky/bitbake/lib/bb/server/process.py", line 399, in __init__ bb.daemonize.createDaemon(self._startServer, logfile) File "TOP/poky/bitbake/lib/bb/main.py", line 438, in setup_bitbake server = bb.server.process.BitBakeServer(lock, sockname, configuration, featureset) File "TOP/poky/bitbake/lib/bb/main.py", line 366, in bitbake_main server_connection, ui_module = setup_bitbake(configParams, configuration) File "TOP/poky/bitbake/bin/bitbake", line 36, in <module> cookerdata.CookerConfiguration())) 87857 1 akuster808 2020-07-30 08:02:58 +0000 Ill take a look at this and see how far I get. 88530 2 akuster808 2020-10-30 21:06:51 +0000 this triggers the monitor to shut things down but did not create the traceback. BB_DISKMON_DIRS ??= "\ STOPTASKS,${TMPDIR},1000G,100G \ STOPTASKS,${DL_DIR},1000G,100G \ STOPTASKS,${SSTATE_DIR},1000G,100G \ STOPTASKS,/tmp,1000G,100G \ ABORT,${TMPDIR},100M,1K \ ABORT,${DL_DIR},100M,1K \ ABORT,${SSTATE_DIR},100M,1K \ ABORT,/tmp,10M,1K" 89962 3 akuster808 2021-04-15 15:06:22 +0000 I can not reproduce this. 94025 4 randy.macleod 2022-09-13 00:56:47 +0000 If you can still reproduce this please, re-open, perhaps with additional details about how to trigger the issue.