From f76452e0760653482afbd4144d60332d4284b67d Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 21 Feb 2024 18:58:07 -0800 Subject: bin/socks5-server.py: Migrate to asyncio.run() Signed-off-by: Zac Medico --- bin/socks5-server.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/bin/socks5-server.py b/bin/socks5-server.py index e898835ff..640c89d5a 100644 --- a/bin/socks5-server.py +++ b/bin/socks5-server.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # SOCKSv5 proxy server for network-sandbox -# Copyright 2015-2022 Gentoo Authors +# Copyright 2015-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import asyncio @@ -218,27 +218,21 @@ class Socks5Server: return -if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"Usage: {sys.argv[0]} ") - sys.exit(1) - - loop = asyncio.new_event_loop() +async def run_socks5_server(socket_path): s = Socks5Server() - server = loop.run_until_complete( - asyncio.start_unix_server(s.handle_proxy_conn, sys.argv[1]) - ) + server = await asyncio.start_unix_server(s.handle_proxy_conn, socket_path) - ret = 0 try: - try: - loop.run_forever() - except KeyboardInterrupt: - pass - except: - ret = 1 + await asyncio.get_running_loop().create_future() finally: server.close() - loop.run_until_complete(server.wait_closed()) - loop.close() - os.unlink(sys.argv[1]) + await server.wait_closed() + os.unlink(socket_path) + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + asyncio.run(run_socks5_server(sys.argv[1])) -- cgit v1.2.3-65-gdbad From d718cea94a180042b2285698b2c19113c5d25987 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 21 Feb 2024 22:41:49 -0800 Subject: _get_running_loop: Support real asyncio.run When called via the real asyncio.run implementation, wrap the running asyncio loop. Otherwise, it's not possible to call portage libraries via the real asyncio.run without triggering Future "attached to a different loop" errors. Bug: https://bugs.gentoo.org/761538 Signed-off-by: Zac Medico --- lib/portage/util/futures/_asyncio/__init__.py | 28 +++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py index 22241f335..4eecc46a8 100644 --- a/lib/portage/util/futures/_asyncio/__init__.py +++ b/lib/portage/util/futures/_asyncio/__init__.py @@ -325,13 +325,37 @@ def _safe_loop(): def _get_running_loop(): + """ + This calls the real asyncio get_running_loop() and wraps that with + portage's internal AsyncioEventLoop wrapper. If there is no running + asyncio event loop but portage has a reference to another running + loop in this thread, then use that instead. + + This behavior enables portage internals to use the real asyncio.run + while remaining compatible with internal code that does not use the + real asyncio.run. + """ + try: + _loop = _real_asyncio.get_running_loop() + except RuntimeError: + _loop = None + with _thread_weakrefs.lock: if _thread_weakrefs.pid == portage.getpid(): try: loop = _thread_weakrefs.loops[threading.get_ident()] except KeyError: - return None - return loop if loop.is_running() else None + pass + else: + if _loop is loop._loop: + return loop + elif _loop is None: + return loop if loop.is_running() else None + + # If _loop it not None here it means it was probably a temporary + # loop created by asyncio.run, so we don't try to cache it, and + # just return a temporary wrapper. + return None if _loop is None else _AsyncioEventLoop(loop=_loop) def _thread_weakrefs_atexit(): -- cgit v1.2.3-65-gdbad From fbaaa4a733aaadc2744b656527756ac4e2b7ab58 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Wed, 21 Feb 2024 22:47:33 -0800 Subject: socks5: Use real asyncio.run Use real asyncio.run to demonstrate that it is compatible with portage internals. Since the socks5 ProxyManager uses the process.spawn function, the internal _running_loop function needs to return the correct loop for use in the wait method of MultiprocessingProcess, or else it will lead to Future "attached to a different loop" errors. Bug: https://bugs.gentoo.org/761538 Signed-off-by: Zac Medico --- lib/portage/tests/util/test_socks5.py | 45 +++++++++++++++++++---------------- lib/portage/util/socks5.py | 30 ++++++++++++++++------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py index 987b41af2..7b33cb3f6 100644 --- a/lib/portage/tests/util/test_socks5.py +++ b/lib/portage/tests/util/test_socks5.py @@ -1,6 +1,7 @@ -# Copyright 2019-2021 Gentoo Authors +# Copyright 2019-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +import asyncio import functools import shutil import socket @@ -10,7 +11,6 @@ import time import portage from portage.tests import TestCase -from portage.util._eventloop.global_event_loop import global_event_loop from portage.util import socks5 from portage.const import PORTAGE_BIN_PATH @@ -88,18 +88,20 @@ class AsyncHTTPServerTestCase(TestCase): if f is not None: f.close() - def test_http_server(self): + async def _test_http_server(self): + asyncio.run(self._test_http_server()) + + async def _test_http_server(self): host = "127.0.0.1" content = b"Hello World!\n" path = "/index.html" - loop = global_event_loop() + + loop = asyncio.get_running_loop() for i in range(2): with AsyncHTTPServer(host, {path: content}, loop) as server: for j in range(2): - result = loop.run_until_complete( - loop.run_in_executor( - None, self._fetch_directly, host, server.server_port, path - ) + result = await loop.run_in_executor( + None, self._fetch_directly, host, server.server_port, path ) self.assertEqual(result, content) @@ -177,7 +179,10 @@ class Socks5ServerTestCase(TestCase): return f.read() def test_socks5_proxy(self): - loop = global_event_loop() + asyncio.run(self._test_socks5_proxy()) + + async def _test_socks5_proxy(self): + loop = asyncio.get_running_loop() host = "127.0.0.1" content = b"Hello World!" @@ -193,20 +198,18 @@ class Socks5ServerTestCase(TestCase): } proxy = socks5.get_socks5_proxy(settings) - loop.run_until_complete(socks5.proxy.ready()) - - result = loop.run_until_complete( - loop.run_in_executor( - None, - self._fetch_via_proxy, - proxy, - host, - server.server_port, - path, - ) + await socks5.proxy.ready() + + result = await loop.run_in_executor( + None, + self._fetch_via_proxy, + proxy, + host, + server.server_port, + path, ) self.assertEqual(result, content) finally: - socks5.proxy.stop() + await socks5.proxy.stop() shutil.rmtree(tempdir) diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index 6c68ff410..74592aeef 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -2,15 +2,16 @@ # Copyright 2015-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 +import asyncio import errno import os import socket +from typing import Union import portage.data from portage import _python_interpreter from portage.data import portage_gid, portage_uid, userpriv_groups from portage.process import atexit_register, spawn -from portage.util.futures import asyncio class ProxyManager: @@ -57,23 +58,36 @@ class ProxyManager: **spawn_kwargs, ) - def stop(self): + def stop(self) -> Union[None, asyncio.Future]: """ Stop the SOCKSv5 server. + + If there is a running asyncio event loop then asyncio.Future is + returned which should be used to wait for the server process + to exit. """ + future = None + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None if self._proc is not None: self._proc.terminate() - loop = asyncio.get_event_loop() - if self._proc_waiter is None: - self._proc_waiter = asyncio.ensure_future(self._proc.wait(), loop) - if loop.is_running(): - self._proc_waiter.add_done_callback(lambda future: future.result()) + if loop is None: + asyncio.run(self._proc.wait()) else: - loop.run_until_complete(self._proc_waiter) + if self._proc_waiter is None: + self._proc_waiter = asyncio.ensure_future(self._proc.wait(), loop) + future = asyncio.shield(self._proc_waiter) + + if loop is not None and future is None: + future = loop.create_future() + future.set_result(None) self.socket_path = None self._proc = None self._proc_waiter = None + return future def is_running(self): """ -- cgit v1.2.3-65-gdbad From 92ff02b9189f8350f44e134d538319e4037f3f71 Mon Sep 17 00:00:00 2001 From: Gábor Oszkár Dénes Date: Wed, 14 Feb 2024 18:40:24 +0100 Subject: emerge: Skip installed packages with emptytree in depgraph selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running emerge with emptytree tries to find the best match for every atom it needs to install. Sometimes the best matches would be already installed packages (with `operation=nomerge`), but these packages would be silently skipped with full emptytree installation. This change makes sure that emerge attempts to install every package. If the package has unmet requirements, emerge will complain. Bug: https://bugs.gentoo.org/651018 Signed-off-by: Gábor Oszkár Dénes Closes: https://github.com/gentoo/portage/pull/1272 Signed-off-by: Sam James --- NEWS | 6 + lib/_emerge/depgraph.py | 13 ++ lib/portage/tests/resolver/test_depth.py | 8 +- .../test_emptytree_reinstall_unsatisfiability.py | 137 +++++++++++++++++++++ lib/portage/tests/resolver/test_useflags.py | 6 +- 5 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 lib/portage/tests/resolver/test_emptytree_reinstall_unsatisfiability.py diff --git a/NEWS b/NEWS index 3fbc72786..94be26de8 100644 --- a/NEWS +++ b/NEWS @@ -6,6 +6,12 @@ Release notes take the form of the following optional categories: * Bug fixes * Cleanups +portage-3.0.63 (UNRELEASED) +-------------- + +Bug fixes: +* emerge: Skip installed packages with emptytree in depgraph selection (bug #651018). + portage-3.0.62 (2024-02-22) -------------- diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index 70b83ee1f..ea96bd58c 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -7639,6 +7639,19 @@ class depgraph: if pkg.installed and root_slot in self._rebuild.reinstall_list: continue + if ( + empty + and pkg.installed + and not self._frozen_config.excluded_pkgs.findAtomForPackage( + pkg, modified_use=self._pkg_use_enabled(pkg) + ) + ): + # With --emptytree option we assume no packages + # are installed, so we do not select them. + # But we allow installed packages to satisfy dependency requirements + # if they're explicitly excluded, so we allow them to be selected. + continue + if ( not pkg.installed and self._frozen_config.excluded_pkgs.findAtomForPackage( diff --git a/lib/portage/tests/resolver/test_depth.py b/lib/portage/tests/resolver/test_depth.py index 9c5289f7d..ab5f8e7ec 100644 --- a/lib/portage/tests/resolver/test_depth.py +++ b/lib/portage/tests/resolver/test_depth.py @@ -1,4 +1,4 @@ -# Copyright 2011-2020 Gentoo Authors +# Copyright 2011-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from portage.tests import TestCase @@ -318,6 +318,12 @@ class ResolverDepthTestCase(TestCase): "sys-fs/udev-164", ], ), + ResolverPlaygroundTestCase( + ["@world"], + options={"--emptytree": True, "--exclude": ["dev-libs/B"]}, + success=True, + mergelist=["dev-libs/C-2", "dev-libs/A-2"], + ), ) playground = ResolverPlayground( diff --git a/lib/portage/tests/resolver/test_emptytree_reinstall_unsatisfiability.py b/lib/portage/tests/resolver/test_emptytree_reinstall_unsatisfiability.py new file mode 100644 index 000000000..fcdc01d7f --- /dev/null +++ b/lib/portage/tests/resolver/test_emptytree_reinstall_unsatisfiability.py @@ -0,0 +1,137 @@ +# Copyright 2024 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from portage.tests import TestCase +from portage.tests.resolver.ResolverPlayground import ( + ResolverPlayground, + ResolverPlaygroundTestCase, +) + + +class EmptytreeReinstallUnsatisfiabilityTestCase(TestCase): + def testEmptytreeReinstallUnsatisfiability(self): + """ + Tests to check if emerge fails and complains when --emptytree + package dependency graph reinstall is unsatisfied, even if the already + installed packages successfully satisfy the dependency tree. + + See bug #651018 where emerge silently skips package + reinstalls because of unsatisfied use flag requirements. + """ + ebuilds = { + "dev-libs/A-1": { + "DEPEND": "dev-libs/B", + "RDEPEND": "dev-libs/B", + "EAPI": "2", + }, + "dev-libs/B-1": { + "DEPEND": "dev-libs/C[foo]", + "RDEPEND": "dev-libs/C[foo]", + "EAPI": "2", + }, + "dev-libs/C-1": { + "IUSE": "foo", + "EAPI": "2", + }, + "dev-libs/X-1": { + "DEPEND": "dev-libs/Y[-baz]", + "RDEPEND": "dev-libs/Y[-baz]", + "EAPI": "2", + }, + "dev-libs/Y-1": { + "IUSE": "baz", + "EAPI": "2", + }, + "dev-libs/Z-1": { + "DEPEND": "dev-libs/W", + "RDEPEND": "dev-libs/W", + "EAPI": "2", + }, + "dev-libs/W-1": { + "EAPI": "2", + }, + } + + installed = { + "dev-libs/A-1": { + "DEPEND": "dev-libs/B", + "RDEPEND": "dev-libs/B", + "EAPI": "2", + }, + "dev-libs/B-1": { + "DEPEND": "dev-libs/C[foo]", + "RDEPEND": "dev-libs/C[foo]", + "EAPI": "2", + }, + "dev-libs/C-1": { + "IUSE": "foo", + "USE": "foo", + "EAPI": "2", + }, + "dev-libs/X-1": { + "DEPEND": "dev-libs/Y[-baz]", + "RDEPEND": "dev-libs/Y[-baz]", + "EAPI": "2", + }, + "dev-libs/Y-1": { + "IUSE": "baz", + "USE": "-baz", + "EAPI": "2", + }, + "dev-libs/Z-1": { + "DEPEND": "dev-libs/W", + "RDEPEND": "dev-libs/W", + "EAPI": "2", + }, + "dev-libs/W-1": { + "EAPI": "2", + }, + } + + user_config = { + "package.use": ("dev-libs/Y baz",), + "package.mask": ("dev-libs/W",), + } + + world = ["dev-libs/X"] + + test_cases = ( + ResolverPlaygroundTestCase( + ["dev-libs/A"], + options={"--emptytree": True}, + success=False, + mergelist=["dev-libs/C-1", "dev-libs/B-1", "dev-libs/A-1"], + use_changes={"dev-libs/C-1": {"foo": True}}, + ), + ResolverPlaygroundTestCase( + ["dev-libs/A"], + options={"--emptytree": True, "--exclude": ["dev-libs/C"]}, + success=True, + mergelist=["dev-libs/B-1", "dev-libs/A-1"], + ), + ResolverPlaygroundTestCase( + ["@world"], + options={"--emptytree": True}, + success=False, + mergelist=["dev-libs/Y-1", "dev-libs/X-1"], + use_changes={"dev-libs/Y-1": {"baz": False}}, + ), + ResolverPlaygroundTestCase( + ["dev-libs/Z"], + options={"--emptytree": True}, + success=False, + ), + ) + + playground = ResolverPlayground( + ebuilds=ebuilds, + installed=installed, + user_config=user_config, + world=world, + ) + try: + for test_case in test_cases: + playground.run_TestCase(test_case) + self.assertEqual(test_case.test_success, True, test_case.fail_msg) + finally: + playground.cleanup() diff --git a/lib/portage/tests/resolver/test_useflags.py b/lib/portage/tests/resolver/test_useflags.py index 86684f7f2..142a31c7f 100644 --- a/lib/portage/tests/resolver/test_useflags.py +++ b/lib/portage/tests/resolver/test_useflags.py @@ -1,4 +1,4 @@ -# Copyright 2014 Gentoo Foundation +# Copyright 2014-2024 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 import sys @@ -292,8 +292,8 @@ class UseFlagsTestCase(TestCase): "--usepkg": True, }, success=False, - mergelist=["[binary]dev-libs/A-2", "dev-libs/B-1"], - slot_collision_solutions=[], + mergelist=None, + slot_collision_solutions=None, ), ) -- cgit v1.2.3-65-gdbad From 3cc986f87ddda86ee93770e03cca06346aee54c5 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Thu, 22 Feb 2024 22:06:14 -0800 Subject: AsyncioEventLoop: Call process.run_exitfuncs() before close For the event loop running in the main thread, call process.run_exitfuncs() before close with the event loop running so that anything attached can clean itself up (like the socks5 ProxyManager for bug 925240). This is necessary because process.spawn uses the event loop to implement the new returnproc parameter related to bug 916566. Bug: https://bugs.gentoo.org/916566 Bug: https://bugs.gentoo.org/925240 Signed-off-by: Zac Medico --- lib/portage/tests/util/test_socks5.py | 51 ++++++++++++++++++++++- lib/portage/util/_eventloop/asyncio_event_loop.py | 44 +++++++++++++++---- lib/portage/util/socks5.py | 16 ++++++- 3 files changed, 101 insertions(+), 10 deletions(-) diff --git a/lib/portage/tests/util/test_socks5.py b/lib/portage/tests/util/test_socks5.py index 7b33cb3f6..4a6d08169 100644 --- a/lib/portage/tests/util/test_socks5.py +++ b/lib/portage/tests/util/test_socks5.py @@ -12,6 +12,8 @@ import time import portage from portage.tests import TestCase from portage.util import socks5 +from portage.util.futures.executor.fork import ForkExecutor +from portage.util._eventloop.global_event_loop import global_event_loop from portage.const import PORTAGE_BIN_PATH from http.server import BaseHTTPRequestHandler, HTTPServer @@ -189,8 +191,10 @@ class Socks5ServerTestCase(TestCase): path = "/index.html" proxy = None tempdir = tempfile.mkdtemp() + previous_exithandlers = portage.process._exithandlers try: + portage.process._exithandlers = [] with AsyncHTTPServer(host, {path: content}, loop) as server: settings = { "PORTAGE_TMPDIR": tempdir, @@ -211,5 +215,50 @@ class Socks5ServerTestCase(TestCase): self.assertEqual(result, content) finally: - await socks5.proxy.stop() + try: + # Also run_exitfuncs to test atexit hook cleanup. + await socks5.proxy.stop() + self.assertNotEqual(portage.process._exithandlers, []) + portage.process.run_exitfuncs() + self.assertEqual(portage.process._exithandlers, []) + finally: + portage.process._exithandlers = previous_exithandlers + shutil.rmtree(tempdir) + + +class Socks5ServerLoopCloseTestCase(TestCase): + """ + For bug 925240, test that the socks5 proxy is automatically + terminated when the main event loop is closed, using a subprocess + for isolation. + """ + + def testSocks5ServerLoopClose(self): + asyncio.run(self._testSocks5ServerLoopClose()) + + async def _testSocks5ServerLoopClose(self): + loop = asyncio.get_running_loop() + self.assertEqual( + await loop.run_in_executor( + ForkExecutor(loop=loop), self._testSocks5ServerLoopCloseSubprocess + ), + True, + ) + + @staticmethod + def _testSocks5ServerLoopCloseSubprocess(): + loop = global_event_loop() + tempdir = tempfile.mkdtemp() + try: + settings = { + "PORTAGE_TMPDIR": tempdir, + "PORTAGE_BIN_PATH": PORTAGE_BIN_PATH, + } + + socks5.get_socks5_proxy(settings) + loop.run_until_complete(socks5.proxy.ready()) + finally: + loop.close() shutil.rmtree(tempdir) + + return not socks5.proxy.is_running() diff --git a/lib/portage/util/_eventloop/asyncio_event_loop.py b/lib/portage/util/_eventloop/asyncio_event_loop.py index b9e96bb20..ee9e4c60e 100644 --- a/lib/portage/util/_eventloop/asyncio_event_loop.py +++ b/lib/portage/util/_eventloop/asyncio_event_loop.py @@ -1,8 +1,9 @@ -# Copyright 2018-2023 Gentoo Authors +# Copyright 2018-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import os import signal +import threading import asyncio as _real_asyncio from asyncio.events import AbstractEventLoop as _AbstractEventLoop @@ -14,6 +15,7 @@ except ImportError: PidfdChildWatcher = None import portage +from portage.util import socks5 class AsyncioEventLoop(_AbstractEventLoop): @@ -25,18 +27,14 @@ class AsyncioEventLoop(_AbstractEventLoop): def __init__(self, loop=None): loop = loop or _real_asyncio.get_event_loop() self._loop = loop - self.run_until_complete = ( - self._run_until_complete - if portage._internal_caller - else loop.run_until_complete - ) + self.run_until_complete = self._run_until_complete self.call_soon = loop.call_soon self.call_soon_threadsafe = loop.call_soon_threadsafe self.call_later = loop.call_later self.call_at = loop.call_at self.is_running = loop.is_running self.is_closed = loop.is_closed - self.close = loop.close + self.close = self._close self.create_future = ( loop.create_future if hasattr(loop, "create_future") @@ -55,10 +53,36 @@ class AsyncioEventLoop(_AbstractEventLoop): self.get_debug = loop.get_debug self._wakeup_fd = -1 self._child_watcher = None + # Used to drop recursive calls to _close. + self._closing = False + # Initialized in _run_until_complete. + self._is_main = None if portage._internal_caller: loop.set_exception_handler(self._internal_caller_exception_handler) + def _close(self): + """ + Before closing the main loop, run portage.process.run_exitfuncs() + with the event loop running so that anything attached can clean + itself up (like the socks5 ProxyManager for bug 925240). + """ + if not (self._closing or self.is_closed()): + self._closing = True + if self._is_main: + self.run_until_complete(self._close_main()) + self._loop.close() + self._closing = False + + async def _close_main(self): + # Even though this has an exit hook, invoke it here so that + # we can properly wait for it and avoid messages like this: + # [ERROR] Task was destroyed but it is pending! + if socks5.proxy.is_running(): + await socks5.proxy.stop() + + portage.process.run_exitfuncs() + @staticmethod def _internal_caller_exception_handler(loop, context): """ @@ -139,6 +163,12 @@ class AsyncioEventLoop(_AbstractEventLoop): In order to avoid potential interference with API consumers, this implementation is only used when portage._internal_caller is True. """ + if self._is_main is None: + self._is_main = threading.current_thread() is threading.main_thread() + + if not portage._internal_caller: + return self._loop.run_until_complete(future) + if self._wakeup_fd != -1: signal.set_wakeup_fd(self._wakeup_fd) self._wakeup_fd = -1 diff --git a/lib/portage/util/socks5.py b/lib/portage/util/socks5.py index 74592aeef..f8fcdf9fc 100644 --- a/lib/portage/util/socks5.py +++ b/lib/portage/util/socks5.py @@ -8,6 +8,13 @@ import os import socket from typing import Union +import portage + +portage.proxy.lazyimport.lazyimport( + globals(), + "portage.util._eventloop.global_event_loop:global_event_loop", +) + import portage.data from portage import _python_interpreter from portage.data import portage_gid, portage_uid, userpriv_groups @@ -74,10 +81,15 @@ class ProxyManager: if self._proc is not None: self._proc.terminate() if loop is None: - asyncio.run(self._proc.wait()) + # In this case spawn internals would have used + # portage's global loop when attaching a waiter to + # self._proc, so we are obligated to use that. + global_event_loop().run_until_complete(self._proc.wait()) else: if self._proc_waiter is None: - self._proc_waiter = asyncio.ensure_future(self._proc.wait(), loop) + self._proc_waiter = asyncio.ensure_future( + self._proc.wait(), loop=loop + ) future = asyncio.shield(self._proc_waiter) if loop is not None and future is None: -- cgit v1.2.3-65-gdbad From 01d06eb1d9dc8c4b16cbc9a6567ed0c07df5901a Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Fri, 23 Feb 2024 18:45:58 -0800 Subject: Fix python 3.9 CI jobs since 92ff02b9189f Use emerge -e to fix this error we've seen since 92ff02b9189f for python 3.9 CI jobs: https://github.com/gentoo/portage/actions/runs/8014796128/job/21893963019 test_portage_baseline[binhost emerge-gpkg] - AssertionError: 'emerge' failed with args '('-e', '--getbinpkgonly', 'dev-libs/A')' emerge: there are no binary packages to satisfy "dev-libs/B[flag]". (dependency required by "dev-libs/A-1::test_repo" [binary]) (dependency required by "dev-libs/A" [argument]) Fixes: 92ff02b9189f ("emerge: Skip installed packages with emptytree in depgraph selection") Bug: https://bugs.gentoo.org/651018 Signed-off-by: Zac Medico --- lib/portage/tests/emerge/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/tests/emerge/conftest.py b/lib/portage/tests/emerge/conftest.py index d9aec7041..580d1e09a 100644 --- a/lib/portage/tests/emerge/conftest.py +++ b/lib/portage/tests/emerge/conftest.py @@ -805,7 +805,7 @@ def _generate_all_baseline_commands(playground, binhost): test_commands["binhost emerge"] = Noop() else: # The next emerge has been added to split this test from the rest: - make_package = Emerge("--buildpkg", "dev-libs/A") + make_package = Emerge("-e", "--buildpkg", "dev-libs/A") getbinpkgonly = Emerge( "-e", "--getbinpkgonly", -- cgit v1.2.3-65-gdbad From acb69a6f234bd412e95e76f5c1db1b1f5b8e1dc5 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Fri, 23 Feb 2024 19:52:47 -0800 Subject: SchedulerInterface/PollScheduler: Add _loop property This allows async_aux_get to easily verify the identity of the underlying loop so that this assertion will not fail: _start_with_metadata (settings.configdict["pkg"]["SRC_URI"],) = aux_get_task.future.result() ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/site-packages/portage/dbapi/porttree.py", line 786, in async_aux_get raise AssertionError( AssertionError: async_aux_get called from thread <_MainThread(MainThread, started 281473559502880)> with loop <_emerge.Scheduler.Scheduler._iface_class object at 0xffff8e3a8840> Terminated Fixes: 389bb304abf5 ("async_aux_get: Use EbuildMetadataPhase deallocate_config future") Bug: https://bugs.gentoo.org/925333 Signed-off-by: Zac Medico --- lib/_emerge/PollScheduler.py | 9 ++++++++- lib/portage/dbapi/porttree.py | 2 +- lib/portage/tests/ebuild/test_doebuild_spawn.py | 3 ++- lib/portage/tests/ebuild/test_ipc_daemon.py | 3 ++- lib/portage/util/_async/SchedulerInterface.py | 9 ++++++++- 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/_emerge/PollScheduler.py b/lib/_emerge/PollScheduler.py index e5dffd8af..b5bfd20b7 100644 --- a/lib/_emerge/PollScheduler.py +++ b/lib/_emerge/PollScheduler.py @@ -1,4 +1,4 @@ -# Copyright 1999-2023 Gentoo Authors +# Copyright 1999-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import threading @@ -39,6 +39,13 @@ class PollScheduler: self._event_loop, is_background=self._is_background ) + @property + def _loop(self): + """ + Returns the real underlying asyncio loop. + """ + return self._event_loop._loop + def _is_background(self): return self._background diff --git a/lib/portage/dbapi/porttree.py b/lib/portage/dbapi/porttree.py index 4eebe1183..de6aa5c82 100644 --- a/lib/portage/dbapi/porttree.py +++ b/lib/portage/dbapi/porttree.py @@ -775,7 +775,7 @@ class portdbapi(dbapi): try: if ( threading.current_thread() is threading.main_thread() - and loop is asyncio._safe_loop() + and loop._loop is asyncio._safe_loop()._loop ): # In this case use self._doebuild_settings_lock to manage concurrency. deallocate_config = loop.create_future() diff --git a/lib/portage/tests/ebuild/test_doebuild_spawn.py b/lib/portage/tests/ebuild/test_doebuild_spawn.py index 9fb2c7fdd..cac844f8f 100644 --- a/lib/portage/tests/ebuild/test_doebuild_spawn.py +++ b/lib/portage/tests/ebuild/test_doebuild_spawn.py @@ -1,4 +1,4 @@ -# Copyright 2010-2015 Gentoo Foundation +# Copyright 2010-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import textwrap @@ -86,6 +86,7 @@ class DoebuildSpawnTestCase(TestCase): open(os.path.join(settings["T"], "environment"), "wb").close() scheduler = SchedulerInterface(global_event_loop()) + self.assertTrue(scheduler._loop is global_event_loop()._loop) for phase in ("_internal_test",): # Test EbuildSpawnProcess by calling doebuild.spawn() with # returnpid=False. This case is no longer used by portage diff --git a/lib/portage/tests/ebuild/test_ipc_daemon.py b/lib/portage/tests/ebuild/test_ipc_daemon.py index 0beb69ddf..b8777fe94 100644 --- a/lib/portage/tests/ebuild/test_ipc_daemon.py +++ b/lib/portage/tests/ebuild/test_ipc_daemon.py @@ -1,4 +1,4 @@ -# Copyright 2010-2023 Gentoo Authors +# Copyright 2010-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import tempfile @@ -77,6 +77,7 @@ class IpcDaemonTestCase(TestCase): task_scheduler = TaskScheduler( iter([daemon, proc]), max_jobs=2, event_loop=event_loop ) + self.assertTrue(task_scheduler._loop is event_loop._loop) self.received_command = False diff --git a/lib/portage/util/_async/SchedulerInterface.py b/lib/portage/util/_async/SchedulerInterface.py index 43a42adff..485958491 100644 --- a/lib/portage/util/_async/SchedulerInterface.py +++ b/lib/portage/util/_async/SchedulerInterface.py @@ -1,4 +1,4 @@ -# Copyright 2012-2021 Gentoo Authors +# Copyright 2012-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import gzip @@ -49,6 +49,13 @@ class SchedulerInterface(SlotObject): for k in self._event_loop_attrs: setattr(self, k, getattr(event_loop, k)) + @property + def _loop(self): + """ + Returns the real underlying asyncio loop. + """ + return self._event_loop._loop + @staticmethod def _return_false(): return False -- cgit v1.2.3-65-gdbad From d06515e1424be60c8ed7af6ed2a80d56145b85c5 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sat, 24 Feb 2024 05:28:22 -0800 Subject: _dynamic_deps_preload: Fix settings reference Use the settings reference from the config pool as intended to prevent a KeyError similar to bug 924319 but triggered by emerge --dynamic-deps. Fixes: f9ea958018c0 ("MetadataRegen: Use EbuildMetadataPhase deallocate_config") Bug: https://bugs.gentoo.org/925350 Signed-off-by: Zac Medico --- lib/_emerge/depgraph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_emerge/depgraph.py b/lib/_emerge/depgraph.py index ea96bd58c..6853ec491 100644 --- a/lib/_emerge/depgraph.py +++ b/lib/_emerge/depgraph.py @@ -783,7 +783,7 @@ class depgraph: ebuild_hash=ebuild_hash, portdb=portdb, repo_path=repo_path, - settings=portdb.doebuild_settings, + settings=settings, deallocate_config=deallocate_config, ) proc.addExitListener(self._dynamic_deps_proc_exit(pkg, fake_vartree)) -- cgit v1.2.3-65-gdbad From 3f4250dc7d32e9915224b1c9c4bc04c2740abcda Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Fri, 23 Feb 2024 12:35:04 -0800 Subject: process.spawn: Fix logic for missing libc.unshare on musl Fix unshare_* variables to be False when the libc is missing, libc.unshare is missing, or libc.unshare fails. Also, if socket.sethostname is missing then _exec2 needs libc for the network-sandbox sethostname call which is wrapped by a blanket Exception handler. Fixes: 419cce79f908 ("process._exec: Use _start_fork for os.fork() error handling") Bug: https://bugs.gentoo.org/925311 Signed-off-by: Zac Medico --- lib/portage/process.py | 213 +++++++++++++++++++++++++------------------------ 1 file changed, 110 insertions(+), 103 deletions(-) diff --git a/lib/portage/process.py b/lib/portage/process.py index f4758c824..d16262e75 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -956,114 +956,119 @@ def _exec( signal.signal(signal.SIGQUIT, signal.SIG_DFL) # Unshare (while still uid==0) + have_unshare = False + libc = None if unshare_net or unshare_ipc or unshare_mount or unshare_pid: filename = find_library("c") if filename is not None: libc = LoadLibrary(filename) if libc is not None: - # unshare() may not be supported by libc - if not hasattr(libc, "unshare"): - unshare_net = False - unshare_ipc = False - unshare_mount = False - unshare_pid = False - else: - # Since a failed unshare call could corrupt process - # state, first validate that the call can succeed. - # The parent process should call _unshare_validate - # before it forks, so that all child processes can - # reuse _unshare_validate results that have been - # cached by the parent process. - errno_value = _unshare_validate(unshare_flags) - if errno_value == 0 and libc.unshare(unshare_flags) != 0: - errno_value = ctypes.get_errno() - if errno_value != 0: - involved_features = [] - if unshare_ipc: - involved_features.append("ipc-sandbox") - if unshare_mount: - involved_features.append("mount-sandbox") - if unshare_net: - involved_features.append("network-sandbox") - if unshare_pid: - involved_features.append("pid-sandbox") - - writemsg( - 'Unable to unshare: %s (for FEATURES="%s")\n' - % ( - errno.errorcode.get(errno_value, "?"), - " ".join(involved_features), - ), - noiselevel=-1, - ) - else: - if unshare_pid: - # pid namespace requires us to become init - binary, myargs = ( - portage._python_interpreter, - [ - portage._python_interpreter, - os.path.join(portage._bin_path, "pid-ns-init"), - _unicode_encode("" if uid is None else str(uid)), - _unicode_encode("" if gid is None else str(gid)), - _unicode_encode( - "" - if groups is None - else ",".join(str(group) for group in groups) - ), - _unicode_encode( - "" if umask is None else str(umask) - ), - _unicode_encode( - ",".join(str(fd) for fd in fd_pipes) - ), - binary, - ] - + myargs, - ) - uid = None - gid = None - groups = None - umask = None - - # Use _start_fork for os.fork() error handling, ensuring - # that if exec fails then the child process will display - # a traceback before it exits via os._exit to suppress any - # finally blocks from parent's call stack (bug 345289). - main_child_pid = _start_fork( - _exec2, - args=( - binary, - myargs, - env, - gid, - groups, - uid, - umask, - cwd, - pre_exec, - unshare_net, - unshare_ipc, - unshare_mount, - unshare_pid, - ), - fd_pipes=None, - close_fds=False, - ) - - # Execute a supervisor process which will forward - # signals to init and forward exit status to the - # parent process. The supervisor process runs in - # the global pid namespace, so skip /proc remount - # and other setup that's intended only for the - # init process. - binary, myargs = portage._python_interpreter, [ - portage._python_interpreter, - os.path.join(portage._bin_path, "pid-ns-init"), - str(main_child_pid), - ] - - os.execve(binary, myargs, env) + have_unshare = hasattr(libc, "unshare") + + if not have_unshare: + # unshare() may not be supported by libc + unshare_net = False + unshare_ipc = False + unshare_mount = False + unshare_pid = False + + if unshare_net or unshare_ipc or unshare_mount or unshare_pid: + # Since a failed unshare call could corrupt process + # state, first validate that the call can succeed. + # The parent process should call _unshare_validate + # before it forks, so that all child processes can + # reuse _unshare_validate results that have been + # cached by the parent process. + errno_value = _unshare_validate(unshare_flags) + if errno_value == 0 and libc.unshare(unshare_flags) != 0: + errno_value = ctypes.get_errno() + if errno_value != 0: + involved_features = [] + if unshare_ipc: + involved_features.append("ipc-sandbox") + if unshare_mount: + involved_features.append("mount-sandbox") + if unshare_net: + involved_features.append("network-sandbox") + if unshare_pid: + involved_features.append("pid-sandbox") + + writemsg( + 'Unable to unshare: %s (for FEATURES="%s")\n' + % ( + errno.errorcode.get(errno_value, "?"), + " ".join(involved_features), + ), + noiselevel=-1, + ) + + unshare_net = False + unshare_ipc = False + unshare_mount = False + unshare_pid = False + + if unshare_pid: + # pid namespace requires us to become init + binary, myargs = ( + portage._python_interpreter, + [ + portage._python_interpreter, + os.path.join(portage._bin_path, "pid-ns-init"), + _unicode_encode("" if uid is None else str(uid)), + _unicode_encode("" if gid is None else str(gid)), + _unicode_encode( + "" if groups is None else ",".join(str(group) for group in groups) + ), + _unicode_encode("" if umask is None else str(umask)), + _unicode_encode(",".join(str(fd) for fd in fd_pipes)), + binary, + ] + + myargs, + ) + uid = None + gid = None + groups = None + umask = None + + # Use _start_fork for os.fork() error handling, ensuring + # that if exec fails then the child process will display + # a traceback before it exits via os._exit to suppress any + # finally blocks from parent's call stack (bug 345289). + main_child_pid = _start_fork( + _exec2, + args=( + binary, + myargs, + env, + gid, + groups, + uid, + umask, + cwd, + pre_exec, + unshare_net, + unshare_ipc, + unshare_mount, + unshare_pid, + libc, + ), + fd_pipes=None, + close_fds=False, + ) + + # Execute a supervisor process which will forward + # signals to init and forward exit status to the + # parent process. The supervisor process runs in + # the global pid namespace, so skip /proc remount + # and other setup that's intended only for the + # init process. + binary, myargs = portage._python_interpreter, [ + portage._python_interpreter, + os.path.join(portage._bin_path, "pid-ns-init"), + str(main_child_pid), + ] + + os.execve(binary, myargs, env) # Reachable only if unshare_pid is False. _exec2( @@ -1080,6 +1085,7 @@ def _exec( unshare_ipc, unshare_mount, unshare_pid, + libc, ) @@ -1097,6 +1103,7 @@ def _exec2( unshare_ipc, unshare_mount, unshare_pid, + libc, ): if unshare_mount: # mark the whole filesystem as slave to avoid -- cgit v1.2.3-65-gdbad From 9e84ef57ba747766c9147c1ac1b247faa1f05956 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sat, 24 Feb 2024 16:31:13 -0800 Subject: testSpawnReturnProcTerminate: Fix integer in spawn command argument The invalid integer in the spawn command argument intermittently triggered this error when SIGTERM did not arrive until after the exec call: ----------------------------- Captured stderr call ----------------------------- Process ForkProcess-23: Traceback (most recent call last): File "/home/runner/work/portage/portage/lib/portage/process.py", line 818, in _exec_wrapper _exec( File "/home/runner/work/portage/portage/lib/portage/process.py", line 1068, in _exec _exec2( File "/home/runner/work/portage/portage/lib/portage/process.py", line 1166, in _exec2 os.execve(binary, myargs, env) TypeError: expected str, bytes or os.PathLike object, not int During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap self.run() File "/opt/hostedtoolcache/Python/3.12.2/x64/lib/python3.12/multiprocessing/process.py", line 108, in run self._target(*self._args, **self._kwargs) File "/home/runner/work/portage/portage/lib/portage/util/_async/ForkProcess.py", line 328, in _bootstrap sys.exit(target(*(args or []), **(kwargs or {}))) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/runner/work/portage/portage/lib/portage/process.py", line 1441, in __call__ return self._target(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/runner/work/portage/portage/lib/portage/process.py", line 853, in _exec_wrapper writemsg(f"{e}:\n {' '.join(mycommand)}\n", noiselevel=-1) ^^^^^^^^^^^^^^^^^^^ TypeError: sequence item 1: expected str instance, int found Bug: https://bugs.gentoo.org/916566#c20 Signed-off-by: Zac Medico --- lib/portage/tests/process/test_spawn_returnproc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/tests/process/test_spawn_returnproc.py b/lib/portage/tests/process/test_spawn_returnproc.py index 6d823d9c3..8fbf54d0d 100644 --- a/lib/portage/tests/process/test_spawn_returnproc.py +++ b/lib/portage/tests/process/test_spawn_returnproc.py @@ -32,7 +32,7 @@ class SpawnReturnProcTestCase(TestCase): loop = global_event_loop() async def watch_pid(): - proc = spawn([sleep_binary, 9999], returnproc=True) + proc = spawn([sleep_binary, "9999"], returnproc=True) proc.terminate() self.assertEqual(await proc.wait(), -signal.SIGTERM) -- cgit v1.2.3-65-gdbad From 9a06b7210562b8d03577cd4043227ea2023db1f2 Mon Sep 17 00:00:00 2001 From: Sam James Date: Sun, 25 Feb 2024 08:13:28 +0000 Subject: bin/install-qa-check.d: 90gcc-warnings: drop GCC warnings with known (heavy) FPs Drop -Wstringop-overflow, -Wstringop-overread, -Wstringop-truncation, -Waddress, and -Wreturn-local-addr for now because they cause too many FPs. We want only reliable warnings for this Portage QA check as we encourage people to report these upstream. Bug: https://gcc.gnu.org/PR88443 Bug: https://gcc.gnu.org/PR88781 Bug: https://gcc.gnu.org/PR93644 Bug: https://gcc.gnu.org/PR97048 Bug: https://gcc.gnu.org/PR103360 Bug: https://bugs.gentoo.org/925460 Signed-off-by: Sam James --- NEWS | 2 ++ bin/install-qa-check.d/90gcc-warnings | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/NEWS b/NEWS index 94be26de8..83be37204 100644 --- a/NEWS +++ b/NEWS @@ -12,6 +12,8 @@ portage-3.0.63 (UNRELEASED) Bug fixes: * emerge: Skip installed packages with emptytree in depgraph selection (bug #651018). +* bin/install-qa-check.d: 90gcc-warnings: drop warnings with too many FPs (bug #925460). + portage-3.0.62 (2024-02-22) -------------- diff --git a/bin/install-qa-check.d/90gcc-warnings b/bin/install-qa-check.d/90gcc-warnings index 449bf2362..1060618df 100644 --- a/bin/install-qa-check.d/90gcc-warnings +++ b/bin/install-qa-check.d/90gcc-warnings @@ -44,9 +44,12 @@ gcc_warn_check() { # those three do not have matching -W flags, it seems 'warning: .*will always overflow destination buffer' # compile-time part of FORTIFY_SOURCE - 'warning: .*\[-Wstringop-overflow\]' - 'warning: .*\[-Wstringop-overread\]' - 'warning: .*\[-Wstringop-truncation\]' + # XXX: Commented out because of gcc FPs (https://gcc.gnu.org/PR88443) + #'warning: .*\[-Wstringop-overflow\]' + # XXX: Commented out because of gcc FPs (https://gcc.gnu.org/PR97048) + #'warning: .*\[-Wstringop-overread\]' + # XXX: Commented out because of gcc FPs (https://gcc.gnu.org/PR88781) + #'warning: .*\[-Wstringop-truncation\]' # clang-only, equivalent of -Wstringop-overflow 'warning: .*\[-Wfortify-source\]' 'warning: .*assuming pointer wraparound does not occur' @@ -68,7 +71,8 @@ gcc_warn_check() { 'warning: .*function.*\[-Wpointer-arith\]' # the address of ... will never be NULL and likes # (uses of function refs & string constants in conditionals) - 'warning: .*\[-Waddress\]' + # XXX: Commented out because of gcc FPs (https://gcc.gnu.org/PR103360) + #'warning: .*\[-Waddress\]' # TODO: we want to enable these but bash currently triggers # them with a trick in random.c where it intentionally wants @@ -95,7 +99,8 @@ gcc_warn_check() { 'warning: .*matching.*\[-Wformat=\]' # function returns address of local variable - 'warning: .*\[-Wreturn-local-addr\]' + # XXX: Commented out for bug #925460 (https://gcc.gnu.org/PR93644) + #'warning: .*\[-Wreturn-local-addr\]' # missing return at end of function, or non-void return in a void function # (clang at least aggressively optimises on this) 'warning: .*\[-Wreturn-type\]' @@ -123,7 +128,8 @@ gcc_warn_check() { 'warning: .*\[-Wodr\]' # warning: argument value A will result in undefined behaviour (Clang) 'warning: .*\[-Wargument-undefined-behaviour\]' - 'warning: .*\[-Wnull-dereference\]' + # XXX: Commented out because of GCC FPs (https://gcc.gnu.org/PR86172) + #'warning: .*\[-Wnull-dereference\]' # general sensible warnings (will be rejected by modern compilers soon) 'warning: .*\[-Wmain\]' -- cgit v1.2.3-65-gdbad From 0855821572f32e81b031a250f7491f34a2fd4078 Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sat, 24 Feb 2024 15:29:29 -0800 Subject: dbapi: Fix TypeError when passing Exception to warnings.warn If an Exception is passed as a message to warnings.warn then it triggers this TypeError: if metadata_updates: try: aux_update(cpv, metadata_updates) except (InvalidBinaryPackageFormat, CorruptionKeyError) as e: > warnings.warn(e) E TypeError: expected string or bytes-like object, got 'CorruptionKeyError' Fixes: fb1d0a22f657 ("dbapi: KeyError tolerance during package moves") Bug: https://bugs.gentoo.org/922935 Signed-off-by: Zac Medico Closes: https://github.com/gentoo/portage/pull/1282 Signed-off-by: Sam James --- lib/portage/dbapi/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/portage/dbapi/__init__.py b/lib/portage/dbapi/__init__.py index 6f95b93a2..9105227c7 100644 --- a/lib/portage/dbapi/__init__.py +++ b/lib/portage/dbapi/__init__.py @@ -1,11 +1,12 @@ -# Copyright 1998-2023 Gentoo Authors +# Copyright 1998-2024 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 __all__ = ["dbapi"] import functools +import logging import re -import warnings +import sys from typing import Any, Dict, List, Optional, Tuple from collections.abc import Sequence @@ -429,7 +430,9 @@ class dbapi: try: aux_update(cpv, metadata_updates) except (InvalidBinaryPackageFormat, CorruptionKeyError) as e: - warnings.warn(e) + logging.warning( + f"{e.__class__.__name__}: {e}", exc_info=sys.exc_info() + ) if onUpdate: onUpdate(maxval, i + 1) if onProgress: @@ -477,6 +480,6 @@ class dbapi: try: self.aux_update(mycpv, mydata) except CorruptionKeyError as e: - warnings.warn(e) + logging.warning(f"{e.__class__.__name__}: {e}", exc_info=sys.exc_info()) continue return moves -- cgit v1.2.3-65-gdbad From 92615cd37d7a1efce923c474e455f59fe61a589c Mon Sep 17 00:00:00 2001 From: Zac Medico Date: Sat, 24 Feb 2024 21:09:48 -0800 Subject: _start_proc: Prevent premature ForkProcess garbage collection In order to prevent the constructed ForkProcess instance from being prematurely garbage collected, return a reference to the caller inside of a _GCProtector object, preventing messages reported in bug 925456 like this: [ERROR] Task was destroyed but it is pending! Fixes: a69c1b853a47 ("process.spawn: Use multiprocessing.Process for returnproc") Bug: https://bugs.gentoo.org/925456 Signed-off-by: Zac Medico Closes: https://github.com/gentoo/portage/pull/1284 Signed-off-by: Sam James --- lib/portage/process.py | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/lib/portage/process.py b/lib/portage/process.py index d16262e75..cc9ed7bf7 100644 --- a/lib/portage/process.py +++ b/lib/portage/process.py @@ -36,6 +36,7 @@ portage.proxy.lazyimport.lazyimport( from portage.const import BASH_BINARY, SANDBOX_BINARY, FAKEROOT_BINARY from portage.exception import CommandNotFound +from portage.proxy.objectproxy import ObjectProxy from portage.util._ctypes import find_library, LoadLibrary, ctypes try: @@ -1493,8 +1494,44 @@ def _start_proc( proc.start() # ForkProcess conveniently holds a MultiprocessingProcess - # instance that is suitable to return here. - return proc._proc + # instance that is suitable to return here, but use _GCProtector + # to protect the ForkProcess instance from being garbage collected + # and triggering messages like this (bug 925456): + # [ERROR] Task was destroyed but it is pending! + return _GCProtector(proc._proc, proc.async_wait) + + +class _GCProtector(ObjectProxy): + """ + Proxy a target object, and also hold a reference to something + extra in order to protect it from garbage collection. Override + the wait method to first call target's wait method and then + wait for extra (a coroutine function) before returning the result. + """ + + __slots__ = ("_extra", "_target") + + def __init__(self, target, extra): + super().__init__() + object.__setattr__(self, "_target", target) + object.__setattr__(self, "_extra", extra) + + def _get_target(self): + return object.__getattribute__(self, "_target") + + def __getattribute__(self, attr): + if attr == "wait": + return object.__getattribute__(self, attr) + return getattr(object.__getattribute__(self, "_target"), attr) + + async def wait(self): + """ + Wrap the target's wait method to also wait for an extra + coroutine function. + """ + result = await object.__getattribute__(self, "_target").wait() + await object.__getattribute__(self, "_extra")() + return result def find_binary(binary): -- cgit v1.2.3-65-gdbad From 8a2f1d14788d107ec54dc53c9ef1cf00ee310d51 Mon Sep 17 00:00:00 2001 From: Gábor Oszkár Dénes Date: Sat, 24 Feb 2024 21:48:05 +0100 Subject: test_baseline: Improve robustness with cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The baseline tests need to cleanup the ResolverPlayground after each testcase, with each different parametrization. This is ensured by making the scope of the playground fixture the function instead of the module. With module the cleanup only happens before/after the switch from/to xpak and gpkg. Signed-off-by: Gábor Oszkár Dénes Closes: https://github.com/gentoo/portage/pull/1281 Signed-off-by: Sam James --- lib/portage/tests/emerge/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/portage/tests/emerge/conftest.py b/lib/portage/tests/emerge/conftest.py index 580d1e09a..356e09879 100644 --- a/lib/portage/tests/emerge/conftest.py +++ b/lib/portage/tests/emerge/conftest.py @@ -406,7 +406,7 @@ def async_loop(): yield asyncio._wrap_loop() -@pytest.fixture(params=SUPPORTED_GENTOO_BINPKG_FORMATS, scope="module") +@pytest.fixture(params=SUPPORTED_GENTOO_BINPKG_FORMATS, scope="function") def playground(request, tmp_path_factory): """Fixture that provides instances of ``ResolverPlayground`` each one with one supported value for ``BINPKG_FORMAT``.""" -- cgit v1.2.3-65-gdbad From 8832264b4e5bff768346ba5a6d2f828dcf468c60 Mon Sep 17 00:00:00 2001 From: Sam James Date: Sun, 25 Feb 2024 08:29:05 +0000 Subject: NEWS, meson.build: prepare for portage-3.0.63 Signed-off-by: Sam James --- NEWS | 30 +++++++++++++++++++++++++++++- meson.build | 2 +- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 83be37204..eb84651b5 100644 --- a/NEWS +++ b/NEWS @@ -6,7 +6,7 @@ Release notes take the form of the following optional categories: * Bug fixes * Cleanups -portage-3.0.63 (UNRELEASED) +portage-3.0.63 (2024-02-25) -------------- Bug fixes: @@ -14,6 +14,34 @@ Bug fixes: * bin/install-qa-check.d: 90gcc-warnings: drop warnings with too many FPs (bug #925460). +* AsyncioEventLoop: Call process.run_exitfuncs() before close (bug #925240). + + Fixes hang with FEATURES="network-sandbox-proxy" or FEATURES="distcc". + +* SchedulerInterface/PollScheduler: Add _loop property (bug #925333). + + Fixes erroneous assert. + +* _dynamic_deps_preload: Fix settings reference (bug #925350). + + Fix KeyError with --dynamic-deps. + +* process.spawn: Fix logic for missing libc.unshare on musl (bug #925311). + + Fix handling when we can't unshare where we might e.g. modify the hostname + of the real system on musl. + +* dbapi: Fix TypeError when passing Exception to warnings.warn (bug #922935). + + Fix testUpdateDbentryDbapiTestCase. + +* _start_proc: Prevent premature ForkProcess garbage collection (bug #925456). + + Fix warnings/errors like "[ERROR] Task was destroyed but it is pending!". + +Cleanups: +* More asyncio porting. + portage-3.0.62 (2024-02-22) -------------- diff --git a/meson.build b/meson.build index aeb3f93e6..e826f6ffa 100644 --- a/meson.build +++ b/meson.build @@ -1,7 +1,7 @@ project( 'portage', 'c', - version : '3.0.62', + version : '3.0.63', license : 'GPL-2.0-or-later', meson_version : '>=0.58.0' ) -- cgit v1.2.3-65-gdbad