diff options
| author |   Matti Picus <matti.picus@gmail.com> | 2020-09-11 16:28:53 +0300 |
|---|---|---|
| committer | Matti Picus <matti.picus@gmail.com> | 2020-09-11 16:28:53 +0300 |
| commit | 7ce220ab3c3941420c8110668736d07b36b73449 (patch) | |
| tree | e6c0250a1b9b81d4eb54328bcce5f68a7a51c0d6 | |
| parent | Import CPython stdlib at tag v3.7.4 (diff) | |
| download | pypy-7ce220ab3c3941420c8110668736d07b36b73449.tar.gz pypy-7ce220ab3c3941420c8110668736d07b36b73449.tar.bz2 pypy-7ce220ab3c3941420c8110668736d07b36b73449.zip | |
update to stdlib 3.7.9
315 files changed, 11203 insertions, 4008 deletions
diff --git a/lib-python/3/_osx_support.py b/lib-python/3/_osx_support.py index db6674ea29..e9efce7d7e 100644 --- a/lib-python/3/_osx_support.py +++ b/lib-python/3/_osx_support.py @@ -211,7 +211,7 @@ def _remove_universal_flags(_config_vars): if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] flags = re.sub(r'-arch\s+\w+\s', ' ', flags, flags=re.ASCII) - flags = re.sub('-isysroot [^ \t]*', ' ', flags) + flags = re.sub(r'-isysroot\s*\S+', ' ', flags) _save_modified_value(_config_vars, cv, flags) return _config_vars @@ -287,7 +287,7 @@ def _check_for_unavailable_sdk(_config_vars): # to /usr and /System/Library by either a standalone CLT # package or the CLT component within Xcode. cflags = _config_vars.get('CFLAGS', '') - m = re.search(r'-isysroot\s+(\S+)', cflags) + m = re.search(r'-isysroot\s*(\S+)', cflags) if m is not None: sdk = m.group(1) if not os.path.exists(sdk): @@ -295,7 +295,7 @@ def _check_for_unavailable_sdk(_config_vars): # Do not alter a config var explicitly overridden by env var if cv in _config_vars and cv not in os.environ: flags = _config_vars[cv] - flags = re.sub(r'-isysroot\s+\S+(?:\s|$)', ' ', flags) + flags = re.sub(r'-isysroot\s*\S+(?:\s|$)', ' ', flags) _save_modified_value(_config_vars, cv, flags) return _config_vars @@ -320,7 +320,7 @@ def compiler_fixup(compiler_so, cc_args): stripArch = stripSysroot = True else: stripArch = '-arch' in cc_args - stripSysroot = '-isysroot' in cc_args + stripSysroot = any(arg for arg in cc_args if arg.startswith('-isysroot')) if stripArch or 'ARCHFLAGS' in os.environ: while True: @@ -338,23 +338,34 @@ def compiler_fixup(compiler_so, cc_args): if stripSysroot: while True: - try: - index = compiler_so.index('-isysroot') + indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')] + if not indices: + break + index = indices[0] + if compiler_so[index] == '-isysroot': # Strip this argument and the next one: del compiler_so[index:index+2] - except ValueError: - break + else: + # It's '-isysroot/some/path' in one arg + del compiler_so[index:index+1] # Check if the SDK that is used during compilation actually exists, # the universal build requires the usage of a universal SDK and not all # users have that installed by default. sysroot = None - if '-isysroot' in cc_args: - idx = cc_args.index('-isysroot') - sysroot = cc_args[idx+1] - elif '-isysroot' in compiler_so: - idx = compiler_so.index('-isysroot') - sysroot = compiler_so[idx+1] + argvar = cc_args + indices = [i for i,x in enumerate(cc_args) if x.startswith('-isysroot')] + if not indices: + argvar = compiler_so + indices = [i for i,x in enumerate(compiler_so) if x.startswith('-isysroot')] + + for idx in indices: + if argvar[idx] == '-isysroot': + sysroot = argvar[idx+1] + break + else: + sysroot = argvar[idx][len('-isysroot'):] + break if sysroot and not os.path.isdir(sysroot): from distutils import log diff --git a/lib-python/3/_pydecimal.py b/lib-python/3/_pydecimal.py index 44ea5b41b2..27ff963303 100644 --- a/lib-python/3/_pydecimal.py +++ b/lib-python/3/_pydecimal.py @@ -140,8 +140,11 @@ __all__ = [ # Limits for the C version for compatibility 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', - # C version: compile time choice that enables the thread local context - 'HAVE_THREADS' + # C version: compile time choice that enables the thread local context (deprecated, now always true) + 'HAVE_THREADS', + + # C version: compile time choice that enables the coroutine local context + 'HAVE_CONTEXTVAR' ] __xname__ = __name__ # sys.modules lookup (--without-threads) @@ -172,6 +175,7 @@ ROUND_05UP = 'ROUND_05UP' # Compatibility with the C version HAVE_THREADS = True +HAVE_CONTEXTVAR = True if sys.maxsize == 2**63-1: MAX_PREC = 999999999999999999 MAX_EMAX = 999999999999999999 diff --git a/lib-python/3/_pyio.py b/lib-python/3/_pyio.py index e81cc51288..d219781345 100644 --- a/lib-python/3/_pyio.py +++ b/lib-python/3/_pyio.py @@ -1543,7 +1543,11 @@ class FileIO(RawIOBase): # For consistent behaviour, we explicitly seek to the # end of file (otherwise, it might be done only on the # first write()). - os.lseek(fd, 0, SEEK_END) + try: + os.lseek(fd, 0, SEEK_END) + except OSError as e: + if e.errno != errno.ESPIPE: + raise except: if owned_fd is not None: os.close(owned_fd) diff --git a/lib-python/3/argparse.py b/lib-python/3/argparse.py index a030749247..ac424f4914 100644 --- a/lib-python/3/argparse.py +++ b/lib-python/3/argparse.py @@ -407,13 +407,19 @@ class HelpFormatter(object): inserts[start] += ' [' else: inserts[start] = '[' - inserts[end] = ']' + if end in inserts: + inserts[end] += ']' + else: + inserts[end] = ']' else: if start in inserts: inserts[start] += ' (' else: inserts[start] = '(' - inserts[end] = ')' + if end in inserts: + inserts[end] += ')' + else: + inserts[end] = ')' for i in range(start + 1, end): inserts[i] = '|' @@ -2074,10 +2080,11 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer): OPTIONAL: _('expected at most one argument'), ONE_OR_MORE: _('expected at least one argument'), } - default = ngettext('expected %s argument', + msg = nargs_errors.get(action.nargs) + if msg is None: + msg = ngettext('expected %s argument', 'expected %s arguments', action.nargs) % action.nargs - msg = nargs_errors.get(action.nargs, default) raise ArgumentError(action, msg) # return the number of arguments matched diff --git a/lib-python/3/ast.py b/lib-python/3/ast.py index bfe346bba8..818d563823 100644 --- a/lib-python/3/ast.py +++ b/lib-python/3/ast.py @@ -93,26 +93,35 @@ def literal_eval(node_or_string): def dump(node, annotate_fields=True, include_attributes=False): """ - Return a formatted dump of the tree in *node*. This is mainly useful for - debugging purposes. The returned string will show the names and the values - for fields. This makes the code impossible to evaluate, so if evaluation is - wanted *annotate_fields* must be set to False. Attributes such as line + Return a formatted dump of the tree in node. This is mainly useful for + debugging purposes. If annotate_fields is true (by default), + the returned string will show the names and the values for fields. + If annotate_fields is false, the result string will be more compact by + omitting unambiguous field names. Attributes such as line numbers and column offsets are not dumped by default. If this is wanted, - *include_attributes* can be set to True. + include_attributes can be set to true. """ def _format(node): if isinstance(node, AST): - fields = [(a, _format(b)) for a, b in iter_fields(node)] - rv = '%s(%s' % (node.__class__.__name__, ', '.join( - ('%s=%s' % field for field in fields) - if annotate_fields else - (b for a, b in fields) - )) + args = [] + keywords = annotate_fields + for field in node._fields: + try: + value = getattr(node, field) + except AttributeError: + keywords = True + else: + if keywords: + args.append('%s=%s' % (field, _format(value))) + else: + args.append(_format(value)) if include_attributes and node._attributes: - rv += fields and ', ' or ' ' - rv += ', '.join('%s=%s' % (a, _format(getattr(node, a))) - for a in node._attributes) - return rv + ')' + for a in node._attributes: + try: + args.append('%s=%s' % (a, _format(getattr(node, a)))) + except AttributeError: + pass + return '%s(%s)' % (node.__class__.__name__, ', '.join(args)) elif isinstance(node, list): return '[%s]' % ', '.join(_format(x) for x in node) return repr(node) @@ -289,11 +298,11 @@ class NodeTransformer(NodeVisitor): class RewriteName(NodeTransformer): def visit_Name(self, node): - return copy_location(Subscript( + return Subscript( value=Name(id='data', ctx=Load()), slice=Index(value=Str(s=node.id)), ctx=node.ctx - ), node) + ) Keep in mind that if the node you're operating on has child nodes you must either transform the child nodes yourself or call the :meth:`generic_visit` diff --git a/lib-python/3/asyncio/base_events.py b/lib-python/3/asyncio/base_events.py index 52134372fa..f723356344 100644 --- a/lib-python/3/asyncio/base_events.py +++ b/lib-python/3/asyncio/base_events.py @@ -61,6 +61,10 @@ _HAS_IPv6 = hasattr(socket, 'AF_INET6') # Maximum timeout passed to select to avoid OS limitations MAXIMUM_SELECT_TIMEOUT = 24 * 3600 +# Used for deprecation and removal of `loop.create_datagram_endpoint()`'s +# *reuse_address* parameter +_unset = object() + def _format_handle(handle): cb = handle._callback @@ -514,14 +518,17 @@ class BaseEventLoop(events.AbstractEventLoop): 'asyncgen': agen }) - def run_forever(self): - """Run until stop() is called.""" - self._check_closed() + def _check_runnung(self): if self.is_running(): raise RuntimeError('This event loop is already running') if events._get_running_loop() is not None: raise RuntimeError( 'Cannot run the event loop while another loop is running') + + def run_forever(self): + """Run until stop() is called.""" + self._check_closed() + self._check_runnung() self._set_coroutine_origin_tracking(self._debug) self._thread_id = threading.get_ident() @@ -553,6 +560,7 @@ class BaseEventLoop(events.AbstractEventLoop): Return the Future's result, or raise its exception. """ self._check_closed() + self._check_runnung() new_task = not futures.isfuture(future) future = tasks.ensure_future(future, loop=self) @@ -1138,7 +1146,7 @@ class BaseEventLoop(events.AbstractEventLoop): async def create_datagram_endpoint(self, protocol_factory, local_addr=None, remote_addr=None, *, family=0, proto=0, flags=0, - reuse_address=None, reuse_port=None, + reuse_address=_unset, reuse_port=None, allow_broadcast=None, sock=None): """Create datagram connection.""" if sock is not None: @@ -1147,7 +1155,7 @@ class BaseEventLoop(events.AbstractEventLoop): f'A UDP Socket was expected, got {sock!r}') if (local_addr or remote_addr or family or proto or flags or - reuse_address or reuse_port or allow_broadcast): + reuse_port or allow_broadcast): # show the problematic kwargs in exception msg opts = dict(local_addr=local_addr, remote_addr=remote_addr, family=family, proto=proto, flags=flags, @@ -1201,8 +1209,18 @@ class BaseEventLoop(events.AbstractEventLoop): exceptions = [] - if reuse_address is None: - reuse_address = os.name == 'posix' and sys.platform != 'cygwin' + # bpo-37228 + if reuse_address is not _unset: + if reuse_address: + raise ValueError("Passing `reuse_address=True` is no " + "longer supported, as the usage of " + "SO_REUSEPORT in UDP poses a significant " + "security concern.") + else: + warnings.warn("The *reuse_address* parameter has been " + "deprecated as of 3.7.6 and is scheduled " + "for removal in 3.11.", DeprecationWarning, + stacklevel=2) for ((family, proto), (local_address, remote_address)) in addr_pairs_info: @@ -1211,9 +1229,6 @@ class BaseEventLoop(events.AbstractEventLoop): try: sock = socket.socket( family=family, type=socket.SOCK_DGRAM, proto=proto) - if reuse_address: - sock.setsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) if reuse_port: _set_reuseport(sock) if allow_broadcast: diff --git a/lib-python/3/asyncio/events.py b/lib-python/3/asyncio/events.py index e4e632206a..e8a6a1beea 100644 --- a/lib-python/3/asyncio/events.py +++ b/lib-python/3/asyncio/events.py @@ -630,9 +630,9 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy): self._local = self._Local() def get_event_loop(self): - """Get the event loop. + """Get the event loop for the current context. - This may be None or an instance of EventLoop. + Returns an instance of EventLoop or raises an exception. """ if (self._local._loop is None and not self._local._set_called and diff --git a/lib-python/3/asyncio/futures.py b/lib-python/3/asyncio/futures.py index 0e0e696a25..1bd6c81406 100644 --- a/lib-python/3/asyncio/futures.py +++ b/lib-python/3/asyncio/futures.py @@ -118,7 +118,10 @@ class Future: def get_loop(self): """Return the event loop the Future is bound to.""" - return self._loop + loop = self._loop + if loop is None: + raise RuntimeError("Future object is not initialized.") + return loop def cancel(self): """Cancel the future and schedule callbacks. diff --git a/lib-python/3/asyncio/runners.py b/lib-python/3/asyncio/runners.py index 5fbab03dd0..2e37e18b45 100644 --- a/lib-python/3/asyncio/runners.py +++ b/lib-python/3/asyncio/runners.py @@ -6,7 +6,7 @@ from . import tasks def run(main, *, debug=False): - """Run a coroutine. + """Execute the coroutine and return the result. This function runs the passed coroutine, taking care of managing the asyncio event loop and finalizing asynchronous diff --git a/lib-python/3/asyncio/selector_events.py b/lib-python/3/asyncio/selector_events.py index 23bd8ad849..fa8f0cd759 100644 --- a/lib-python/3/asyncio/selector_events.py +++ b/lib-python/3/asyncio/selector_events.py @@ -39,6 +39,11 @@ def _test_selector_event(selector, fd, event): return bool(key.events & event) +def _check_ssl_socket(sock): + if ssl is not None and isinstance(sock, ssl.SSLSocket): + raise TypeError("Socket cannot be of type SSLSocket") + + class BaseSelectorEventLoop(base_events.BaseEventLoop): """Selector event loop. @@ -345,6 +350,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): The maximum amount of data to be received at once is specified by nbytes. """ + _check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() @@ -378,6 +384,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): The received data is written into *buf* (a writable buffer). The return value is the number of bytes written. """ + _check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() @@ -415,6 +422,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): raised, and there is no way to determine how much data, if any, was successfully processed by the receiving end of the connection. """ + _check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() @@ -451,6 +459,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): This method is a coroutine. """ + _check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") @@ -508,6 +517,7 @@ class BaseSelectorEventLoop(base_events.BaseEventLoop): object usable to send and receive data on the connection, and address is the address bound to the socket on the other end of the connection. """ + _check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() diff --git a/lib-python/3/asyncio/unix_events.py b/lib-python/3/asyncio/unix_events.py index a0fc996d23..e037e12965 100644 --- a/lib-python/3/asyncio/unix_events.py +++ b/lib-python/3/asyncio/unix_events.py @@ -98,7 +98,7 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): try: # Register a dummy signal handler to ask Python to write the signal - # number in the wakup file descriptor. _process_self_data() will + # number in the wakeup file descriptor. _process_self_data() will # read signal numbers from this file descriptor to handle signals. signal.signal(sig, _sighandler_noop) @@ -431,6 +431,7 @@ class _UnixReadPipeTransport(transports.ReadTransport): self._fileno = pipe.fileno() self._protocol = protocol self._closing = False + self._paused = False mode = os.fstat(self._fileno).st_mode if not (stat.S_ISFIFO(mode) or @@ -492,10 +493,20 @@ class _UnixReadPipeTransport(transports.ReadTransport): self._loop.call_soon(self._call_connection_lost, None) def pause_reading(self): + if self._closing or self._paused: + return + self._paused = True self._loop._remove_reader(self._fileno) + if self._loop.get_debug(): + logger.debug("%r pauses reading", self) def resume_reading(self): + if self._closing or not self._paused: + return + self._paused = False self._loop._add_reader(self._fileno, self._read_ready) + if self._loop.get_debug(): + logger.debug("%r resumes reading", self) def set_protocol(self, protocol): self._protocol = protocol diff --git a/lib-python/3/base64.py b/lib-python/3/base64.py index 2be9c395a9..2e70223dfe 100755 --- a/lib-python/3/base64.py +++ b/lib-python/3/base64.py @@ -82,7 +82,7 @@ def b64decode(s, altchars=None, validate=False): altchars = _bytes_from_decode_data(altchars) assert len(altchars) == 2, repr(altchars) s = s.translate(bytes.maketrans(altchars, b'+/')) - if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s): + if validate and not re.fullmatch(b'[A-Za-z0-9+/]*={0,2}', s): raise binascii.Error('Non-base64 digit found') return binascii.a2b_base64(s) diff --git a/lib-python/3/bdb.py b/lib-python/3/bdb.py index caf207733b..fcdc5731f8 100644 --- a/lib-python/3/bdb.py +++ b/lib-python/3/bdb.py @@ -546,14 +546,7 @@ class Bdb: s += frame.f_code.co_name else: s += "<lambda>" - if '__args__' in frame.f_locals: - args = frame.f_locals['__args__'] - else: - args = None - if args: - s += reprlib.repr(args) - else: - s += '()' + s += '()' if '__return__' in frame.f_locals: rv = frame.f_locals['__return__'] s += '->' diff --git a/lib-python/3/cgi.py b/lib-python/3/cgi.py index 8cf668718d..5a001667ef 100755 --- a/lib-python/3/cgi.py +++ b/lib-python/3/cgi.py @@ -217,7 +217,10 @@ def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"): ctype = "multipart/form-data; boundary={}".format(boundary) headers = Message() headers.set_type(ctype) - headers['Content-Length'] = pdict['CONTENT-LENGTH'] + try: + headers['Content-Length'] = pdict['CONTENT-LENGTH'] + except KeyError: + pass fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, environ={'REQUEST_METHOD': 'POST'}) return {k: fs.getlist(k) for k in fs} @@ -478,7 +481,7 @@ class FieldStorage: if maxlen and clen > maxlen: raise ValueError('Maximum content length exceeded') self.length = clen - if self.limit is None and clen: + if self.limit is None and clen >= 0: self.limit = clen self.list = self.file = None @@ -659,8 +662,10 @@ class FieldStorage: if 'content-length' in headers: del headers['content-length'] + limit = None if self.limit is None \ + else self.limit - self.bytes_read part = klass(self.fp, headers, ib, environ, keep_blank_values, - strict_parsing,self.limit-self.bytes_read, + strict_parsing, limit, self.encoding, self.errors, max_num_fields) if max_num_fields is not None: @@ -751,7 +756,8 @@ class FieldStorage: last_line_lfend = True _read = 0 while 1: - if _read >= self.limit: + + if self.limit is not None and 0 <= self.limit <= _read: break line = self.fp.readline(1<<16) # bytes self.bytes_read += len(line) diff --git a/lib-python/3/code.py b/lib-python/3/code.py index d8106ae612..76000f8c8b 100644 --- a/lib-python/3/code.py +++ b/lib-python/3/code.py @@ -40,7 +40,7 @@ class InteractiveInterpreter: Arguments are as for compile_command(). - One several things can happen: + One of several things can happen: 1) The input is incorrect; compile_command() raised an exception (SyntaxError or OverflowError). A syntax traceback diff --git a/lib-python/3/codecs.py b/lib-python/3/codecs.py index cfca5d38b0..5d95f25612 100644 --- a/lib-python/3/codecs.py +++ b/lib-python/3/codecs.py @@ -904,11 +904,16 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=1): file = builtins.open(filename, mode, buffering) if encoding is None: return file - info = lookup(encoding) - srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors) - # Add attributes to simplify introspection - srw.encoding = encoding - return srw + + try: + info = lookup(encoding) + srw = StreamReaderWriter(file, info.streamreader, info.streamwriter, errors) + # Add attributes to simplify introspection + srw.encoding = encoding + return srw + except: + file.close() + raise def EncodedFile(file, data_encoding, file_encoding=None, errors='strict'): diff --git a/lib-python/3/codeop.py b/lib-python/3/codeop.py index fb759da42a..3c2bb60835 100644 --- a/lib-python/3/codeop.py +++ b/lib-python/3/codeop.py @@ -57,6 +57,7 @@ Compile(): """ import __future__ +import warnings _features = [getattr(__future__, fname) for fname in __future__.all_feature_names] @@ -83,20 +84,26 @@ def _maybe_compile(compiler, source, filename, symbol): except SyntaxError as err: pass - try: - code1 = compiler(source + "\n", filename, symbol) - except SyntaxError as e: - err1 = e + # Suppress warnings after the first compile to avoid duplication. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + code1 = compiler(source + "\n", filename, symbol) + except SyntaxError as e: + err1 = e - try: - code2 = compiler(source + "\n\n", filename, symbol) - except SyntaxError as e: - err2 = e + try: + code2 = compiler(source + "\n\n", filename, symbol) + except SyntaxError as e: + err2 = e - if code: - return code - if not code1 and repr(err1) == repr(err2): - raise err1 + try: + if code: + return code + if not code1 and repr(err1) == repr(err2): + raise err1 + finally: + err1 = err2 = None def _compile(source, filename, symbol): return compile(source, filename, symbol, PyCF_DONT_IMPLY_DEDENT) @@ -109,7 +116,8 @@ def compile_command(source, filename="<input>", symbol="single"): source -- the source string; may contain \n characters filename -- optional filename from which source was read; default "<input>" - symbol -- optional grammar start symbol; "single" (default) or "eval" + symbol -- optional grammar start symbol; "single" (default), "exec" + or "eval" Return value / exceptions raised: diff --git a/lib-python/3/collections/__init__.py b/lib-python/3/collections/__init__.py index 64bbee8fab..d353ff2b6f 100644 --- a/lib-python/3/collections/__init__.py +++ b/lib-python/3/collections/__init__.py @@ -47,8 +47,8 @@ def __getattr__(name): obj = getattr(_collections_abc, name) import warnings warnings.warn("Using or importing the ABCs from 'collections' instead " - "of from 'collections.abc' is deprecated, " - "and in 3.8 it will stop working", + "of from 'collections.abc' is deprecated since Python 3.3," + "and in 3.9 it will stop working", DeprecationWarning, stacklevel=2) globals()[name] = obj return obj @@ -444,7 +444,7 @@ def namedtuple(typename, field_names, *, rename=False, defaults=None, module=Non '__slots__': (), '_fields': field_names, '_field_defaults': field_defaults, - # alternate spelling for backward compatiblity + # alternate spelling for backward compatibility '_fields_defaults': field_defaults, '__new__': __new__, '_make': _make, diff --git a/lib-python/3/compileall.py b/lib-python/3/compileall.py index aa65c6b904..ff5af96d5b 100644 --- a/lib-python/3/compileall.py +++ b/lib-python/3/compileall.py @@ -41,7 +41,7 @@ def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): else: dfile = None if not os.path.isdir(fullname): - yield fullname + yield fullname, ddir elif (maxlevels > 0 and name != os.curdir and name != os.pardir and os.path.isdir(fullname) and not os.path.islink(fullname)): yield from _walk_dir(fullname, ddir=dfile, @@ -77,27 +77,32 @@ def compile_dir(dir, maxlevels=10, ddir=None, force=False, rx=None, from concurrent.futures import ProcessPoolExecutor except ImportError: workers = 1 - files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, - ddir=ddir) + files_and_ddirs = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels, + ddir=ddir) success = True if workers is not None and workers != 1 and ProcessPoolExecutor is not None: workers = workers or None with ProcessPoolExecutor(max_workers=workers) as executor: - results = executor.map(partial(compile_file, - ddir=ddir, force=force, - rx=rx, quiet=quiet, - legacy=legacy, - optimize=optimize, - invalidation_mode=invalidation_mode), - files) + results = executor.map( + partial(_compile_file_tuple, + force=force, rx=rx, quiet=quiet, + legacy=legacy, optimize=optimize, + invalidation_mode=invalidation_mode, + ), + files_and_ddirs) success = min(results, default=True) else: - for file in files: - if not compile_file(file, ddir, force, rx, quiet, + for file, dfile in files_and_ddirs: + if not compile_file(file, dfile, force, rx, quiet, legacy, optimize, invalidation_mode): success = False return success +def _compile_file_tuple(file_and_dfile, **kwargs): + """Needs to be toplevel for ProcessPoolExecutor.""" + file, dfile = file_and_dfile + return compile_file(file, dfile, **kwargs) + def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None): diff --git a/lib-python/3/contextlib.py b/lib-python/3/contextlib.py index 2d745ea3e3..13a3e1a553 100644 --- a/lib-python/3/contextlib.py +++ b/lib-python/3/contextlib.py @@ -186,7 +186,7 @@ class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, # in this implementation try: await self.gen.athrow(typ, value, traceback) - raise RuntimeError("generator didn't stop after throw()") + raise RuntimeError("generator didn't stop after athrow()") except StopAsyncIteration as exc: return exc is not value except RuntimeError as exc: diff --git a/lib-python/3/copy.py b/lib-python/3/copy.py index f86040a33c..e634c43e34 100644 --- a/lib-python/3/copy.py +++ b/lib-python/3/copy.py @@ -111,7 +111,7 @@ _copy_dispatch = d = {} def _copy_immutable(x): return x for t in (type(None), int, float, bool, complex, str, tuple, - bytes, frozenset, type, range, slice, + bytes, frozenset, type, range, slice, property, types.BuiltinFunctionType, type(Ellipsis), type(NotImplemented), types.FunctionType, weakref.ref): d[t] = _copy_immutable @@ -206,6 +206,7 @@ d[type] = _deepcopy_atomic d[types.BuiltinFunctionType] = _deepcopy_atomic d[types.FunctionType] = _deepcopy_atomic d[weakref.ref] = _deepcopy_atomic +d[property] = _deepcopy_atomic def _deepcopy_list(x, memo, deepcopy=deepcopy): y = [] diff --git a/lib-python/3/ctypes/macholib/dyld.py b/lib-python/3/ctypes/macholib/dyld.py index c158e672f0..9d86b05876 100644 --- a/lib-python/3/ctypes/macholib/dyld.py +++ b/lib-python/3/ctypes/macholib/dyld.py @@ -149,6 +149,8 @@ def framework_find(fn, executable_path=None, env=None): return dyld_find(fn, executable_path=executable_path, env=env) except ValueError: raise error + finally: + error = None def test_dyld_find(): env = {} diff --git a/lib-python/3/ctypes/test/test_callbacks.py b/lib-python/3/ctypes/test/test_callbacks.py index f622093df6..937a06d981 100644 --- a/lib-python/3/ctypes/test/test_callbacks.py +++ b/lib-python/3/ctypes/test/test_callbacks.py @@ -287,6 +287,21 @@ class SampleCallbacksTestCase(unittest.TestCase): self.assertEqual(s.second, check.second) self.assertEqual(s.third, check.third) + def test_callback_too_many_args(self): + def func(*args): + return len(args) + + CTYPES_MAX_ARGCOUNT = 1024 + proto = CFUNCTYPE(c_int, *(c_int,) * CTYPES_MAX_ARGCOUNT) + cb = proto(func) + args1 = (1,) * CTYPES_MAX_ARGCOUNT + self.assertEqual(cb(*args1), CTYPES_MAX_ARGCOUNT) + + args2 = (1,) * (CTYPES_MAX_ARGCOUNT + 1) + with self.assertRaises(ArgumentError): + cb(*args2) + + ################################################################ if __name__ == '__main__': diff --git a/lib-python/3/ctypes/test/test_structures.py b/lib-python/3/ctypes/test/test_structures.py index d1ea43bc7e..2eb057a9f3 100644 --- a/lib-python/3/ctypes/test/test_structures.py +++ b/lib-python/3/ctypes/test/test_structures.py @@ -1,3 +1,5 @@ +import platform +import sys import unittest from ctypes import * from ctypes.test import need_symbol @@ -438,6 +440,259 @@ class StructureTestCase(unittest.TestCase): self.assertEqual(s.first, got.first) self.assertEqual(s.second, got.second) + def test_array_in_struct(self): + # See bpo-22273 + + # These should mirror the structures in Modules/_ctypes/_ctypes_test.c + class Test2(Structure): + _fields_ = [ + ('data', c_ubyte * 16), + ] + + class Test3(Structure): + _fields_ = [ + ('data', c_double * 2), + ] + + class Test3A(Structure): + _fields_ = [ + ('data', c_float * 2), + ] + + class Test3B(Test3A): + _fields_ = [ + ('more_data', c_float * 2), + ] + + s = Test2() + expected = 0 + for i in range(16): + s.data[i] = i + expected += i + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_array_in_struct1 + func.restype = c_int + func.argtypes = (Test2,) + result = func(s) + self.assertEqual(result, expected) + # check the passed-in struct hasn't changed + for i in range(16): + self.assertEqual(s.data[i], i) + + s = Test3() + s.data[0] = 3.14159 + s.data[1] = 2.71828 + expected = 3.14159 + 2.71828 + func = dll._testfunc_array_in_struct2 + func.restype = c_double + func.argtypes = (Test3,) + result = func(s) + self.assertEqual(result, expected) + # check the passed-in struct hasn't changed + self.assertEqual(s.data[0], 3.14159) + self.assertEqual(s.data[1], 2.71828) + + s = Test3B() + s.data[0] = 3.14159 + s.data[1] = 2.71828 + s.more_data[0] = -3.0 + s.more_data[1] = -2.0 + + expected = 3.14159 + 2.71828 - 5.0 + func = dll._testfunc_array_in_struct2a + func.restype = c_double + func.argtypes = (Test3B,) + result = func(s) + self.assertAlmostEqual(result, expected, places=6) + # check the passed-in struct hasn't changed + self.assertAlmostEqual(s.data[0], 3.14159, places=6) + self.assertAlmostEqual(s.data[1], 2.71828, places=6) + self.assertAlmostEqual(s.more_data[0], -3.0, places=6) + self.assertAlmostEqual(s.more_data[1], -2.0, places=6) + + def test_38368(self): + class U(Union): + _fields_ = [ + ('f1', c_uint8 * 16), + ('f2', c_uint16 * 8), + ('f3', c_uint32 * 4), + ] + u = U() + u.f3[0] = 0x01234567 + u.f3[1] = 0x89ABCDEF + u.f3[2] = 0x76543210 + u.f3[3] = 0xFEDCBA98 + f1 = [u.f1[i] for i in range(16)] + f2 = [u.f2[i] for i in range(8)] + if sys.byteorder == 'little': + self.assertEqual(f1, [0x67, 0x45, 0x23, 0x01, + 0xef, 0xcd, 0xab, 0x89, + 0x10, 0x32, 0x54, 0x76, + 0x98, 0xba, 0xdc, 0xfe]) + self.assertEqual(f2, [0x4567, 0x0123, 0xcdef, 0x89ab, + 0x3210, 0x7654, 0xba98, 0xfedc]) + + @unittest.skipIf(True, 'Test disabled for now - see bpo-16575/bpo-16576') + def test_union_by_value(self): + # See bpo-16575 + + # These should mirror the structures in Modules/_ctypes/_ctypes_test.c + + class Nested1(Structure): + _fields_ = [ + ('an_int', c_int), + ('another_int', c_int), + ] + + class Test4(Union): + _fields_ = [ + ('a_long', c_long), + ('a_struct', Nested1), + ] + + class Nested2(Structure): + _fields_ = [ + ('an_int', c_int), + ('a_union', Test4), + ] + + class Test5(Structure): + _fields_ = [ + ('an_int', c_int), + ('nested', Nested2), + ('another_int', c_int), + ] + + test4 = Test4() + dll = CDLL(_ctypes_test.__file__) + with self.assertRaises(TypeError) as ctx: + func = dll._testfunc_union_by_value1 + func.restype = c_long + func.argtypes = (Test4,) + result = func(test4) + self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' + 'a union by value, which is unsupported.') + test5 = Test5() + with self.assertRaises(TypeError) as ctx: + func = dll._testfunc_union_by_value2 + func.restype = c_long + func.argtypes = (Test5,) + result = func(test5) + self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' + 'a union by value, which is unsupported.') + + # passing by reference should be OK + test4.a_long = 12345; + func = dll._testfunc_union_by_reference1 + func.restype = c_long + func.argtypes = (POINTER(Test4),) + result = func(byref(test4)) + self.assertEqual(result, 12345) + self.assertEqual(test4.a_long, 0) + self.assertEqual(test4.a_struct.an_int, 0) + self.assertEqual(test4.a_struct.another_int, 0) + test4.a_struct.an_int = 0x12340000 + test4.a_struct.another_int = 0x5678 + func = dll._testfunc_union_by_reference2 + func.restype = c_long + func.argtypes = (POINTER(Test4),) + result = func(byref(test4)) + self.assertEqual(result, 0x12345678) + self.assertEqual(test4.a_long, 0) + self.assertEqual(test4.a_struct.an_int, 0) + self.assertEqual(test4.a_struct.another_int, 0) + test5.an_int = 0x12000000 + test5.nested.an_int = 0x345600 + test5.another_int = 0x78 + func = dll._testfunc_union_by_reference3 + func.restype = c_long + func.argtypes = (POINTER(Test5),) + result = func(byref(test5)) + self.assertEqual(result, 0x12345678) + self.assertEqual(test5.an_int, 0) + self.assertEqual(test5.nested.an_int, 0) + self.assertEqual(test5.another_int, 0) + + @unittest.skipIf(True, 'Test disabled for now - see bpo-16575/bpo-16576') + def test_bitfield_by_value(self): + # See bpo-16576 + + # These should mirror the structures in Modules/_ctypes/_ctypes_test.c + + class Test6(Structure): + _fields_ = [ + ('A', c_int, 1), + ('B', c_int, 2), + ('C', c_int, 3), + ('D', c_int, 2), + ] + + test6 = Test6() + # As these are signed int fields, all are logically -1 due to sign + # extension. + test6.A = 1 + test6.B = 3 + test6.C = 7 + test6.D = 3 + dll = CDLL(_ctypes_test.__file__) + with self.assertRaises(TypeError) as ctx: + func = dll._testfunc_bitfield_by_value1 + func.restype = c_long + func.argtypes = (Test6,) + result = func(test6) + self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' + 'a struct/union with a bitfield by value, which is ' + 'unsupported.') + # passing by reference should be OK + func = dll._testfunc_bitfield_by_reference1 + func.restype = c_long + func.argtypes = (POINTER(Test6),) + result = func(byref(test6)) + self.assertEqual(result, -4) + self.assertEqual(test6.A, 0) + self.assertEqual(test6.B, 0) + self.assertEqual(test6.C, 0) + self.assertEqual(test6.D, 0) + + class Test7(Structure): + _fields_ = [ + ('A', c_uint, 1), + ('B', c_uint, 2), + ('C', c_uint, 3), + ('D', c_uint, 2), + ] + test7 = Test7() + test7.A = 1 + test7.B = 3 + test7.C = 7 + test7.D = 3 + func = dll._testfunc_bitfield_by_reference2 + func.restype = c_long + func.argtypes = (POINTER(Test7),) + result = func(byref(test7)) + self.assertEqual(result, 14) + self.assertEqual(test7.A, 0) + self.assertEqual(test7.B, 0) + self.assertEqual(test7.C, 0) + self.assertEqual(test7.D, 0) + + # for a union with bitfields, the union check happens first + class Test8(Union): + _fields_ = [ + ('A', c_int, 1), + ('B', c_int, 2), + ('C', c_int, 3), + ('D', c_int, 2), + ] + + test8 = Test8() + with self.assertRaises(TypeError) as ctx: + func = dll._testfunc_bitfield_by_value2 + func.restype = c_long + func.argtypes = (Test8,) + result = func(test8) + self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' + 'a union by value, which is unsupported.') class PointerMemberTestCase(unittest.TestCase): diff --git a/lib-python/3/dataclasses.py b/lib-python/3/dataclasses.py index 325b822d9f..146468ba6c 100644 --- a/lib-python/3/dataclasses.py +++ b/lib-python/3/dataclasses.py @@ -368,23 +368,24 @@ def _create_fn(name, args, body, *, globals=None, locals=None, # worries about external callers. if locals is None: locals = {} - # __builtins__ may be the "builtins" module or - # the value of its "__dict__", - # so make sure "__builtins__" is the module. - if globals is not None and '__builtins__' not in globals: - globals['__builtins__'] = builtins + if 'BUILTINS' not in locals: + locals['BUILTINS'] = builtins return_annotation = '' if return_type is not MISSING: locals['_return_type'] = return_type return_annotation = '->_return_type' args = ','.join(args) - body = '\n'.join(f' {b}' for b in body) + body = '\n'.join(f' {b}' for b in body) # Compute the text of the entire function. - txt = f'def {name}({args}){return_annotation}:\n{body}' + txt = f' def {name}({args}){return_annotation}:\n{body}' - exec(txt, globals, locals) - return locals[name] + local_vars = ', '.join(locals.keys()) + txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" + + ns = {} + exec(txt, globals, ns) + return ns['__create_fn__'](**locals) def _field_assign(frozen, name, value, self_name): @@ -395,7 +396,7 @@ def _field_assign(frozen, name, value, self_name): # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'__builtins__.object.__setattr__({self_name},{name!r},{value})' + return f'BUILTINS.object.__setattr__({self_name},{name!r},{value})' return f'{self_name}.{name}={value}' @@ -472,7 +473,7 @@ def _init_param(f): return f'{f.name}:_type_{f.name}{default}' -def _init_fn(fields, frozen, has_post_init, self_name): +def _init_fn(fields, frozen, has_post_init, self_name, globals): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -490,12 +491,15 @@ def _init_fn(fields, frozen, has_post_init, self_name): raise TypeError(f'non-default argument {f.name!r} ' 'follows default argument') - globals = {'MISSING': MISSING, - '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY} + locals = {f'_type_{f.name}': f.type for f in fields} + locals.update({ + 'MISSING': MISSING, + '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY, + }) body_lines = [] for f in fields: - line = _field_init(f, frozen, globals, self_name) + line = _field_init(f, frozen, locals, self_name) # line is None means that this field doesn't require # initialization (it's a pseudo-field). Just skip it. if line: @@ -511,7 +515,6 @@ def _init_fn(fields, frozen, has_post_init, self_name): if not body_lines: body_lines = ['pass'] - locals = {f'_type_{f.name}': f.type for f in fields} return _create_fn('__init__', [self_name] + [_init_param(f) for f in fields if f.init], body_lines, @@ -520,20 +523,19 @@ def _init_fn(fields, frozen, has_post_init, self_name): return_type=None) -def _repr_fn(fields): +def _repr_fn(fields, globals): fn = _create_fn('__repr__', ('self',), ['return self.__class__.__qualname__ + f"(' + ', '.join([f"{f.name}={{self.{f.name}!r}}" for f in fields]) + - ')"']) + ')"'], + globals=globals) return _recursive_repr(fn) -def _frozen_get_del_attr(cls, fields): - # XXX: globals is modified on the first call to _create_fn, then - # the modified version is used in the second call. Is this okay? - globals = {'cls': cls, +def _frozen_get_del_attr(cls, fields, globals): + locals = {'cls': cls, 'FrozenInstanceError': FrozenInstanceError} if fields: fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' @@ -545,17 +547,19 @@ def _frozen_get_del_attr(cls, fields): (f'if type(self) is cls or name in {fields_str}:', ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', f'super(cls, self).__setattr__(name, value)'), + locals=locals, globals=globals), _create_fn('__delattr__', ('self', 'name'), (f'if type(self) is cls or name in {fields_str}:', ' raise FrozenInstanceError(f"cannot delete field {name!r}")', f'super(cls, self).__delattr__(name)'), + locals=locals, globals=globals), ) -def _cmp_fn(name, op, self_tuple, other_tuple): +def _cmp_fn(name, op, self_tuple, other_tuple, globals): # Create a comparison function. If the fields in the object are # named 'x' and 'y', then self_tuple is the string # '(self.x,self.y)' and other_tuple is the string @@ -565,14 +569,16 @@ def _cmp_fn(name, op, self_tuple, other_tuple): ('self', 'other'), [ 'if other.__class__ is self.__class__:', f' return {self_tuple}{op}{other_tuple}', - 'return NotImplemented']) + 'return NotImplemented'], + globals=globals) -def _hash_fn(fields): +def _hash_fn(fields, globals): self_tuple = _tuple_str('self', fields) return _create_fn('__hash__', ('self',), - [f'return hash({self_tuple})']) + [f'return hash({self_tuple})'], + globals=globals) def _is_classvar(a_type, typing): @@ -744,14 +750,14 @@ def _set_new_attribute(cls, name, value): # take. The common case is to do nothing, so instead of providing a # function that is a no-op, use None to signify that. -def _hash_set_none(cls, fields): +def _hash_set_none(cls, fields, globals): return None -def _hash_add(cls, fields): +def _hash_add(cls, fields, globals): flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] - return _hash_fn(flds) + return _hash_fn(flds, globals) -def _hash_exception(cls, fields): +def _hash_exception(cls, fields, globals): # Raise an exception. raise TypeError(f'Cannot overwrite attribute __hash__ ' f'in class {cls.__name__}') @@ -793,6 +799,16 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # is defined by the base class, which is found first. fields = {} + if cls.__module__ in sys.modules: + globals = sys.modules[cls.__module__].__dict__ + else: + # Theoretically this can happen if someone writes + # a custom string to cls.__module__. In which case + # such dataclass won't be fully introspectable + # (w.r.t. typing.get_type_hints) but will still function + # correctly. + globals = {} + setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, unsafe_hash, frozen)) @@ -902,6 +918,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # if possible. '__dataclass_self__' if 'self' in fields else 'self', + globals, )) # Get the fields as a list, and include only real fields. This is @@ -910,7 +927,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): if repr: flds = [f for f in field_list if f.repr] - _set_new_attribute(cls, '__repr__', _repr_fn(flds)) + _set_new_attribute(cls, '__repr__', _repr_fn(flds, globals)) if eq: # Create _eq__ method. There's no need for a __ne__ method, @@ -920,7 +937,8 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): other_tuple = _tuple_str('other', flds) _set_new_attribute(cls, '__eq__', _cmp_fn('__eq__', '==', - self_tuple, other_tuple)) + self_tuple, other_tuple, + globals=globals)) if order: # Create and set the ordering methods. @@ -933,13 +951,14 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): ('__ge__', '>='), ]: if _set_new_attribute(cls, name, - _cmp_fn(name, op, self_tuple, other_tuple)): + _cmp_fn(name, op, self_tuple, other_tuple, + globals=globals)): raise TypeError(f'Cannot overwrite attribute {name} ' f'in class {cls.__name__}. Consider using ' 'functools.total_ordering') if frozen: - for fn in _frozen_get_del_attr(cls, field_list): + for fn in _frozen_get_del_attr(cls, field_list, globals): if _set_new_attribute(cls, fn.__name__, fn): raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' f'in class {cls.__name__}') @@ -952,7 +971,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): if hash_action: # No need to call _set_new_attribute here, since by the time # we're here the overwriting is unconditional. - cls.__hash__ = hash_action(cls, field_list) + cls.__hash__ = hash_action(cls, field_list, globals) if not getattr(cls, '__doc__'): # Create a class doc-string. @@ -1011,13 +1030,14 @@ def fields(class_or_instance): def _is_dataclass_instance(obj): """Returns True if obj is an instance of a dataclass.""" - return not isinstance(obj, type) and hasattr(obj, _FIELDS) + return hasattr(type(obj), _FIELDS) def is_dataclass(obj): """Returns True if obj is a dataclass or an instance of a dataclass.""" - return hasattr(obj, _FIELDS) + cls = obj if isinstance(obj, type) else type(obj) + return hasattr(cls, _FIELDS) def asdict(obj, *, dict_factory=dict): @@ -1185,7 +1205,7 @@ def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, raise TypeError(f'Invalid field: {item!r}') if not isinstance(name, str) or not name.isidentifier(): - raise TypeError(f'Field names must be valid identifers: {name!r}') + raise TypeError(f'Field names must be valid identifiers: {name!r}') if keyword.iskeyword(name): raise TypeError(f'Field names must not be keywords: {name!r}') if name in seen: @@ -1202,7 +1222,7 @@ def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, unsafe_hash=unsafe_hash, frozen=frozen) -def replace(obj, **changes): +def replace(*args, **changes): """Return a new object replacing specified fields with new values. This is especially useful for frozen classes. Example usage: @@ -1216,6 +1236,14 @@ def replace(obj, **changes): c1 = replace(c, x=3) assert c1.x == 3 and c1.y == 2 """ + if len(args) > 1: + raise TypeError(f'replace() takes 1 positional argument but {len(args)} were given') + if args: + obj, = args + elif 'obj' in changes: + obj = changes.pop('obj') + else: + raise TypeError("replace() missing 1 required positional argument: 'obj'") # We're going to mutate 'changes', but that's okay because it's a # new dict, even if called with 'replace(obj, **my_changes)'. diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py index a964b202e3..5a2ee6a203 100644 --- a/lib-python/3/datetime.py +++ b/lib-python/3/datetime.py @@ -718,31 +718,31 @@ class timedelta: if isinstance(other, timedelta): return self._cmp(other) == 0 else: - return False + return NotImplemented def __le__(self, other): if isinstance(other, timedelta): return self._cmp(other) <= 0 else: - _cmperror(self, other) + return NotImplemented def __lt__(self, other): if isinstance(other, timedelta): return self._cmp(other) < 0 else: - _cmperror(self, other) + return NotImplemented def __ge__(self, other): if isinstance(other, timedelta): return self._cmp(other) >= 0 else: - _cmperror(self, other) + return NotImplemented def __gt__(self, other): if isinstance(other, timedelta): return self._cmp(other) > 0 else: - _cmperror(self, other) + return NotImplemented def _cmp(self, other): assert isinstance(other, timedelta) @@ -1261,31 +1261,31 @@ class time: if isinstance(other, time): return self._cmp(other, allow_mixed=True) == 0 else: - return False + return NotImplemented def __le__(self, other): if isinstance(other, time): return self._cmp(other) <= 0 else: - _cmperror(self, other) + return NotImplemented def __lt__(self, other): if isinstance(other, time): return self._cmp(other) < 0 else: - _cmperror(self, other) + return NotImplemented def __ge__(self, other): if isinstance(other, time): return self._cmp(other) >= 0 else: - _cmperror(self, other) + return NotImplemented def __gt__(self, other): if isinstance(other, time): return self._cmp(other) > 0 else: - _cmperror(self, other) + return NotImplemented def _cmp(self, other, allow_mixed=False): assert isinstance(other, time) @@ -2167,9 +2167,9 @@ class timezone(tzinfo): return (self._offset, self._name) def __eq__(self, other): - if type(other) != timezone: - return False - return self._offset == other._offset + if isinstance(other, timezone): + return self._offset == other._offset + return NotImplemented def __hash__(self): return hash(self._offset) @@ -2226,7 +2226,7 @@ class timezone(tzinfo): raise TypeError("fromutc() argument must be a datetime instance" " or None") - _maxoffset = timedelta(hours=23, minutes=59) + _maxoffset = timedelta(hours=24, microseconds=-1) _minoffset = -_maxoffset @staticmethod @@ -2250,8 +2250,11 @@ class timezone(tzinfo): return f'UTC{sign}{hours:02d}:{minutes:02d}' timezone.utc = timezone._create(timedelta(0)) -timezone.min = timezone._create(timezone._minoffset) -timezone.max = timezone._create(timezone._maxoffset) +# bpo-37642: These attributes are rounded to the nearest minute for backwards +# compatibility, even though the constructor will accept a wider range of +# values. This may change in the future. +timezone.min = timezone._create(-timedelta(hours=23, minutes=59)) +timezone.max = timezone._create(timedelta(hours=23, minutes=59)) _EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) # Some time zone algebra. For a datetime x, let diff --git a/lib-python/3/difflib.py b/lib-python/3/difflib.py index 887c3c26ca..9528690e78 100644 --- a/lib-python/3/difflib.py +++ b/lib-python/3/difflib.py @@ -1085,7 +1085,7 @@ import re def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): r""" - Return 1 for ignorable line: iff `line` is blank or contains a single '#'. + Return True for ignorable line: iff `line` is blank or contains a single '#'. Examples: @@ -1101,7 +1101,7 @@ def IS_LINE_JUNK(line, pat=re.compile(r"\s*(?:#\s*)?$").match): def IS_CHARACTER_JUNK(ch, ws=" \t"): r""" - Return 1 for ignorable character: iff `ch` is a space or tab. + Return True for ignorable character: iff `ch` is a space or tab. Examples: diff --git a/lib-python/3/distutils/_msvccompiler.py b/lib-python/3/distutils/_msvccompiler.py index 58b20a2102..37ac478fb6 100644 --- a/lib-python/3/distutils/_msvccompiler.py +++ b/lib-python/3/distutils/_msvccompiler.py @@ -90,23 +90,11 @@ def _find_vc2017(): return None, None def _find_vcvarsall(plat_spec): + # bpo-38597: Removed vcruntime return value _, best_dir = _find_vc2017() - vcruntime = None - vcruntime_plat = 'x64' if 'amd64' in plat_spec else 'x86' - if best_dir: - vcredist = os.path.join(best_dir, "..", "..", "redist", "MSVC", "**", - "Microsoft.VC141.CRT", "vcruntime140.dll") - try: - import glob - vcruntime = glob.glob(vcredist, recursive=True)[-1] - except (ImportError, OSError, LookupError): - vcruntime = None if not best_dir: best_version, best_dir = _find_vc2015() - if best_version: - vcruntime = os.path.join(best_dir, 'redist', vcruntime_plat, - "Microsoft.VC140.CRT", "vcruntime140.dll") if not best_dir: log.debug("No suitable Visual C++ version found") @@ -117,11 +105,7 @@ def _find_vcvarsall(plat_spec): log.debug("%s cannot be found", vcvarsall) return None, None - if not vcruntime or not os.path.isfile(vcruntime): - log.debug("%s cannot be found", vcruntime) - vcruntime = None - - return vcvarsall, vcruntime + return vcvarsall, None def _get_vc_env(plat_spec): if os.getenv("DISTUTILS_USE_SDK"): @@ -130,7 +114,7 @@ def _get_vc_env(plat_spec): for key, value in os.environ.items() } - vcvarsall, vcruntime = _find_vcvarsall(plat_spec) + vcvarsall, _ = _find_vcvarsall(plat_spec) if not vcvarsall: raise DistutilsPlatformError("Unable to find vcvarsall.bat") @@ -151,8 +135,6 @@ def _get_vc_env(plat_spec): if key and value } - if vcruntime: - env['py_vcruntime_redist'] = vcruntime return env def _find_exe(exe, paths=None): @@ -180,12 +162,6 @@ PLAT_TO_VCVARS = { 'win-amd64' : 'x86_amd64', } -# A set containing the DLLs that are guaranteed to be available for -# all micro versions of this Python version. Known extension -# dependencies that are not in this set will be copied to the output -# path. -_BUNDLED_DLLS = frozenset(['vcruntime140.dll']) - class MSVCCompiler(CCompiler) : """Concrete class that implements an interface to Microsoft Visual C++, as defined by the CCompiler abstract class.""" @@ -249,7 +225,6 @@ class MSVCCompiler(CCompiler) : self.rc = _find_exe("rc.exe", paths) # resource compiler self.mc = _find_exe("mc.exe", paths) # message compiler self.mt = _find_exe("mt.exe", paths) # message compiler - self._vcruntime_redist = vc_env.get('py_vcruntime_redist', '') for dir in vc_env.get('include', '').split(os.pathsep): if dir: @@ -260,13 +235,12 @@ class MSVCCompiler(CCompiler) : self.add_library_dir(dir.rstrip(os.sep)) self.preprocess_options = None - # If vcruntime_redist is available, link against it dynamically. Otherwise, - # use /MT[d] to build statically, then switch from libucrt[d].lib to ucrt[d].lib - # later to dynamically link to ucrtbase but not vcruntime. + # bpo-38597: Always compile with dynamic linking + # Future releases of Python 3.x will include all past + # versions of vcruntime*.dll for compatibility. self.compile_options = [ - '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG' + '/nologo', '/Ox', '/W3', '/GL', '/DNDEBUG', '/MD' ] - self.compile_options.append('/MD' if self._vcruntime_redist else '/MT') self.compile_options_debug = [ '/nologo', '/Od', '/MDd', '/Zi', '/W3', '/D_DEBUG' @@ -275,8 +249,6 @@ class MSVCCompiler(CCompiler) : ldflags = [ '/nologo', '/INCREMENTAL:NO', '/LTCG' ] - if not self._vcruntime_redist: - ldflags.extend(('/nodefaultlib:libucrt.lib', 'ucrt.lib')) ldflags_debug = [ '/nologo', '/INCREMENTAL:NO', '/LTCG', '/DEBUG:FULL' @@ -518,24 +490,11 @@ class MSVCCompiler(CCompiler) : try: log.debug('Executing "%s" %s', self.linker, ' '.join(ld_args)) self.spawn([self.linker] + ld_args) - self._copy_vcruntime(output_dir) except DistutilsExecError as msg: raise LinkError(msg) else: log.debug("skipping %s (up-to-date)", output_filename) - def _copy_vcruntime(self, output_dir): - vcruntime = self._vcruntime_redist - if not vcruntime or not os.path.isfile(vcruntime): - return - - if os.path.basename(vcruntime).lower() in _BUNDLED_DLLS: - return - - log.debug('Copying "%s"', vcruntime) - vcruntime = shutil.copy(vcruntime, output_dir) - os.chmod(vcruntime, stat.S_IWRITE) - def spawn(self, cmd): old_path = os.getenv('path') try: diff --git a/lib-python/3/distutils/ccompiler.py b/lib-python/3/distutils/ccompiler.py index b71d1d39bc..b70e5e4b5f 100644 --- a/lib-python/3/distutils/ccompiler.py +++ b/lib-python/3/distutils/ccompiler.py @@ -781,8 +781,9 @@ class CCompiler: for incl in includes: f.write("""#include "%s"\n""" % incl) f.write("""\ -main (int argc, char **argv) { +int main (int argc, char **argv) { %s(); + return 0; } """ % funcname) finally: diff --git a/lib-python/3/distutils/command/bdist_wininst.py b/lib-python/3/distutils/command/bdist_wininst.py index fde56754e8..15434c3a98 100644 --- a/lib-python/3/distutils/command/bdist_wininst.py +++ b/lib-python/3/distutils/command/bdist_wininst.py @@ -55,6 +55,9 @@ class bdist_wininst(Command): boolean_options = ['keep-temp', 'no-target-compile', 'no-target-optimize', 'skip-build'] + # bpo-10945: bdist_wininst requires mbcs encoding only available on Windows + _unsupported = (sys.platform != "win32") + def initialize_options(self): self.bdist_dir = None self.plat_name = None diff --git a/lib-python/3/distutils/tests/__init__.py b/lib-python/3/distutils/tests/__init__.py index 1b939cbd5d..5d2e69e3e6 100644 --- a/lib-python/3/distutils/tests/__init__.py +++ b/lib-python/3/distutils/tests/__init__.py @@ -15,6 +15,7 @@ by import rather than matching pre-defined names. import os import sys import unittest +import warnings from test.support import run_unittest @@ -22,6 +23,7 @@ here = os.path.dirname(__file__) or os.curdir def test_suite(): + old_filters = warnings.filters[:] suite = unittest.TestSuite() for fn in os.listdir(here): if fn.startswith("test") and fn.endswith(".py"): @@ -29,6 +31,10 @@ def test_suite(): __import__(modname) module = sys.modules[modname] suite.addTest(module.test_suite()) + # bpo-40055: Save/restore warnings filters to leave them unchanged. + # Importing tests imports docutils which imports pkg_resources which adds a + # warnings filter. + warnings.filters[:] = old_filters return suite diff --git a/lib-python/3/distutils/tests/test_build_ext.py b/lib-python/3/distutils/tests/test_build_ext.py index 88847f9e9a..d0428599a4 100644 --- a/lib-python/3/distutils/tests/test_build_ext.py +++ b/lib-python/3/distutils/tests/test_build_ext.py @@ -470,7 +470,7 @@ class BuildExtTestCase(TempdirManager, # format the target value as defined in the Apple # Availability Macros. We can't use the macro names since # at least one value we test with will not exist yet. - if target[1] < 10: + if target[:2] < (10, 10): # for 10.1 through 10.9.x -> "10n0" target = '%02d%01d0' % target else: diff --git a/lib-python/3/distutils/tests/test_msvccompiler.py b/lib-python/3/distutils/tests/test_msvccompiler.py index 70a9c93a4e..b518d6a78b 100644 --- a/lib-python/3/distutils/tests/test_msvccompiler.py +++ b/lib-python/3/distutils/tests/test_msvccompiler.py @@ -32,57 +32,6 @@ class msvccompilerTestCase(support.TempdirManager, finally: _msvccompiler._find_vcvarsall = old_find_vcvarsall - def test_compiler_options(self): - import distutils._msvccompiler as _msvccompiler - # suppress path to vcruntime from _find_vcvarsall to - # check that /MT is added to compile options - old_find_vcvarsall = _msvccompiler._find_vcvarsall - def _find_vcvarsall(plat_spec): - return old_find_vcvarsall(plat_spec)[0], None - _msvccompiler._find_vcvarsall = _find_vcvarsall - try: - compiler = _msvccompiler.MSVCCompiler() - compiler.initialize() - - self.assertIn('/MT', compiler.compile_options) - self.assertNotIn('/MD', compiler.compile_options) - finally: - _msvccompiler._find_vcvarsall = old_find_vcvarsall - - def test_vcruntime_copy(self): - import distutils._msvccompiler as _msvccompiler - # force path to a known file - it doesn't matter - # what we copy as long as its name is not in - # _msvccompiler._BUNDLED_DLLS - old_find_vcvarsall = _msvccompiler._find_vcvarsall - def _find_vcvarsall(plat_spec): - return old_find_vcvarsall(plat_spec)[0], __file__ - _msvccompiler._find_vcvarsall = _find_vcvarsall - try: - tempdir = self.mkdtemp() - compiler = _msvccompiler.MSVCCompiler() - compiler.initialize() - compiler._copy_vcruntime(tempdir) - - self.assertTrue(os.path.isfile(os.path.join( - tempdir, os.path.basename(__file__)))) - finally: - _msvccompiler._find_vcvarsall = old_find_vcvarsall - - def test_vcruntime_skip_copy(self): - import distutils._msvccompiler as _msvccompiler - - tempdir = self.mkdtemp() - compiler = _msvccompiler.MSVCCompiler() - compiler.initialize() - dll = compiler._vcruntime_redist - self.assertTrue(os.path.isfile(dll), dll or "<None>") - - compiler._copy_vcruntime(tempdir) - - self.assertFalse(os.path.isfile(os.path.join( - tempdir, os.path.basename(dll))), dll or "<None>") - def test_get_vc_env_unicode(self): import distutils._msvccompiler as _msvccompiler diff --git a/lib-python/3/distutils/unixccompiler.py b/lib-python/3/distutils/unixccompiler.py index d10a78da31..4d7a6de740 100644 --- a/lib-python/3/distutils/unixccompiler.py +++ b/lib-python/3/distutils/unixccompiler.py @@ -288,7 +288,7 @@ class UnixCCompiler(CCompiler): # vs # /usr/lib/libedit.dylib cflags = sysconfig.get_config_var('CFLAGS') - m = re.search(r'-isysroot\s+(\S+)', cflags) + m = re.search(r'-isysroot\s*(\S+)', cflags) if m is None: sysroot = '/' else: diff --git a/lib-python/3/doctest.py b/lib-python/3/doctest.py index 79d91a040c..47917b486e 100644 --- a/lib-python/3/doctest.py +++ b/lib-python/3/doctest.py @@ -211,6 +211,13 @@ def _normalize_module(module, depth=2): else: raise TypeError("Expected a module, string, or None") +def _newline_convert(data): + # We have two cases to cover and we need to make sure we do + # them in the right order + for newline in ('\r\n', '\r'): + data = data.replace(newline, '\n') + return data + def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) @@ -221,7 +228,7 @@ def _load_testfile(filename, package, module_relative, encoding): file_contents = file_contents.decode(encoding) # get_data() opens files as 'rb', so one must do the equivalent # conversion as universal newlines would do. - return file_contents.replace(os.linesep, '\n'), filename + return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename @@ -1059,7 +1066,8 @@ class DocTestFinder: if module is None: filename = None else: - filename = getattr(module, '__file__', module.__name__) + # __file__ can be None for namespace packages. + filename = getattr(module, '__file__', None) or module.__name__ if filename[-4:] == ".pyc": filename = filename[:-1] return self._parser.get_doctest(docstring, globs, name, diff --git a/lib-python/3/email/_header_value_parser.py b/lib-python/3/email/_header_value_parser.py index fc00b4a098..b7c30c47b8 100644 --- a/lib-python/3/email/_header_value_parser.py +++ b/lib-python/3/email/_header_value_parser.py @@ -566,6 +566,8 @@ class DisplayName(Phrase): @property def display_name(self): res = TokenList(self) + if len(res) == 0: + return res.value if res[0].token_type == 'cfws': res.pop(0) else: @@ -587,7 +589,7 @@ class DisplayName(Phrase): for x in self: if x.token_type == 'quoted-string': quote = True - if quote: + if len(self) != 0 and quote: pre = post = '' if self[0].token_type=='cfws' or self[0][0].token_type=='cfws': pre = ' ' @@ -931,6 +933,10 @@ class EWWhiteSpaceTerminal(WhiteSpaceTerminal): return '' +class _InvalidEwError(errors.HeaderParseError): + """Invalid encoded word found while parsing headers.""" + + # XXX these need to become classes and used as instances so # that a program can't change them in a parse tree and screw # up other parse trees. Maybe should have tests for that, too. @@ -1035,7 +1041,10 @@ def get_encoded_word(value): raise errors.HeaderParseError( "expected encoded word but found {}".format(value)) remstr = ''.join(remainder) - if len(remstr) > 1 and remstr[0] in hexdigits and remstr[1] in hexdigits: + if (len(remstr) > 1 and + remstr[0] in hexdigits and + remstr[1] in hexdigits and + tok.count('?') < 2): # The ? after the CTE was followed by an encoded word escape (=XX). rest, *remainder = remstr.split('?=', 1) tok = tok + '?=' + rest @@ -1046,8 +1055,8 @@ def get_encoded_word(value): value = ''.join(remainder) try: text, charset, lang, defects = _ew.decode('=?' + tok + '?=') - except ValueError: - raise errors.HeaderParseError( + except (ValueError, KeyError): + raise _InvalidEwError( "encoded word format invalid: '{}'".format(ew.cte)) ew.charset = charset ew.lang = lang @@ -1097,9 +1106,12 @@ def get_unstructured(value): token, value = get_fws(value) unstructured.append(token) continue + valid_ew = True if value.startswith('=?'): try: token, value = get_encoded_word(value) + except _InvalidEwError: + valid_ew = False except errors.HeaderParseError: # XXX: Need to figure out how to register defects when # appropriate here. @@ -1121,7 +1133,10 @@ def get_unstructured(value): # Split in the middle of an atom if there is a rfc2047 encoded word # which does not have WSP on both sides. The defect will be registered # the next time through the loop. - if rfc2047_matcher.search(tok): + # This needs to only be performed when the encoded word is valid; + # otherwise, performing it on an invalid encoded word can cause + # the parser to go in an infinite loop. + if valid_ew and rfc2047_matcher.search(tok): tok, *remainder = value.partition('=?') vtext = ValueTerminal(tok, 'vtext') _validate_xtext(vtext) @@ -1189,19 +1204,28 @@ def get_bare_quoted_string(value): "expected '\"' but found '{}'".format(value)) bare_quoted_string = BareQuotedString() value = value[1:] - if value[0] == '"': + if value and value[0] == '"': token, value = get_qcontent(value) bare_quoted_string.append(token) while value and value[0] != '"': if value[0] in WSP: token, value = get_fws(value) elif value[:2] == '=?': + valid_ew = False try: token, value = get_encoded_word(value) bare_quoted_string.defects.append(errors.InvalidHeaderDefect( "encoded word inside quoted string")) + valid_ew = True except errors.HeaderParseError: token, value = get_qcontent(value) + # Collapse the whitespace between two encoded words that occur in a + # bare-quoted-string. + if valid_ew and len(bare_quoted_string) > 1: + if (bare_quoted_string[-1].token_type == 'fws' and + bare_quoted_string[-2].token_type == 'encoded-word'): + bare_quoted_string[-1] = EWWhiteSpaceTerminal( + bare_quoted_string[-1], 'fws') else: token, value = get_qcontent(value) bare_quoted_string.append(token) @@ -1358,6 +1382,9 @@ def get_word(value): leader, value = get_cfws(value) else: leader = None + if not value: + raise errors.HeaderParseError( + "Expected 'atom' or 'quoted-string' but found nothing.") if value[0]=='"': token, value = get_quoted_string(value) elif value[0] in SPECIALS: @@ -1582,6 +1609,8 @@ def get_domain(value): token, value = get_dot_atom(value) except errors.HeaderParseError: token, value = get_atom(value) + if value and value[0] == '@': + raise errors.HeaderParseError('Invalid Domain') if leader is not None: token[:0] = [leader] domain.append(token) @@ -2387,6 +2416,9 @@ def get_parameter(value): while value: if value[0] in WSP: token, value = get_fws(value) + elif value[0] == '"': + token = ValueTerminal('"', 'DQUOTE') + value = value[1:] else: token, value = get_qcontent(value) v.append(token) @@ -2627,6 +2659,9 @@ def _refold_parse_tree(parse_tree, *, policy): wrap_as_ew_blocked -= 1 continue tstr = str(part) + if part.token_type == 'ptext' and set(tstr) & SPECIALS: + # Encode if tstr contains special characters. + want_encoding = True try: tstr.encode(encoding) charset = encoding @@ -2737,15 +2772,22 @@ def _fold_as_ew(to_encode, lines, maxlen, last_ew, ew_combine_allowed, charset): trailing_wsp = to_encode[-1] to_encode = to_encode[:-1] new_last_ew = len(lines[-1]) if last_ew is None else last_ew + + encode_as = 'utf-8' if charset == 'us-ascii' else charset + + # The RFC2047 chrome takes up 7 characters plus the length + # of the charset name. + chrome_len = len(encode_as) + 7 + + if (chrome_len + 1) >= maxlen: + raise errors.HeaderParseError( + "max_line_length is too small to fit an encoded word") + while to_encode: remaining_space = maxlen - len(lines[-1]) - # The RFC2047 chrome takes up 7 characters plus the length - # of the charset name. - encode_as = 'utf-8' if charset == 'us-ascii' else charset - text_space = remaining_space - len(encode_as) - 7 + text_space = remaining_space - chrome_len if text_space <= 0: lines.append(' ') - # XXX We'll get an infinite loop here if maxlen is <= 7 continue to_encode_word = to_encode[:text_space] diff --git a/lib-python/3/email/_parseaddr.py b/lib-python/3/email/_parseaddr.py index cdfa3729ad..41ff6f8c00 100644 --- a/lib-python/3/email/_parseaddr.py +++ b/lib-python/3/email/_parseaddr.py @@ -379,7 +379,12 @@ class AddrlistClass: aslist.append('@') self.pos += 1 self.gotonext() - return EMPTYSTRING.join(aslist) + self.getdomain() + domain = self.getdomain() + if not domain: + # Invalid domain, return an empty address instead of returning a + # local part to denote failed parsing. + return EMPTYSTRING + return EMPTYSTRING.join(aslist) + domain def getdomain(self): """Get the complete domain name from an address.""" @@ -394,6 +399,10 @@ class AddrlistClass: elif self.field[self.pos] == '.': self.pos += 1 sdlist.append('.') + elif self.field[self.pos] == '@': + # bpo-34155: Don't parse domains with two `@` like + # `a@malicious.org@important.com`. + return EMPTYSTRING elif self.field[self.pos] in self.atomends: break else: diff --git a/lib-python/3/email/headerregistry.py b/lib-python/3/email/headerregistry.py index 00652049f2..fe30fc2c7e 100644 --- a/lib-python/3/email/headerregistry.py +++ b/lib-python/3/email/headerregistry.py @@ -31,6 +31,11 @@ class Address: without any Content Transfer Encoding. """ + + inputs = ''.join(filter(None, (display_name, username, domain, addr_spec))) + if '\r' in inputs or '\n' in inputs: + raise ValueError("invalid arguments; address parts cannot contain CR or LF") + # This clause with its potential 'raise' may only happen when an # application program creates an Address object using an addr_spec # keyword. The email library code itself must always supply username @@ -245,7 +250,7 @@ class BaseHeader(str): the header name and the ': ' separator. """ - # At some point we need to put fws here iif it was in the source. + # At some point we need to put fws here if it was in the source. header = parser.Header([ parser.HeaderLabel([ parser.ValueTerminal(self.name, 'header-name'), diff --git a/lib-python/3/email/message.py b/lib-python/3/email/message.py index b6512f2198..1262602617 100644 --- a/lib-python/3/email/message.py +++ b/lib-python/3/email/message.py @@ -1041,7 +1041,16 @@ class MIMEPart(Message): maintype, subtype = self.get_content_type().split('/') if maintype != 'multipart' or subtype == 'alternative': return - parts = self.get_payload().copy() + payload = self.get_payload() + # Certain malformed messages can have content type set to `multipart/*` + # but still have single part body, in which case payload.copy() can + # fail with AttributeError. + try: + parts = payload.copy() + except AttributeError: + # payload is not a list, it is most probably a string. + return + if maintype == 'multipart' and subtype == 'related': # For related, we treat everything but the root as an attachment. # The root may be indicated by 'start'; if there's no start or we diff --git a/lib-python/3/email/parser.py b/lib-python/3/email/parser.py index 555b172560..7db4da1ff0 100644 --- a/lib-python/3/email/parser.py +++ b/lib-python/3/email/parser.py @@ -13,7 +13,6 @@ from email.feedparser import FeedParser, BytesFeedParser from email._policybase import compat32 - class Parser: def __init__(self, _class=None, *, policy=compat32): """Parser of RFC 2822 and MIME email messages. diff --git a/lib-python/3/encodings/punycode.py b/lib-python/3/encodings/punycode.py index 66c51013ea..1c57264470 100644 --- a/lib-python/3/encodings/punycode.py +++ b/lib-python/3/encodings/punycode.py @@ -143,7 +143,7 @@ def decode_generalized_number(extended, extpos, bias, errors): digit = char - 22 # 0x30-26 elif errors == "strict": raise UnicodeError("Invalid extended code point '%s'" - % extended[extpos]) + % extended[extpos-1]) else: return extpos, None t = T(j, bias) diff --git a/lib-python/3/encodings/uu_codec.py b/lib-python/3/encodings/uu_codec.py index 2a5728fb5b..4e58c62fe9 100644 --- a/lib-python/3/encodings/uu_codec.py +++ b/lib-python/3/encodings/uu_codec.py @@ -20,6 +20,10 @@ def uu_encode(input, errors='strict', filename='<data>', mode=0o666): read = infile.read write = outfile.write + # Remove newline chars from filename + filename = filename.replace('\n','\\n') + filename = filename.replace('\r','\\r') + # Encode write(('begin %o %s\n' % (mode & 0o777, filename)).encode('ascii')) chunk = read(45) diff --git a/lib-python/3/ensurepip/__init__.py b/lib-python/3/ensurepip/__init__.py index 526dfd004a..94d40b0c8d 100644 --- a/lib-python/3/ensurepip/__init__.py +++ b/lib-python/3/ensurepip/__init__.py @@ -2,19 +2,20 @@ import os import os.path import pkgutil import sys +import runpy import tempfile __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "40.8.0" +_SETUPTOOLS_VERSION = "47.1.0" -_PIP_VERSION = "19.0.3" +_PIP_VERSION = "20.1.1" _PROJECTS = [ - ("setuptools", _SETUPTOOLS_VERSION), - ("pip", _PIP_VERSION), + ("setuptools", _SETUPTOOLS_VERSION, "py3"), + ("pip", _PIP_VERSION, "py2.py3"), ] @@ -23,9 +24,18 @@ def _run_pip(args, additional_paths=None): if additional_paths is not None: sys.path = additional_paths + sys.path - # Install the bundled software - import pip._internal - return pip._internal.main(args) + # Invoke pip as if it's the main module, and catch the exit. + backup_argv = sys.argv[:] + sys.argv[1:] = args + try: + # run_module() alters sys.modules and sys.argv, but restores them at exit + runpy.run_module("pip", run_name="__main__", alter_sys=True) + except SystemExit as exc: + return exc.code + finally: + sys.argv[:] = backup_argv + + raise SystemError("pip did not exit, this should never happen") def version(): @@ -92,8 +102,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path additional_paths = [] - for project, version in _PROJECTS: - wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + for project, version, py_tag in _PROJECTS: + wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag) whl = pkgutil.get_data( "ensurepip", "_bundled/{}".format(wheel_name), @@ -104,7 +114,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False, additional_paths.append(os.path.join(tmpdir, wheel_name)) # Construct the arguments to be passed to the pip command - args = ["install", "--no-index", "--find-links", tmpdir] + args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] if root: args += ["--root", root] if upgrade: diff --git a/lib-python/3/ensurepip/_bundled/pip-19.0.3-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/pip-19.0.3-py2.py3-none-any.whl Binary files differdeleted file mode 100644 index 24f247caa0..0000000000 --- a/lib-python/3/ensurepip/_bundled/pip-19.0.3-py2.py3-none-any.whl +++ /dev/null diff --git a/lib-python/3/ensurepip/_bundled/pip-20.1.1-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/pip-20.1.1-py2.py3-none-any.whl Binary files differnew file mode 100644 index 0000000000..ea1d0f7c86 --- /dev/null +++ b/lib-python/3/ensurepip/_bundled/pip-20.1.1-py2.py3-none-any.whl diff --git a/lib-python/3/ensurepip/_bundled/setuptools-40.8.0-py2.py3-none-any.whl b/lib-python/3/ensurepip/_bundled/setuptools-47.1.0-py3-none-any.whl Binary files differindex fdc66e9330..f87867ff98 100644 --- a/lib-python/3/ensurepip/_bundled/setuptools-40.8.0-py2.py3-none-any.whl +++ b/lib-python/3/ensurepip/_bundled/setuptools-47.1.0-py3-none-any.whl diff --git a/lib-python/3/enum.py b/lib-python/3/enum.py index 5e97a9e8d8..83e6410107 100644 --- a/lib-python/3/enum.py +++ b/lib-python/3/enum.py @@ -66,6 +66,7 @@ class _EnumDict(dict): self._member_names = [] self._last_values = [] self._ignore = [] + self._auto_called = False def __setitem__(self, key, value): """Changes anything not dundered or not a descriptor. @@ -83,6 +84,9 @@ class _EnumDict(dict): ): raise ValueError('_names_ are reserved for future Enum use') if key == '_generate_next_value_': + # check if members already defined as auto() + if self._auto_called: + raise TypeError("_generate_next_value_ must be defined before members") setattr(self, '_generate_next_value', value) elif key == '_ignore_': if isinstance(value, str): @@ -106,6 +110,7 @@ class _EnumDict(dict): # enum overwriting a descriptor? raise TypeError('%r already defined as: %r' % (key, self[key])) if isinstance(value, auto): + self._auto_called = True if value.value == _auto_null: value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:]) value = value.value @@ -682,7 +687,7 @@ class Flag(Enum): Generate the next value when not given. name: the name of the member - start: the initital start value or None + start: the initial start value or None count: the number of existing members last_value: the last value assigned or None """ diff --git a/lib-python/3/fractions.py b/lib-python/3/fractions.py index 8330202d70..64bff70060 100644 --- a/lib-python/3/fractions.py +++ b/lib-python/3/fractions.py @@ -625,7 +625,9 @@ class Fraction(numbers.Rational): def __bool__(a): """a != 0""" - return a._numerator != 0 + # bpo-39274: Use bool() because (a._numerator != 0) can return an + # object which is not a bool. + return bool(a._numerator) # support for pickling, copy, and deepcopy diff --git a/lib-python/3/functools.py b/lib-python/3/functools.py index 1daa1d1775..badaa826af 100644 --- a/lib-python/3/functools.py +++ b/lib-python/3/functools.py @@ -471,7 +471,7 @@ def lru_cache(maxsize=128, typed=False): with f.cache_info(). Clear the cache and statistics with f.cache_clear(). Access the underlying function with f.__wrapped__. - See: http://en.wikipedia.org/wiki/Cache_algorithms#Least_Recently_Used + See: http://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU) """ diff --git a/lib-python/3/genericpath.py b/lib-python/3/genericpath.py index 303b3b349a..85f4a57582 100644 --- a/lib-python/3/genericpath.py +++ b/lib-python/3/genericpath.py @@ -92,7 +92,11 @@ def samestat(s1, s2): # Are two filenames really pointing to the same file? def samefile(f1, f2): - """Test whether two pathnames reference the same actual file""" + """Test whether two pathnames reference the same actual file or directory + + This is determined by the device number and i-node number and + raises an exception if an os.stat() call on either pathname fails. + """ s1 = os.stat(f1) s2 = os.stat(f2) return samestat(s1, s2) diff --git a/lib-python/3/gzip.py b/lib-python/3/gzip.py index ddc7bda1fe..e59b454814 100644 --- a/lib-python/3/gzip.py +++ b/lib-python/3/gzip.py @@ -17,6 +17,11 @@ FTEXT, FHCRC, FEXTRA, FNAME, FCOMMENT = 1, 2, 4, 8, 16 READ, WRITE = 1, 2 +_COMPRESS_LEVEL_FAST = 1 +_COMPRESS_LEVEL_TRADEOFF = 6 +_COMPRESS_LEVEL_BEST = 9 + + def open(filename, mode="rb", compresslevel=9, encoding=None, errors=None, newline=None): """Open a gzip-compressed file in binary or text mode. @@ -191,7 +196,7 @@ class GzipFile(_compression.BaseStream): self.fileobj = fileobj if self.mode == WRITE: - self._write_gzip_header() + self._write_gzip_header(compresslevel) @property def filename(self): @@ -218,7 +223,7 @@ class GzipFile(_compression.BaseStream): self.bufsize = 0 self.offset = 0 # Current file offset for seek(), tell(), etc - def _write_gzip_header(self): + def _write_gzip_header(self, compresslevel): self.fileobj.write(b'\037\213') # magic header self.fileobj.write(b'\010') # compression method try: @@ -239,7 +244,13 @@ class GzipFile(_compression.BaseStream): if mtime is None: mtime = time.time() write32u(self.fileobj, int(mtime)) - self.fileobj.write(b'\002') + if compresslevel == _COMPRESS_LEVEL_BEST: + xfl = b'\002' + elif compresslevel == _COMPRESS_LEVEL_FAST: + xfl = b'\004' + else: + xfl = b'\000' + self.fileobj.write(xfl) self.fileobj.write(b'\377') if fname: self.fileobj.write(fname + b'\000') diff --git a/lib-python/3/http/__init__.py b/lib-python/3/http/__init__.py index aed94a5850..e14a1eb074 100644 --- a/lib-python/3/http/__init__.py +++ b/lib-python/3/http/__init__.py @@ -59,7 +59,7 @@ class HTTPStatus(IntEnum): TEMPORARY_REDIRECT = (307, 'Temporary Redirect', 'Object moved temporarily -- see URI list') PERMANENT_REDIRECT = (308, 'Permanent Redirect', - 'Object moved temporarily -- see URI list') + 'Object moved permanently -- see URI list') # client error BAD_REQUEST = (400, 'Bad Request', diff --git a/lib-python/3/http/client.py b/lib-python/3/http/client.py index dd23edcd59..04cd8f7d84 100644 --- a/lib-python/3/http/client.py +++ b/lib-python/3/http/client.py @@ -150,6 +150,10 @@ _contains_disallowed_url_pchar_re = re.compile('[\x00-\x20\x7f]') # _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$") # We are more lenient for assumed real world compatibility purposes. +# These characters are not allowed within HTTP method names +# to prevent http header injection. +_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]') + # We always set the Content-Length header for these methods because some # servers will otherwise respond with a 411 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} @@ -850,6 +854,8 @@ class HTTPConnection: (self.host, self.port) = self._get_hostport(host, port) + self._validate_host(self.host) + # This is stored as an instance variable to allow unit # tests to replace it with a suitable mockup self._create_connection = socket.create_connection @@ -1107,19 +1113,17 @@ class HTTPConnection: else: raise CannotSendRequest(self.__state) - # Save the method we use, we need it later in the response phase + self._validate_method(method) + + # Save the method for use later in the response phase self._method = method - if not url: - url = '/' - # Prevent CVE-2019-9740. - match = _contains_disallowed_url_pchar_re.search(url) - if match: - raise InvalidURL(f"URL can't contain control characters. {url!r} " - f"(found at least {match.group()!r})") + + url = url or '/' + self._validate_path(url) + request = '%s %s %s' % (method, url, self._http_vsn_str) - # Non-ASCII characters should have been eliminated earlier - self._output(request.encode('ascii')) + self._output(self._encode_request(request)) if self._http_vsn == 11: # Issue some standard headers for better HTTP/1.1 compliance @@ -1197,6 +1201,35 @@ class HTTPConnection: # For HTTP/1.0, the server will assume "not chunked" pass + def _encode_request(self, request): + # ASCII also helps prevent CVE-2019-9740. + return request.encode('ascii') + + def _validate_method(self, method): + """Validate a method name for putrequest.""" + # prevent http header injection + match = _contains_disallowed_method_pchar_re.search(method) + if match: + raise ValueError( + f"method can't contain control characters. {method!r} " + f"(found at least {match.group()!r})") + + def _validate_path(self, url): + """Validate a url for putrequest.""" + # Prevent CVE-2019-9740. + match = _contains_disallowed_url_pchar_re.search(url) + if match: + raise InvalidURL(f"URL can't contain control characters. {url!r} " + f"(found at least {match.group()!r})") + + def _validate_host(self, host): + """Validate a host so it doesn't contain control characters.""" + # Prevent CVE-2019-18348. + match = _contains_disallowed_url_pchar_re.search(host) + if match: + raise InvalidURL(f"URL can't contain control characters. {host!r} " + f"(found at least {match.group()!r})") + def putheader(self, header, *values): """Send a request header line to the server. diff --git a/lib-python/3/http/cookiejar.py b/lib-python/3/http/cookiejar.py index d63544a5f5..d43a2193f9 100644 --- a/lib-python/3/http/cookiejar.py +++ b/lib-python/3/http/cookiejar.py @@ -213,10 +213,14 @@ LOOSE_HTTP_DATE_RE = re.compile( (?::(\d\d))? # optional seconds )? # optional clock \s* - ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+)? # timezone + (?: + ([-+]?\d{2,4}|(?![APap][Mm]\b)[A-Za-z]+) # timezone + \s* + )? + (?: + \(\w+\) # ASCII representation of timezone in parens. \s* - (?:\(\w+\))? # ASCII representation of timezone in parens. - \s*$""", re.X | re.ASCII) + )?$""", re.X | re.ASCII) def http2time(text): """Returns time in seconds since epoch of time represented by a string. @@ -286,9 +290,11 @@ ISO_DATE_RE = re.compile( (?::?(\d\d(?:\.\d*)?))? # optional seconds (and fractional) )? # optional clock \s* - ([-+]?\d\d?:?(:?\d\d)? - |Z|z)? # timezone (Z is "zero meridian", i.e. GMT) - \s*$""", re.X | re. ASCII) + (?: + ([-+]?\d\d?:?(:?\d\d)? + |Z|z) # timezone (Z is "zero meridian", i.e. GMT) + \s* + )?$""", re.X | re. ASCII) def iso2time(text): """ As for http2time, but parses the ISO 8601 formats: @@ -1590,6 +1596,7 @@ class CookieJar: headers = response.info() rfc2965_hdrs = headers.get_all("Set-Cookie2", []) ns_hdrs = headers.get_all("Set-Cookie", []) + self._policy._now = self._now = int(time.time()) rfc2965 = self._policy.rfc2965 netscape = self._policy.netscape @@ -1669,8 +1676,6 @@ class CookieJar: _debug("extract_cookies: %s", response.info()) self._cookies_lock.acquire() try: - self._policy._now = self._now = int(time.time()) - for cookie in self.make_cookies(response, request): if self._policy.set_ok(cookie, request): _debug(" setting cookie: %s", cookie) diff --git a/lib-python/3/idlelib/Icons/README.txt b/lib-python/3/idlelib/Icons/README.txt new file mode 100644 index 0000000000..8b471629ec --- /dev/null +++ b/lib-python/3/idlelib/Icons/README.txt @@ -0,0 +1,9 @@ +The IDLE icons are from https://bugs.python.org/issue1490384 + +Created by Andrew Clover. + +The original sources are available from Andrew's website: +https://www.doxdesk.com/software/py/pyicons.html + +Various different formats and sizes are available at this GitHub Pull Request: +https://github.com/python/cpython/pull/17473 diff --git a/lib-python/3/idlelib/Icons/idle.icns b/lib-python/3/idlelib/Icons/idle.icns Binary files differdeleted file mode 100644 index f65e3130f0..0000000000 --- a/lib-python/3/idlelib/Icons/idle.icns +++ /dev/null diff --git a/lib-python/3/idlelib/Icons/idle_256.png b/lib-python/3/idlelib/Icons/idle_256.png Binary files differnew file mode 100644 index 0000000000..99ffa6fad4 --- /dev/null +++ b/lib-python/3/idlelib/Icons/idle_256.png diff --git a/lib-python/3/idlelib/NEWS.txt b/lib-python/3/idlelib/NEWS.txt index 42227b60e7..c751dc3bb0 100644 --- a/lib-python/3/idlelib/NEWS.txt +++ b/lib-python/3/idlelib/NEWS.txt @@ -1,12 +1,169 @@ +What's New in IDLE 3.7.8 +Released on 2020-06-27? +====================================== + + +bpo-40723: Make test_idle pass when run after import. +Patch by Florian Dahlitz. + +bpo-38689: IDLE will no longer freeze when inspect.signature fails +when fetching a calltip. + + +What's New in IDLE 3.7.7 +Released on 2020-03-10 +====================================== + +bpo-27115: For 'Go to Line', use a Query entry box subclass with +IDLE standard behavior and improved error checking. + +bpo-39885: When a context menu is invoked by right-clicking outside +of a selection, clear the selection and move the cursor. Cut and +Copy require that the click be within the selection. + +bpo-39852: Edit "Go to line" now clears any selection, preventing +accidental deletion. It also updates Ln and Col on the status bar. + +bpo-39781: Selecting code context lines no longer causes a jump. + +bpo-39663: Add tests for pyparse find_good_parse_start(). + +bpo-39600: Remove duplicate font names from configuration list. + +bpo-38792: Close a shell calltip if a :exc:`KeyboardInterrupt` +or shell restart occurs. Patch by Zackery Spytz. + +bpo-30780: Add remaining configdialog tests for buttons and +highlights and keys tabs. + +bpo-39388: Settings dialog Cancel button cancels pending changes. + +bpo-39050: Settings dialog Help button again displays help text. + +bpo-32989: Add tests for editor newline_and_indent_event method. +Remove unneeded arguments and dead code from pyparse +find_good_parse_start method. + + +What's New in IDLE 3.7.6 +Released on 2019-12-18 +====================================== + +bpo-38943: Fix autocomplete windows not always appearing on some +systems. Patch by Johnny Najera. + +bpo-38944: Escape key now closes IDLE completion windows. Patch by +Johnny Najera. + +bpo-38862: 'Strip Trailing Whitespace' on the Format menu removes extra +newlines at the end of non-shell files. + +bpo-38636: Fix IDLE Format menu tab toggle and file indent width. These +functions (default shortcuts Alt-T and Alt-U) were mistakenly disabled +in 3.7.5 and 3.8.0. + +bpo-4360: Add an option to toggle IDLE's cursor blink for shell, +editor, and output windows. See Settings, General, Window Preferences, +Cursor Blink. Patch by Zachary Spytz. + +bpo-26353: Stop adding newline when saving an IDLE shell window. + +bpo-38598: Do not try to compile IDLE shell or output windows. + + +What's New in IDLE 3.7.5 +Released on 2019-10-15 +====================================== + +bpo-36698: IDLE no longer fails when writing non-encodable characters +to stderr. It now escapes them with a backslash, like the regular +Python interpreter. Add an errors field to the standard streams. + +bpo-13153: Improve tkinter's handing of non-BMP (astral) unicode +characters, such as 'rocket \U0001f680'. Whether a proper glyph or +replacement char is displayed depends on the OS and font. For IDLE, +astral chars in code interfere with editing. + +bpo-35379: When exiting IDLE, catch any AttributeError. One happens +when EditorWindow.close is called twice. Printing a traceback, when +IDLE is run from a terminal, is useless and annoying. + +bpo-38183: To avoid test issues, test_idle ignores the user config +directory. It no longer tries to create or access .idlerc or any files +within. Users must run IDLE to discover problems with saving settings. + +bpo-38077: IDLE no longer adds 'argv' to the user namespace when +initializing it. This bug only affected 3.7.4 and 3.8.0b2 to 3.8.0b4. + +bpo-38401: Shell restart lines now fill the window width, always start +with '=', and avoid wrapping unnecessarily. The line will still wrap +if the included file name is long relative to the width. + +bpo-37092: Add mousewheel scrolling for IDLE module, path, and stack +browsers. Patch by George Zhang. + +bpo-35771: To avoid occasional spurious test_idle failures on slower +machines, increase the ``hover_delay`` in test_tooltip. + +bpo-37824: Properly handle user input warnings in IDLE shell. +Cease turning SyntaxWarnings into SyntaxErrors. + +bpo-37929: IDLE Settings dialog now closes properly when there is no +shell window. + +bpo-37849: Fix completions list appearing too high or low when shown +above the current line. + +bpo-36419: Refactor autocompete and improve testing. + +bpo-37748: Reorder the Run menu. Put the most common choice, +Run Module, at the top. + +bpo-37692: Improve highlight config sample with example shell +interaction and better labels for shell elements. + +bpo-37628: Settings dialog no longer expands with font size. +The font and highlight sample boxes gain scrollbars instead. + +bpo-17535: Add optional line numbers for IDLE editor windows. + +bpo-37627: Initialize the Customize Run dialog with the command line +arguments most recently entered before. The user can optionally edit +before submitting them. + +bpo-33610: Code context always shows the correct context when toggled on. + +bpo-36390: Gather Format menu functions into format.py. Combine +paragraph.py, rstrip.py, and format methods from editor.py. + +bpo-37530: Optimize code context to reduce unneeded background activity. +Font and highlight changes now occur along with text changes instead +of after a random delay. + +bpo-27452: Cleanup config.py by inlining RemoveFile and simplifying +the handling of __file__ in CreateConfigHandlers/ + + What's New in IDLE 3.7.4 -Released on 2019-06-24? +Released on 2019-07-08 ====================================== +bpo-26806: To compensate for stack frames added by IDLE and avoid +possible problems with low recursion limits, add 30 to limits in the +user code execution process. Subtract 30 when reporting recursion +limits to make this addition mostly transparent. + +bpo-37325: Fix tab focus traversal order for help source and custom +run dialogs. bpo-37321: Both subprocess connection error messages now refer to the 'Startup failure' section of the IDLE doc. -bpo-37039: Adjust "Zoom Height" to individual screens by momemtarily +bpo-37177: Properly attach search dialogs to their main window so +that they behave like other dialogs and do not get hidden behind +their main window. + +bpo-37039: Adjust "Zoom Height" to individual screens by momentarily maximizing the window on first use with a particular screen. Changing screen settings may invalidate the saved height. While a window is maximized, "Zoom Height" has no effect. @@ -14,6 +171,10 @@ maximized, "Zoom Height" has no effect. bpo-35763: Make calltip reminder about '/' meaning positional-only less obtrusive by only adding it when there is room on the first line. +bpo-5680: Add 'Run Customized' to the Run menu to run a module with +customized settings. Any command line arguments entered are added +to sys.argv. One can suppress the normal Shell main module restart. + bpo-35610: Replace now redundant editor.context_use_ps1 with .prompt_last_line. This finishes change started in bpo-31858. @@ -58,6 +219,9 @@ None or False since 2007. bpo-36096: Make colorizer state variables instance-only. +bpo-32129: Avoid blurry IDLE application icon on macOS with Tk 8.6. +Patch by Kevin Walzer. + bpo-24310: Document settings dialog font tab sample. bpo-35689: Add docstrings and tests for colorizer. diff --git a/lib-python/3/idlelib/README.txt b/lib-python/3/idlelib/README.txt index c784a1a0b6..bc3d978f43 100644 --- a/lib-python/3/idlelib/README.txt +++ b/lib-python/3/idlelib/README.txt @@ -68,7 +68,7 @@ pyshell.py # Start IDLE, manage shell, complete editor window query.py # Query user for information redirector.py # Intercept widget subcommands (for percolator) (nim). replace.py # Search and replace pattern in text. -rpc.py # Commuicate between idle and user processes (nim). +rpc.py # Communicate between idle and user processes (nim). rstrip.py # Strip trailing whitespace. run.py # Manage user code execution subprocess. runscript.py # Check and run user code. @@ -80,7 +80,7 @@ stackviewer.py # View stack after exception. statusbar.py # Define status bar for windows (nim). tabbedpages.py # Define tabbed pages widget (nim). textview.py # Define read-only text widget (nim). -tree.py # Define tree widger, used in browsers (nim). +tree.py # Define tree widget, used in browsers (nim). undo.py # Manage undo stack. windows.py # Manage window list and define listed top level. zoomheight.py # Zoom window to full height of screen. @@ -90,14 +90,14 @@ Configuration config-extensions.def # Defaults for extensions config-highlight.def # Defaults for colorizing config-keys.def # Defaults for key bindings -config-main.def # Defai;ts fpr font and geneal +config-main.def # Defaults for font and general tabs Text ---- CREDITS.txt # not maintained, displayed by About IDLE HISTORY.txt # NEWS up to July 2001 NEWS.txt # commits, displayed by About IDLE -README.txt # this file, displeyed by About IDLE +README.txt # this file, displayed by About IDLE TODO.txt # needs review extend.txt # about writing extensions help.html # copy of idle.html in docs, displayed by IDLE Help @@ -115,7 +115,7 @@ tooltip.py # unused IDLE MENUS Top level items and most submenu items are defined in mainmenu. -Extenstions add submenu items when active. The names given are +Extensions add submenu items when active. The names given are found, quoted, in one of these modules, paired with a '<<pseudoevent>>'. Each pseudoevent is bound to an event handler. Some event handlers call another function that does the actual work. The annotations below @@ -166,7 +166,7 @@ Shell # pyshell Debug (Shell only) Go to File/Line - debugger # debugger, debugger_r, PyShell.toggle_debuger + debugger # debugger, debugger_r, PyShell.toggle_debugger Stack Viewer # stackviewer, PyShell.open_stack_viewer Auto-open Stack Viewer # stackviewer diff --git a/lib-python/3/idlelib/autocomplete.py b/lib-python/3/idlelib/autocomplete.py index e20b757d87..c623d45a15 100644 --- a/lib-python/3/idlelib/autocomplete.py +++ b/lib-python/3/idlelib/autocomplete.py @@ -8,42 +8,45 @@ import os import string import sys -# These constants represent the two different types of completions. -# They must be defined here so autocomple_w can import them. -COMPLETE_ATTRIBUTES, COMPLETE_FILES = range(1, 2+1) - +# Two types of completions; defined here for autocomplete_w import below. +ATTRS, FILES = 0, 1 from idlelib import autocomplete_w from idlelib.config import idleConf from idlelib.hyperparser import HyperParser +# Tuples passed to open_completions. +# EvalFunc, Complete, WantWin, Mode +FORCE = True, False, True, None # Control-Space. +TAB = False, True, True, None # Tab. +TRY_A = False, False, False, ATTRS # '.' for attributes. +TRY_F = False, False, False, FILES # '/' in quotes for file name. + # This string includes all chars that may be in an identifier. # TODO Update this here and elsewhere. ID_CHARS = string.ascii_letters + string.digits + "_" -SEPS = os.sep -if os.altsep: # e.g. '/' on Windows... - SEPS += os.altsep - +SEPS = f"{os.sep}{os.altsep if os.altsep else ''}" +TRIGGERS = f".{SEPS}" class AutoComplete: def __init__(self, editwin=None): self.editwin = editwin - if editwin is not None: # not in subprocess or test + if editwin is not None: # not in subprocess or no-gui test self.text = editwin.text - self.autocompletewindow = None - # id of delayed call, and the index of the text insert when - # the delayed call was issued. If _delayed_completion_id is - # None, there is no delayed call. - self._delayed_completion_id = None - self._delayed_completion_index = None + self.autocompletewindow = None + # id of delayed call, and the index of the text insert when + # the delayed call was issued. If _delayed_completion_id is + # None, there is no delayed call. + self._delayed_completion_id = None + self._delayed_completion_index = None @classmethod def reload(cls): cls.popupwait = idleConf.GetOption( "extensions", "AutoComplete", "popupwait", type="int", default=0) - def _make_autocomplete_window(self): + def _make_autocomplete_window(self): # Makes mocking easier. return autocomplete_w.AutoCompleteWindow(self.text) def _remove_autocomplete_window(self, event=None): @@ -52,30 +55,12 @@ class AutoComplete: self.autocompletewindow = None def force_open_completions_event(self, event): - """Happens when the user really wants to open a completion list, even - if a function call is needed. - """ - self.open_completions(True, False, True) + "(^space) Open completion list, even if a function call is needed." + self.open_completions(FORCE) return "break" - def try_open_completions_event(self, event): - """Happens when it would be nice to open a completion list, but not - really necessary, for example after a dot, so function - calls won't be made. - """ - lastchar = self.text.get("insert-1c") - if lastchar == ".": - self._open_completions_later(False, False, False, - COMPLETE_ATTRIBUTES) - elif lastchar in SEPS: - self._open_completions_later(False, False, False, - COMPLETE_FILES) - def autocomplete_event(self, event): - """Happens when the user wants to complete his word, and if necessary, - open a completion list after that (if there is more than one - completion) - """ + "(tab) Complete word or open list if multiple options." if hasattr(event, "mc_state") and event.mc_state or\ not self.text.get("insert linestart", "insert").strip(): # A modifier was pressed along with the tab or @@ -85,34 +70,34 @@ class AutoComplete: self.autocompletewindow.complete() return "break" else: - opened = self.open_completions(False, True, True) + opened = self.open_completions(TAB) return "break" if opened else None - def _open_completions_later(self, *args): - self._delayed_completion_index = self.text.index("insert") - if self._delayed_completion_id is not None: - self.text.after_cancel(self._delayed_completion_id) - self._delayed_completion_id = \ - self.text.after(self.popupwait, self._delayed_open_completions, - *args) - - def _delayed_open_completions(self, *args): + def try_open_completions_event(self, event=None): + "(./) Open completion list after pause with no movement." + lastchar = self.text.get("insert-1c") + if lastchar in TRIGGERS: + args = TRY_A if lastchar == "." else TRY_F + self._delayed_completion_index = self.text.index("insert") + if self._delayed_completion_id is not None: + self.text.after_cancel(self._delayed_completion_id) + self._delayed_completion_id = self.text.after( + self.popupwait, self._delayed_open_completions, args) + + def _delayed_open_completions(self, args): + "Call open_completions if index unchanged." self._delayed_completion_id = None if self.text.index("insert") == self._delayed_completion_index: - self.open_completions(*args) + self.open_completions(args) - def open_completions(self, evalfuncs, complete, userWantsWin, mode=None): + def open_completions(self, args): """Find the completions and create the AutoCompleteWindow. Return True if successful (no syntax error or so found). If complete is True, then if there's nothing to complete and no start of completion, won't open completions and return False. If mode is given, will open a completion list only in this mode. - - Action Function Eval Complete WantWin Mode - ^space force_open_completions True, False, True no - . or / try_open_completions False, False, False yes - tab autocomplete False, True, True no """ + evalfuncs, complete, wantwin, mode = args # Cancel another delayed call, if it exists. if self._delayed_completion_id is not None: self.text.after_cancel(self._delayed_completion_id) @@ -121,14 +106,14 @@ class AutoComplete: hp = HyperParser(self.editwin, "insert") curline = self.text.get("insert linestart", "insert") i = j = len(curline) - if hp.is_in_string() and (not mode or mode==COMPLETE_FILES): + if hp.is_in_string() and (not mode or mode==FILES): # Find the beginning of the string. # fetch_completions will look at the file system to determine # whether the string value constitutes an actual file name # XXX could consider raw strings here and unescape the string # value if it's not raw. self._remove_autocomplete_window() - mode = COMPLETE_FILES + mode = FILES # Find last separator or string start while i and curline[i-1] not in "'\"" + SEPS: i -= 1 @@ -138,17 +123,17 @@ class AutoComplete: while i and curline[i-1] not in "'\"": i -= 1 comp_what = curline[i:j] - elif hp.is_in_code() and (not mode or mode==COMPLETE_ATTRIBUTES): + elif hp.is_in_code() and (not mode or mode==ATTRS): self._remove_autocomplete_window() - mode = COMPLETE_ATTRIBUTES + mode = ATTRS while i and (curline[i-1] in ID_CHARS or ord(curline[i-1]) > 127): i -= 1 comp_start = curline[i:j] - if i and curline[i-1] == '.': + if i and curline[i-1] == '.': # Need object with attributes. hp.set_index("insert-%dc" % (len(curline)-(i-1))) comp_what = hp.get_expression() - if not comp_what or \ - (not evalfuncs and comp_what.find('(') != -1): + if (not comp_what or + (not evalfuncs and comp_what.find('(') != -1)): return None else: comp_what = "" @@ -163,7 +148,7 @@ class AutoComplete: self.autocompletewindow = self._make_autocomplete_window() return not self.autocompletewindow.show_window( comp_lists, "insert-%dc" % len(comp_start), - complete, mode, userWantsWin) + complete, mode, wantwin) def fetch_completions(self, what, mode): """Return a pair of lists of completions for something. The first list @@ -185,7 +170,7 @@ class AutoComplete: return rpcclt.remotecall("exec", "get_the_completion_list", (what, mode), {}) else: - if mode == COMPLETE_ATTRIBUTES: + if mode == ATTRS: if what == "": namespace = {**__main__.__builtins__.__dict__, **__main__.__dict__} @@ -207,7 +192,7 @@ class AutoComplete: except: return [], [] - elif mode == COMPLETE_FILES: + elif mode == FILES: if what == "": what = "." try: diff --git a/lib-python/3/idlelib/autocomplete_w.py b/lib-python/3/idlelib/autocomplete_w.py index c249625277..fe7a6be83d 100644 --- a/lib-python/3/idlelib/autocomplete_w.py +++ b/lib-python/3/idlelib/autocomplete_w.py @@ -4,9 +4,9 @@ An auto-completion window for IDLE, used by the autocomplete extension import platform from tkinter import * -from tkinter.ttk import Frame, Scrollbar +from tkinter.ttk import Scrollbar -from idlelib.autocomplete import COMPLETE_FILES, COMPLETE_ATTRIBUTES +from idlelib.autocomplete import FILES, ATTRS from idlelib.multicall import MC_SHIFT HIDE_VIRTUAL_EVENT_NAME = "<<autocompletewindow-hide>>" @@ -17,7 +17,7 @@ KEYPRESS_VIRTUAL_EVENT_NAME = "<<autocompletewindow-keypress>>" # before the default specific IDLE function KEYPRESS_SEQUENCES = ("<Key>", "<Key-BackSpace>", "<Key-Return>", "<Key-Tab>", "<Key-Up>", "<Key-Down>", "<Key-Home>", "<Key-End>", - "<Key-Prior>", "<Key-Next>") + "<Key-Prior>", "<Key-Next>", "<Key-Escape>") KEYRELEASE_VIRTUAL_EVENT_NAME = "<<autocompletewindow-keyrelease>>" KEYRELEASE_SEQUENCE = "<KeyRelease>" LISTUPDATE_SEQUENCE = "<B1-ButtonRelease>" @@ -39,8 +39,7 @@ class AutoCompleteWindow: self.completions = None # A list with more completions, or None self.morecompletions = None - # The completion mode. Either autocomplete.COMPLETE_ATTRIBUTES or - # autocomplete.COMPLETE_FILES + # The completion mode, either autocomplete.ATTRS or .FILES. self.mode = None # The current completion start, on the text box (a string) self.start = None @@ -53,10 +52,12 @@ class AutoCompleteWindow: # (for example, he clicked the list) self.userwantswindow = None # event ids - self.hideid = self.keypressid = self.listupdateid = self.winconfigid \ - = self.keyreleaseid = self.doubleclickid = None + self.hideid = self.keypressid = self.listupdateid = \ + self.winconfigid = self.keyreleaseid = self.doubleclickid = None # Flag set if last keypress was a tab self.lastkey_was_tab = False + # Flag set to avoid recursive <Configure> callback invocations. + self.is_configuring = False def _change_start(self, newstart): min_len = min(len(self.start), len(newstart)) @@ -73,8 +74,8 @@ class AutoCompleteWindow: def _binary_search(self, s): """Find the first index in self.completions where completions[i] is - greater or equal to s, or the last index if there is no such - one.""" + greater or equal to s, or the last index if there is no such. + """ i = 0; j = len(self.completions) while j > i: m = (i + j) // 2 @@ -87,7 +88,8 @@ class AutoCompleteWindow: def _complete_string(self, s): """Assuming that s is the prefix of a string in self.completions, return the longest string which is a prefix of all the strings which - s is a prefix of them. If s is not a prefix of a string, return s.""" + s is a prefix of them. If s is not a prefix of a string, return s. + """ first = self._binary_search(s) if self.completions[first][:len(s)] != s: # There is not even one completion which s is a prefix of. @@ -116,8 +118,10 @@ class AutoCompleteWindow: return first_comp[:i] def _selection_changed(self): - """Should be called when the selection of the Listbox has changed. - Updates the Listbox display and calls _change_start.""" + """Call when the selection of the Listbox has changed. + + Updates the Listbox display and calls _change_start. + """ cursel = int(self.listbox.curselection()[0]) self.listbox.see(cursel) @@ -153,8 +157,10 @@ class AutoCompleteWindow: def show_window(self, comp_lists, index, complete, mode, userWantsWin): """Show the autocomplete list, bind events. - If complete is True, complete the text, and if there is exactly one - matching completion, don't open a list.""" + + If complete is True, complete the text, and if there is exactly + one matching completion, don't open a list. + """ # Handle the start we already have self.completions, self.morecompletions = comp_lists self.mode = mode @@ -219,12 +225,18 @@ class AutoCompleteWindow: self.widget.event_add(KEYRELEASE_VIRTUAL_EVENT_NAME,KEYRELEASE_SEQUENCE) self.listupdateid = listbox.bind(LISTUPDATE_SEQUENCE, self.listselect_event) + self.is_configuring = False self.winconfigid = acw.bind(WINCONFIG_SEQUENCE, self.winconfig_event) self.doubleclickid = listbox.bind(DOUBLECLICK_SEQUENCE, self.doubleclick_event) return None def winconfig_event(self, event): + if self.is_configuring: + # Avoid running on recursive <Configure> callback invocations. + return + + self.is_configuring = True if not self.is_active(): return # Position the completion list window @@ -232,6 +244,7 @@ class AutoCompleteWindow: text.see(self.startindex) x, y, cx, cy = text.bbox(self.startindex) acw = self.autocompletewindow + acw.update() acw_width, acw_height = acw.winfo_width(), acw.winfo_height() text_width, text_height = text.winfo_width(), text.winfo_height() new_x = text.winfo_rootx() + min(x, max(0, text_width - acw_width)) @@ -244,6 +257,7 @@ class AutoCompleteWindow: # place acw above current line new_y -= acw_height acw.wm_geometry("+%d+%d" % (new_x, new_y)) + acw.update_idletasks() if platform.system().startswith('Windows'): # See issue 15786. When on Windows platform, Tk will misbehave @@ -252,6 +266,8 @@ class AutoCompleteWindow: acw.unbind(WINCONFIG_SEQUENCE, self.winconfigid) self.winconfigid = None + self.is_configuring = False + def _hide_event_check(self): if not self.autocompletewindow: return @@ -300,7 +316,7 @@ class AutoCompleteWindow: if keysym != "Tab": self.lastkey_was_tab = False if (len(keysym) == 1 or keysym in ("underscore", "BackSpace") - or (self.mode == COMPLETE_FILES and keysym in + or (self.mode == FILES and keysym in ("period", "minus"))) \ and not (state & ~MC_SHIFT): # Normal editing of text @@ -329,10 +345,10 @@ class AutoCompleteWindow: self.hide_window() return 'break' - elif (self.mode == COMPLETE_ATTRIBUTES and keysym in + elif (self.mode == ATTRS and keysym in ("period", "space", "parenleft", "parenright", "bracketleft", "bracketright")) or \ - (self.mode == COMPLETE_FILES and keysym in + (self.mode == FILES and keysym in ("slash", "backslash", "quotedbl", "apostrophe")) \ and not (state & ~MC_SHIFT): # If start is a prefix of the selection, but is not '' when @@ -340,7 +356,7 @@ class AutoCompleteWindow: # selected completion. Anyway, close the list. cursel = int(self.listbox.curselection()[0]) if self.completions[cursel][:len(self.start)] == self.start \ - and (self.mode == COMPLETE_ATTRIBUTES or self.start): + and (self.mode == ATTRS or self.start): self._change_start(self.completions[cursel]) self.hide_window() return None diff --git a/lib-python/3/idlelib/browser.py b/lib-python/3/idlelib/browser.py index e5b0bc53c6..3c3a53a659 100644 --- a/lib-python/3/idlelib/browser.py +++ b/lib-python/3/idlelib/browser.py @@ -29,7 +29,7 @@ def transform_children(child_dict, modname=None): The dictionary maps names to pyclbr information objects. Filter out imported objects. Augment class names with bases. - The insertion order of the dictonary is assumed to have been in line + The insertion order of the dictionary is assumed to have been in line number order, so sorting is not necessary. The current tree only calls this once per child_dict as it saves diff --git a/lib-python/3/idlelib/calltip.py b/lib-python/3/idlelib/calltip.py index a3dda2678b..d4092c7847 100644 --- a/lib-python/3/idlelib/calltip.py +++ b/lib-python/3/idlelib/calltip.py @@ -33,7 +33,7 @@ class Calltip: # See __init__ for usage return calltip_w.CalltipWindow(self.text) - def _remove_calltip_window(self, event=None): + def remove_calltip_window(self, event=None): if self.active_calltip: self.active_calltip.hidetip() self.active_calltip = None @@ -55,7 +55,7 @@ class Calltip: self.open_calltip(False) def open_calltip(self, evalfuncs): - self._remove_calltip_window() + self.remove_calltip_window() hp = HyperParser(self.editwin, "insert") sur_paren = hp.get_surrounding_brackets('(') @@ -129,20 +129,22 @@ def get_argspec(ob): empty line or _MAX_LINES. For builtins, this typically includes the arguments in addition to the return value. ''' - argspec = default = "" + # Determine function object fob to inspect. try: ob_call = ob.__call__ - except BaseException: - return default - + except BaseException: # Buggy user object could raise anything. + return '' # No popup for non-callables. fob = ob_call if isinstance(ob_call, types.MethodType) else ob + # Initialize argspec and wrap it to get lines. try: argspec = str(inspect.signature(fob)) - except ValueError as err: + except Exception as err: msg = str(err) if msg.startswith(_invalid_method): return _invalid_method + else: + argspec = '' if '/' in argspec and len(argspec) < _MAX_COLS - len(_argument_positional): # Add explanation TODO remove after 3.7, before 3.9. @@ -154,6 +156,7 @@ def get_argspec(ob): lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) if len(argspec) > _MAX_COLS else [argspec] if argspec else []) + # Augment lines from docstring, if any, and join to get argspec. if isinstance(ob_call, types.MethodType): doc = ob_call.__doc__ else: @@ -167,9 +170,8 @@ def get_argspec(ob): line = line[: _MAX_COLS - 3] + '...' lines.append(line) argspec = '\n'.join(lines) - if not argspec: - argspec = _default_callable_argspec - return argspec + + return argspec or _default_callable_argspec if __name__ == '__main__': diff --git a/lib-python/3/idlelib/codecontext.py b/lib-python/3/idlelib/codecontext.py index 2aed76de7f..989b30e599 100644 --- a/lib-python/3/idlelib/codecontext.py +++ b/lib-python/3/idlelib/codecontext.py @@ -7,20 +7,17 @@ the lines which contain the block opening keywords, e.g. 'if', for the enclosing block. The number of hint lines is determined by the maxlines variable in the codecontext section of config-extensions.def. Lines which do not open blocks are not shown in the context hints pane. - """ import re from sys import maxsize as INFINITY import tkinter -from tkinter.constants import TOP, X, SUNKEN +from tkinter.constants import NSEW, SUNKEN from idlelib.config import idleConf -BLOCKOPENERS = {"class", "def", "elif", "else", "except", "finally", "for", - "if", "try", "while", "with", "async"} -UPDATEINTERVAL = 100 # millisec -CONFIGUPDATEINTERVAL = 1000 # millisec +BLOCKOPENERS = {'class', 'def', 'if', 'elif', 'else', 'while', 'for', + 'try', 'except', 'finally', 'with', 'async'} def get_spaces_firstword(codeline, c=re.compile(r"^(\s*)(\w*)")): @@ -44,13 +41,13 @@ def get_line_info(codeline): class CodeContext: "Display block context above the edit window." + UPDATEINTERVAL = 100 # millisec def __init__(self, editwin): """Initialize settings for context block. editwin is the Editor window for the context block. self.text is the editor window text widget. - self.textfont is the editor window font. self.context displays the code context text above the editor text. Initially None, it is toggled via <<toggle-code-context>>. @@ -65,29 +62,30 @@ class CodeContext: """ self.editwin = editwin self.text = editwin.text - self.textfont = self.text["font"] - self.contextcolors = CodeContext.colors + self._reset() + + def _reset(self): self.context = None + self.cell00 = None + self.t1 = None self.topvisible = 1 self.info = [(0, -1, "", False)] - # Start two update cycles, one for context lines, one for font changes. - self.t1 = self.text.after(UPDATEINTERVAL, self.timer_event) - self.t2 = self.text.after(CONFIGUPDATEINTERVAL, self.config_timer_event) @classmethod def reload(cls): "Load class variables from config." cls.context_depth = idleConf.GetOption("extensions", "CodeContext", - "maxlines", type="int", default=15) - cls.colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'context') + "maxlines", type="int", + default=15) def __del__(self): "Cancel scheduled events." - try: - self.text.after_cancel(self.t1) - self.text.after_cancel(self.t2) - except: - pass + if self.t1 is not None: + try: + self.text.after_cancel(self.t1) + except tkinter.TclError: # pragma: no cover + pass + self.t1 = None def toggle_code_context_event(self, event=None): """Toggle code context display. @@ -96,7 +94,7 @@ class CodeContext: window text (toggle on). If it does exist, destroy it (toggle off). Return 'break' to complete the processing of the binding. """ - if not self.context: + if self.context is None: # Calculate the border width and horizontal padding required to # align the context with the text in the main Text widget. # @@ -107,25 +105,39 @@ class CodeContext: padx = 0 border = 0 for widget in widgets: - padx += widget.tk.getint(widget.pack_info()['padx']) + info = (widget.grid_info() + if widget is self.editwin.text + else widget.pack_info()) + padx += widget.tk.getint(info['padx']) padx += widget.tk.getint(widget.cget('padx')) border += widget.tk.getint(widget.cget('border')) - self.context = tkinter.Text( - self.editwin.top, font=self.textfont, - bg=self.contextcolors['background'], - fg=self.contextcolors['foreground'], - height=1, - width=1, # Don't request more than we get. - padx=padx, border=border, relief=SUNKEN, state='disabled') - self.context.bind('<ButtonRelease-1>', self.jumptoline) - # Pack the context widget before and above the text_frame widget, - # thus ensuring that it will appear directly above text_frame. - self.context.pack(side=TOP, fill=X, expand=False, - before=self.editwin.text_frame) + context = self.context = tkinter.Text( + self.editwin.text_frame, + height=1, + width=1, # Don't request more than we get. + highlightthickness=0, + padx=padx, border=border, relief=SUNKEN, state='disabled') + self.update_font() + self.update_highlight_colors() + context.bind('<ButtonRelease-1>', self.jumptoline) + # Get the current context and initiate the recurring update event. + self.timer_event() + # Grid the context widget above the text widget. + context.grid(row=0, column=1, sticky=NSEW) + + line_number_colors = idleConf.GetHighlight(idleConf.CurrentTheme(), + 'linenumber') + self.cell00 = tkinter.Frame(self.editwin.text_frame, + bg=line_number_colors['background']) + self.cell00.grid(row=0, column=0, sticky=NSEW) menu_status = 'Hide' else: self.context.destroy() self.context = None + self.cell00.destroy() + self.cell00 = None + self.text.after_cancel(self.t1) + self._reset() menu_status = 'Show' self.editwin.update_menu_label(menu='options', index='* Code Context', label=f'{menu_status} Code Context') @@ -169,7 +181,7 @@ class CodeContext: be retrieved and the context area will be updated with the code, up to the number of maxlines. """ - new_topvisible = int(self.text.index("@0,0").split('.')[0]) + new_topvisible = self.editwin.getlineno("@0,0") if self.topvisible == new_topvisible: # Haven't scrolled. return if self.topvisible < new_topvisible: # Scroll down. @@ -202,36 +214,47 @@ class CodeContext: self.context['state'] = 'disabled' def jumptoline(self, event=None): - "Show clicked context line at top of editor." - lines = len(self.info) - if lines == 1: # No context lines are showing. - newtop = 1 - else: - # Line number clicked. - contextline = int(float(self.context.index('insert'))) - # Lines not displayed due to maxlines. - offset = max(1, lines - self.context_depth) - 1 - newtop = self.info[offset + contextline][0] - self.text.yview(f'{newtop}.0') - self.update_code_context() + """ Show clicked context line at top of editor. + + If a selection was made, don't jump; allow copying. + If no visible context, show the top line of the file. + """ + try: + self.context.index("sel.first") + except tkinter.TclError: + lines = len(self.info) + if lines == 1: # No context lines are showing. + newtop = 1 + else: + # Line number clicked. + contextline = int(float(self.context.index('insert'))) + # Lines not displayed due to maxlines. + offset = max(1, lines - self.context_depth) - 1 + newtop = self.info[offset + contextline][0] + self.text.yview(f'{newtop}.0') + self.update_code_context() def timer_event(self): "Event on editor text widget triggered every UPDATEINTERVAL ms." - if self.context: + if self.context is not None: self.update_code_context() - self.t1 = self.text.after(UPDATEINTERVAL, self.timer_event) - - def config_timer_event(self): - "Event on editor text widget triggered every CONFIGUPDATEINTERVAL ms." - newtextfont = self.text["font"] - if (self.context and (newtextfont != self.textfont or - CodeContext.colors != self.contextcolors)): - self.textfont = newtextfont - self.contextcolors = CodeContext.colors - self.context["font"] = self.textfont - self.context['background'] = self.contextcolors['background'] - self.context['foreground'] = self.contextcolors['foreground'] - self.t2 = self.text.after(CONFIGUPDATEINTERVAL, self.config_timer_event) + self.t1 = self.text.after(self.UPDATEINTERVAL, self.timer_event) + + def update_font(self): + if self.context is not None: + font = idleConf.GetFont(self.text, 'main', 'EditorWindow') + self.context['font'] = font + + def update_highlight_colors(self): + if self.context is not None: + colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'context') + self.context['background'] = colors['background'] + self.context['foreground'] = colors['foreground'] + + if self.cell00 is not None: + line_number_colors = idleConf.GetHighlight(idleConf.CurrentTheme(), + 'linenumber') + self.cell00.config(bg=line_number_colors['background']) CodeContext.reload() diff --git a/lib-python/3/idlelib/config-highlight.def b/lib-python/3/idlelib/config-highlight.def index aaa2b57a7f..a7b0433831 100644 --- a/lib-python/3/idlelib/config-highlight.def +++ b/lib-python/3/idlelib/config-highlight.def @@ -22,6 +22,10 @@ hit-foreground= #ffffff hit-background= #000000 error-foreground= #000000 error-background= #ff7777 +context-foreground= #000000 +context-background= lightgray +linenumber-foreground= gray +linenumber-background= #ffffff #cursor (only foreground can be set, restart IDLE) cursor-foreground= black #shell window @@ -31,8 +35,6 @@ stderr-foreground= red stderr-background= #ffffff console-foreground= #770000 console-background= #ffffff -context-foreground= #000000 -context-background= lightgray [IDLE New] normal-foreground= #000000 @@ -55,6 +57,10 @@ hit-foreground= #ffffff hit-background= #000000 error-foreground= #000000 error-background= #ff7777 +context-foreground= #000000 +context-background= lightgray +linenumber-foreground= gray +linenumber-background= #ffffff #cursor (only foreground can be set, restart IDLE) cursor-foreground= black #shell window @@ -64,8 +70,6 @@ stderr-foreground= red stderr-background= #ffffff console-foreground= #770000 console-background= #ffffff -context-foreground= #000000 -context-background= lightgray [IDLE Dark] comment-foreground = #dd0000 @@ -97,3 +101,5 @@ comment-background = #002240 break-foreground = #FFFFFF context-foreground= #ffffff context-background= #454545 +linenumber-foreground= gray +linenumber-background= #002240 diff --git a/lib-python/3/idlelib/config-main.def b/lib-python/3/idlelib/config-main.def index 06e3c5adb0..28ae94161d 100644 --- a/lib-python/3/idlelib/config-main.def +++ b/lib-python/3/idlelib/config-main.def @@ -36,7 +36,7 @@ # Additional help sources are listed in the [HelpFiles] section below # and should be viewable by a web browser (or the Windows Help viewer in # the case of .chm files). These sources will be listed on the Help -# menu. The pattern, and two examples, are +# menu. The pattern, and two examples, are: # # <sequence_number = menu item;/path/to/help/source> # 1 = IDLE;C:/Programs/Python36/Lib/idlelib/help.html @@ -46,7 +46,7 @@ # platform specific because of path separators, drive specs etc. # # The default files should not be edited except to add new sections to -# config-extensions.def for added extensions . The user files should be +# config-extensions.def for added extensions. The user files should be # modified through the Settings dialog. [General] @@ -59,12 +59,14 @@ delete-exitfunc= 1 [EditorWindow] width= 80 height= 40 +cursor-blink= 1 font= TkFixedFont # For TkFixedFont, the actual size and boldness are obtained from tk # and override 10 and 0. See idlelib.config.IdleConf.GetFont font-size= 10 font-bold= 0 encoding= none +line-numbers-default= 0 [PyShell] auto-squeeze-min-lines= 50 diff --git a/lib-python/3/idlelib/config.py b/lib-python/3/idlelib/config.py index 2233dacd66..04444a3bf2 100644 --- a/lib-python/3/idlelib/config.py +++ b/lib-python/3/idlelib/config.py @@ -123,20 +123,14 @@ class IdleUserConfParser(IdleConfParser): self.RemoveEmptySections() return not self.sections() - def RemoveFile(self): - "Remove user config file self.file from disk if it exists." - if os.path.exists(self.file): - os.remove(self.file) - def Save(self): """Update user configuration file. If self not empty after removing empty sections, write the file to disk. Otherwise, remove the file from disk if it exists. - """ fname = self.file - if fname: + if fname and fname[0] != '#': if not self.IsEmpty(): try: cfgFile = open(fname, 'w') @@ -145,8 +139,8 @@ class IdleUserConfParser(IdleConfParser): cfgFile = open(fname, 'w') with cfgFile: self.write(cfgFile) - else: - self.RemoveFile() + elif os.path.exists(self.file): + os.remove(self.file) class IdleConf: """Hold config parsers for all idle config files in singleton instance. @@ -164,6 +158,8 @@ class IdleConf: self.defaultCfg = {} self.userCfg = {} self.cfg = {} # TODO use to select userCfg vs defaultCfg + # self.blink_off_time = <first editor text>['insertofftime'] + # See https:/bugs.python.org/issue4630, msg356516. if not _utest: self.CreateConfigHandlers() @@ -171,24 +167,13 @@ class IdleConf: def CreateConfigHandlers(self): "Populate default and user config parser dictionaries." - #build idle install path - if __name__ != '__main__': # we were imported - idleDir = os.path.dirname(__file__) - else: # we were exec'ed (for testing only) - idleDir = os.path.abspath(sys.path[0]) - self.userdir = userDir = self.GetUserCfgDir() - - defCfgFiles = {} - usrCfgFiles = {} - # TODO eliminate these temporaries by combining loops - for cfgType in self.config_types: #build config file names - defCfgFiles[cfgType] = os.path.join( - idleDir, 'config-' + cfgType + '.def') - usrCfgFiles[cfgType] = os.path.join( - userDir, 'config-' + cfgType + '.cfg') - for cfgType in self.config_types: #create config parsers - self.defaultCfg[cfgType] = IdleConfParser(defCfgFiles[cfgType]) - self.userCfg[cfgType] = IdleUserConfParser(usrCfgFiles[cfgType]) + idledir = os.path.dirname(__file__) + self.userdir = userdir = '' if idlelib.testing else self.GetUserCfgDir() + for cfg_type in self.config_types: + self.defaultCfg[cfg_type] = IdleConfParser( + os.path.join(idledir, f'config-{cfg_type}.def')) + self.userCfg[cfg_type] = IdleUserConfParser( + os.path.join(userdir or '#', f'config-{cfg_type}.cfg')) def GetUserCfgDir(self): """Return a filesystem directory for storing user config files. @@ -199,12 +184,13 @@ class IdleConf: userDir = os.path.expanduser('~') if userDir != '~': # expanduser() found user home dir if not os.path.exists(userDir): - warn = ('\n Warning: os.path.expanduser("~") points to\n ' + - userDir + ',\n but the path does not exist.') - try: - print(warn, file=sys.stderr) - except OSError: - pass + if not idlelib.testing: + warn = ('\n Warning: os.path.expanduser("~") points to\n ' + + userDir + ',\n but the path does not exist.') + try: + print(warn, file=sys.stderr) + except OSError: + pass userDir = '~' if userDir == "~": # still no path to home! # traditionally IDLE has defaulted to os.getcwd(), is this adequate? @@ -214,10 +200,13 @@ class IdleConf: try: os.mkdir(userDir) except OSError: - warn = ('\n Warning: unable to create user config directory\n' + - userDir + '\n Check path and permissions.\n Exiting!\n') if not idlelib.testing: - print(warn, file=sys.stderr) + warn = ('\n Warning: unable to create user config directory\n' + + userDir + '\n Check path and permissions.\n Exiting!\n') + try: + print(warn, file=sys.stderr) + except OSError: + pass raise SystemExit # TODO continue without userDIr instead of exit return userDir @@ -336,6 +325,10 @@ class IdleConf: 'hit-background':'#000000', 'error-foreground':'#ffffff', 'error-background':'#000000', + 'context-foreground':'#000000', + 'context-background':'#ffffff', + 'linenumber-foreground':'#000000', + 'linenumber-background':'#ffffff', #cursor (only foreground can be set) 'cursor-foreground':'#000000', #shell window @@ -345,11 +338,11 @@ class IdleConf: 'stderr-background':'#ffffff', 'console-foreground':'#000000', 'console-background':'#ffffff', - 'context-foreground':'#000000', - 'context-background':'#ffffff', } for element in theme: - if not cfgParser.has_option(themeName, element): + if not (cfgParser.has_option(themeName, element) or + # Skip warning for new elements. + element.startswith(('context-', 'linenumber-'))): # Print warning that will return a default color warning = ('\n Warning: config.IdleConf.GetThemeDict' ' -\n problem retrieving theme element %r' diff --git a/lib-python/3/idlelib/config_key.py b/lib-python/3/idlelib/config_key.py index 4478323fcc..7510aa9f3d 100644 --- a/lib-python/3/idlelib/config_key.py +++ b/lib-python/3/idlelib/config_key.py @@ -1,7 +1,7 @@ """ Dialog for building Tkinter accelerator key bindings """ -from tkinter import Toplevel, Listbox, Text, StringVar, TclError +from tkinter import Toplevel, Listbox, StringVar, TclError from tkinter.ttk import Frame, Button, Checkbutton, Entry, Label, Scrollbar from tkinter import messagebox import string diff --git a/lib-python/3/idlelib/configdialog.py b/lib-python/3/idlelib/configdialog.py index 807ff60413..82596498d3 100644 --- a/lib-python/3/idlelib/configdialog.py +++ b/lib-python/3/idlelib/configdialog.py @@ -9,7 +9,9 @@ Note that tab width in IDLE is currently fixed at eight due to Tk issues. Refer to comments in EditorWindow autoindent code for details. """ -from tkinter import (Toplevel, Listbox, Text, Scale, Canvas, +import re + +from tkinter import (Toplevel, Listbox, Scale, Canvas, StringVar, BooleanVar, IntVar, TRUE, FALSE, TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, NONE, BOTH, X, Y, W, E, EW, NS, NSEW, NW, @@ -29,8 +31,9 @@ from idlelib.textview import view_text from idlelib.autocomplete import AutoComplete from idlelib.codecontext import CodeContext from idlelib.parenmatch import ParenMatch -from idlelib.paragraph import FormatParagraph +from idlelib.format import FormatParagraph from idlelib.squeezer import Squeezer +from idlelib.textview import ScrollableTextFrame changes = ConfigChanges() # Reload changed options in the following classes. @@ -146,17 +149,19 @@ class ConfigDialog(Toplevel): else: padding_args = {'padding': (6, 3)} outer = Frame(self, padding=2) - buttons = Frame(outer, padding=2) + buttons_frame = Frame(outer, padding=2) + self.buttons = {} for txt, cmd in ( ('Ok', self.ok), ('Apply', self.apply), ('Cancel', self.cancel), ('Help', self.help)): - Button(buttons, text=txt, command=cmd, takefocus=FALSE, - **padding_args).pack(side=LEFT, padx=5) + self.buttons[txt] = Button(buttons_frame, text=txt, command=cmd, + takefocus=FALSE, **padding_args) + self.buttons[txt].pack(side=LEFT, padx=5) # Add space above buttons. Frame(outer, height=2, borderwidth=0).pack(side=TOP) - buttons.pack(side=BOTTOM) + buttons_frame.pack(side=BOTTOM) return outer def ok(self): @@ -188,6 +193,7 @@ class ConfigDialog(Toplevel): Methods: destroy: inherited """ + changes.clear() self.destroy() def destroy(self): @@ -201,13 +207,12 @@ class ConfigDialog(Toplevel): Attributes accessed: note - Methods: view_text: Method from textview module. """ page = self.note.tab(self.note.select(), option='text').strip() view_text(self, title='Help for IDLE preferences', - text=help_common+help_pages.get(page, '')) + contents=help_common+help_pages.get(page, '')) def deactivate_current_config(self): """Remove current key bindings. @@ -233,6 +238,7 @@ class ConfigDialog(Toplevel): instance.set_notabs_indentwidth() instance.ApplyKeybindings() instance.reset_help_menu_entries() + instance.update_cursor_blink() for klass in reloadables: klass.reload() @@ -554,7 +560,9 @@ class FontPage(Frame): frame_font_param, variable=self.font_bold, onvalue=1, offvalue=0, text='Bold') # frame_sample. - self.font_sample = Text(frame_sample, width=20, height=20) + font_sample_frame = ScrollableTextFrame(frame_sample) + self.font_sample = font_sample_frame.text + self.font_sample.config(wrap=NONE, width=1, height=1) self.font_sample.insert(END, font_sample_text) # frame_indent. indent_title = Label( @@ -566,8 +574,9 @@ class FontPage(Frame): # Grid and pack widgets: self.columnconfigure(1, weight=1) + self.rowconfigure(2, weight=1) frame_font.grid(row=0, column=0, padx=5, pady=5) - frame_sample.grid(row=0, column=1, rowspan=2, padx=5, pady=5, + frame_sample.grid(row=0, column=1, rowspan=3, padx=5, pady=5, sticky='nsew') frame_indent.grid(row=1, column=0, padx=5, pady=5, sticky='ew') # frame_font. @@ -580,7 +589,7 @@ class FontPage(Frame): self.sizelist.pack(side=LEFT, anchor=W) self.bold_toggle.pack(side=LEFT, anchor=W, padx=20) # frame_sample. - self.font_sample.pack(expand=TRUE, fill=BOTH) + font_sample_frame.pack(expand=TRUE, fill=BOTH) # frame_indent. indent_title.pack(side=TOP, anchor=W, padx=5) self.indent_scale.pack(side=TOP, padx=5, fill=X) @@ -597,9 +606,8 @@ class FontPage(Frame): font_size = configured_font[1] font_bold = configured_font[2]=='bold' - # Set editor font selection list and font_name. - fonts = list(tkFont.families(self)) - fonts.sort() + # Set sorted no-duplicate editor font selection list and font_name. + fonts = sorted(set(tkFont.families(self))) for font in fonts: self.fontlist.insert(END, font) self.font_name.set(font_name) @@ -802,7 +810,7 @@ class HighPage(Frame): (*)theme_message: Label """ self.theme_elements = { - 'Normal Text': ('normal', '00'), + 'Normal Code or Text': ('normal', '00'), 'Code Context': ('context', '01'), 'Python Keywords': ('keyword', '02'), 'Python Definitions': ('definition', '03'), @@ -813,10 +821,11 @@ class HighPage(Frame): 'Found Text': ('hit', '08'), 'Cursor': ('cursor', '09'), 'Editor Breakpoint': ('break', '10'), - 'Shell Normal Text': ('console', '11'), - 'Shell Error Text': ('error', '12'), - 'Shell Stdout Text': ('stdout', '13'), - 'Shell Stderr Text': ('stderr', '14'), + 'Shell Prompt': ('console', '11'), + 'Error Text': ('error', '12'), + 'Shell User Output': ('stdout', '13'), + 'Shell User Exception': ('stderr', '14'), + 'Line Number': ('linenumber', '16'), } self.builtin_name = tracers.add( StringVar(self), self.var_changed_builtin_name) @@ -837,33 +846,40 @@ class HighPage(Frame): frame_theme = LabelFrame(self, borderwidth=2, relief=GROOVE, text=' Highlighting Theme ') # frame_custom. - text = self.highlight_sample = Text( - frame_custom, relief=SOLID, borderwidth=1, - font=('courier', 12, ''), cursor='hand2', width=21, height=13, + sample_frame = ScrollableTextFrame( + frame_custom, relief=SOLID, borderwidth=1) + text = self.highlight_sample = sample_frame.text + text.configure( + font=('courier', 12, ''), cursor='hand2', width=1, height=1, takefocus=FALSE, highlightthickness=0, wrap=NONE) + # Prevent perhaps invisible selection of word or slice. text.bind('<Double-Button-1>', lambda e: 'break') text.bind('<B1-Motion>', lambda e: 'break') - text_and_tags=( - ('\n', 'normal'), - ('#you can click here', 'comment'), ('\n', 'normal'), - ('#to choose items', 'comment'), ('\n', 'normal'), - ('code context section', 'context'), ('\n\n', 'normal'), + string_tags=( + ('# Click selects item.', 'comment'), ('\n', 'normal'), + ('code context section', 'context'), ('\n', 'normal'), + ('| cursor', 'cursor'), ('\n', 'normal'), ('def', 'keyword'), (' ', 'normal'), ('func', 'definition'), ('(param):\n ', 'normal'), - ('"""string"""', 'string'), ('\n var0 = ', 'normal'), + ('"Return None."', 'string'), ('\n var0 = ', 'normal'), ("'string'", 'string'), ('\n var1 = ', 'normal'), ("'selected'", 'hilite'), ('\n var2 = ', 'normal'), ("'found'", 'hit'), ('\n var3 = ', 'normal'), ('list', 'builtin'), ('(', 'normal'), ('None', 'keyword'), (')\n', 'normal'), (' breakpoint("line")', 'break'), ('\n\n', 'normal'), - (' error ', 'error'), (' ', 'normal'), - ('cursor |', 'cursor'), ('\n ', 'normal'), - ('shell', 'console'), (' ', 'normal'), - ('stdout', 'stdout'), (' ', 'normal'), - ('stderr', 'stderr'), ('\n\n', 'normal')) - for texttag in text_and_tags: - text.insert(END, texttag[0], texttag[1]) + ('>>>', 'console'), (' 3.14**2\n', 'normal'), + ('9.8596', 'stdout'), ('\n', 'normal'), + ('>>>', 'console'), (' pri ', 'normal'), + ('n', 'error'), ('t(\n', 'normal'), + ('SyntaxError', 'stderr'), ('\n', 'normal')) + for string, tag in string_tags: + text.insert(END, string, tag) + n_lines = len(text.get('1.0', END).splitlines()) + for lineno in range(1, n_lines): + text.insert(f'{lineno}.0', + f'{lineno:{len(str(n_lines))}d} ', + 'linenumber') for element in self.theme_elements: def tem(event, elem=element): # event.widget.winfo_top_level().highlight_target.set(elem) @@ -912,9 +928,9 @@ class HighPage(Frame): frame_custom.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) frame_theme.pack(side=TOP, padx=5, pady=5, fill=X) # frame_custom. - self.frame_color_set.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=X) + self.frame_color_set.pack(side=TOP, padx=5, pady=5, fill=X) frame_fg_bg_toggle.pack(side=TOP, padx=5, pady=0) - self.highlight_sample.pack( + sample_frame.pack( side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) self.button_set_color.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=4) self.targetlist.pack(side=TOP, expand=TRUE, fill=X, padx=8, pady=3) @@ -1269,8 +1285,7 @@ class HighPage(Frame): theme_name - string, the name of the new theme theme - dictionary containing the new theme """ - if not idleConf.userCfg['highlight'].has_section(theme_name): - idleConf.userCfg['highlight'].add_section(theme_name) + idleConf.userCfg['highlight'].AddSection(theme_name) for element in theme: value = theme[element] idleConf.userCfg['highlight'].SetOption(theme_name, element, value) @@ -1715,8 +1730,7 @@ class KeysPage(Frame): keyset_name - string, the name of the new key set keyset - dictionary containing the new keybindings """ - if not idleConf.userCfg['keys'].has_section(keyset_name): - idleConf.userCfg['keys'].add_section(keyset_name) + idleConf.userCfg['keys'].AddSection(keyset_name) for event in keyset: value = keyset[event] idleConf.userCfg['keys'].SetOption(keyset_name, event, value) @@ -1764,14 +1778,23 @@ class GenPage(Frame): def __init__(self, master): super().__init__(master) + + self.init_validators() self.create_page_general() self.load_general_cfg() + def init_validators(self): + digits_or_empty_re = re.compile(r'[0-9]*') + def is_digits_or_empty(s): + "Return 's is blank or contains only digits'" + return digits_or_empty_re.fullmatch(s) is not None + self.digits_only = (self.register(is_digits_or_empty), '%P',) + def create_page_general(self): """Return frame of widgets for General tab. Enable users to provisionally change general options. Function - load_general_cfg intializes tk variables and helplist using + load_general_cfg initializes tk variables and helplist using idleConf. Radiobuttons startup_shell_on and startup_editor_on set var startup_edit. Radiobuttons save_ask_on and save_auto_on set var autosave. Entry boxes win_width_int and win_height_int @@ -1798,6 +1821,9 @@ class GenPage(Frame): (*)win_width_int: Entry - win_width win_height_title: Label (*)win_height_int: Entry - win_height + frame_cursor_blink: Frame + cursor_blink_title: Label + (*)cursor_blink_bool: Checkbutton - cursor_blink frame_autocomplete: Frame auto_wait_title: Label (*)auto_wait_int: Entry - autocomplete_wait @@ -1816,6 +1842,9 @@ class GenPage(Frame): frame_format: Frame format_width_title: Label (*)format_width_int: Entry - format_width + frame_line_numbers_default: Frame + line_numbers_default_title: Label + (*)line_numbers_default_bool: Checkbutton - line_numbers_default frame_context: Frame context_title: Label (*)context_int: Entry - context_lines @@ -1839,6 +1868,8 @@ class GenPage(Frame): StringVar(self), ('main', 'EditorWindow', 'width')) self.win_height = tracers.add( StringVar(self), ('main', 'EditorWindow', 'height')) + self.cursor_blink = tracers.add( + BooleanVar(self), ('main', 'EditorWindow', 'cursor-blink')) self.autocomplete_wait = tracers.add( StringVar(self), ('extensions', 'AutoComplete', 'popupwait')) self.paren_style = tracers.add( @@ -1855,6 +1886,9 @@ class GenPage(Frame): IntVar(self), ('main', 'General', 'autosave')) self.format_width = tracers.add( StringVar(self), ('extensions', 'FormatParagraph', 'max-width')) + self.line_numbers_default = tracers.add( + BooleanVar(self), + ('main', 'EditorWindow', 'line-numbers-default')) self.context_lines = tracers.add( StringVar(self), ('extensions', 'CodeContext', 'maxlines')) @@ -1883,16 +1917,28 @@ class GenPage(Frame): frame_win_size, text='Initial Window Size (in characters)') win_width_title = Label(frame_win_size, text='Width') self.win_width_int = Entry( - frame_win_size, textvariable=self.win_width, width=3) + frame_win_size, textvariable=self.win_width, width=3, + validatecommand=self.digits_only, validate='key', + ) win_height_title = Label(frame_win_size, text='Height') self.win_height_int = Entry( - frame_win_size, textvariable=self.win_height, width=3) + frame_win_size, textvariable=self.win_height, width=3, + validatecommand=self.digits_only, validate='key', + ) + + frame_cursor_blink = Frame(frame_window, borderwidth=0) + cursor_blink_title = Label(frame_cursor_blink, text='Cursor Blink') + self.cursor_blink_bool = Checkbutton(frame_cursor_blink, + variable=self.cursor_blink, width=1) frame_autocomplete = Frame(frame_window, borderwidth=0,) auto_wait_title = Label(frame_autocomplete, text='Completions Popup Wait (milliseconds)') self.auto_wait_int = Entry(frame_autocomplete, width=6, - textvariable=self.autocomplete_wait) + textvariable=self.autocomplete_wait, + validatecommand=self.digits_only, + validate='key', + ) frame_paren1 = Frame(frame_window, borderwidth=0) paren_style_title = Label(frame_paren1, text='Paren Match Style') @@ -1922,20 +1968,34 @@ class GenPage(Frame): format_width_title = Label(frame_format, text='Format Paragraph Max Width') self.format_width_int = Entry( - frame_format, textvariable=self.format_width, width=4) + frame_format, textvariable=self.format_width, width=4, + validatecommand=self.digits_only, validate='key', + ) + + frame_line_numbers_default = Frame(frame_editor, borderwidth=0) + line_numbers_default_title = Label( + frame_line_numbers_default, text='Show line numbers in new windows') + self.line_numbers_default_bool = Checkbutton( + frame_line_numbers_default, + variable=self.line_numbers_default, + width=1) frame_context = Frame(frame_editor, borderwidth=0) context_title = Label(frame_context, text='Max Context Lines :') self.context_int = Entry( - frame_context, textvariable=self.context_lines, width=3) + frame_context, textvariable=self.context_lines, width=3, + validatecommand=self.digits_only, validate='key', + ) # Frame_shell. frame_auto_squeeze_min_lines = Frame(frame_shell, borderwidth=0) auto_squeeze_min_lines_title = Label(frame_auto_squeeze_min_lines, text='Auto-Squeeze Min. Lines:') self.auto_squeeze_min_lines_int = Entry( - frame_auto_squeeze_min_lines, width=4, - textvariable=self.auto_squeeze_min_lines) + frame_auto_squeeze_min_lines, width=4, + textvariable=self.auto_squeeze_min_lines, + validatecommand=self.digits_only, validate='key', + ) # frame_help. frame_helplist = Frame(frame_help) @@ -1975,6 +2035,10 @@ class GenPage(Frame): win_height_title.pack(side=RIGHT, anchor=E, pady=5) self.win_width_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) win_width_title.pack(side=RIGHT, anchor=E, pady=5) + # frame_cursor_blink. + frame_cursor_blink.pack(side=TOP, padx=5, pady=0, fill=X) + cursor_blink_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.cursor_blink_bool.pack(side=LEFT, padx=5, pady=5) # frame_autocomplete. frame_autocomplete.pack(side=TOP, padx=5, pady=0, fill=X) auto_wait_title.pack(side=LEFT, anchor=W, padx=5, pady=5) @@ -1997,6 +2061,10 @@ class GenPage(Frame): frame_format.pack(side=TOP, padx=5, pady=0, fill=X) format_width_title.pack(side=LEFT, anchor=W, padx=5, pady=5) self.format_width_int.pack(side=TOP, padx=10, pady=5) + # frame_line_numbers_default. + frame_line_numbers_default.pack(side=TOP, padx=5, pady=0, fill=X) + line_numbers_default_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.line_numbers_default_bool.pack(side=LEFT, padx=5, pady=5) # frame_context. frame_context.pack(side=TOP, padx=5, pady=0, fill=X) context_title.pack(side=LEFT, anchor=W, padx=5, pady=5) @@ -2025,6 +2093,8 @@ class GenPage(Frame): 'main', 'EditorWindow', 'width', type='int')) self.win_height.set(idleConf.GetOption( 'main', 'EditorWindow', 'height', type='int')) + self.cursor_blink.set(idleConf.GetOption( + 'main', 'EditorWindow', 'cursor-blink', type='bool')) self.autocomplete_wait.set(idleConf.GetOption( 'extensions', 'AutoComplete', 'popupwait', type='int')) self.paren_style.set(idleConf.GetOption( @@ -2039,6 +2109,8 @@ class GenPage(Frame): 'main', 'General', 'autosave', default=0, type='bool')) self.format_width.set(idleConf.GetOption( 'extensions', 'FormatParagraph', 'max-width', type='int')) + self.line_numbers_default.set(idleConf.GetOption( + 'main', 'EditorWindow', 'line-numbers-default', type='bool')) self.context_lines.set(idleConf.GetOption( 'extensions', 'CodeContext', 'maxlines', type='int')) diff --git a/lib-python/3/idlelib/editor.py b/lib-python/3/idlelib/editor.py index 606de71a6a..a178eaf93c 100644 --- a/lib-python/3/idlelib/editor.py +++ b/lib-python/3/idlelib/editor.py @@ -2,12 +2,15 @@ import importlib.abc import importlib.util import os import platform +import re import string +import sys import tokenize import traceback import webbrowser from tkinter import * +from tkinter.font import Font from tkinter.ttk import Scrollbar import tkinter.simpledialog as tkSimpleDialog import tkinter.messagebox as tkMessageBox @@ -23,6 +26,7 @@ from idlelib import pyparse from idlelib import query from idlelib import replace from idlelib import search +from idlelib.tree import wheel_event from idlelib import window # The default tab setting for a Text widget, in average-width characters. @@ -53,15 +57,18 @@ class EditorWindow(object): from idlelib.autoexpand import AutoExpand from idlelib.calltip import Calltip from idlelib.codecontext import CodeContext - from idlelib.paragraph import FormatParagraph + from idlelib.sidebar import LineNumbers + from idlelib.format import FormatParagraph, FormatRegion, Indents, Rstrip from idlelib.parenmatch import ParenMatch - from idlelib.rstrip import Rstrip from idlelib.squeezer import Squeezer from idlelib.zoomheight import ZoomHeight filesystemencoding = sys.getfilesystemencoding() # for file names help_url = None + allow_code_context = True + allow_line_numbers = True + def __init__(self, flist=None, filename=None, key=None, root=None): # Delay import: runscript imports pyshell imports EditorWindow. from idlelib.runscript import ScriptBinding @@ -109,20 +116,19 @@ class EditorWindow(object): self.tkinter_vars = {} # keys: Tkinter event names # values: Tkinter variable instances self.top.instance_dict = {} - self.recent_files_path = os.path.join( + self.recent_files_path = idleConf.userdir and os.path.join( idleConf.userdir, 'recent-files.lst') self.prompt_last_line = '' # Override in PyShell self.text_frame = text_frame = Frame(top) self.vbar = vbar = Scrollbar(text_frame, name='vbar') - self.width = idleConf.GetOption('main', 'EditorWindow', - 'width', type='int') + width = idleConf.GetOption('main', 'EditorWindow', 'width', type='int') text_options = { 'name': 'text', 'padx': 5, 'wrap': 'none', 'highlightthickness': 0, - 'width': self.width, + 'width': width, 'tabstyle': 'wordprocessor', # new in 8.5 'height': idleConf.GetOption( 'main', 'EditorWindow', 'height', type='int'), @@ -146,9 +152,11 @@ class EditorWindow(object): else: # Elsewhere, use right-click for popup menus. text.bind("<3>",self.right_menu_event) - text.bind('<MouseWheel>', self.mousescroll) - text.bind('<Button-4>', self.mousescroll) - text.bind('<Button-5>', self.mousescroll) + + text.bind('<MouseWheel>', wheel_event) + text.bind('<Button-4>', wheel_event) + text.bind('<Button-5>', wheel_event) + text.bind('<Configure>', self.handle_winconfig) text.bind("<<cut>>", self.cut) text.bind("<<copy>>", self.copy) text.bind("<<paste>>", self.paste) @@ -170,14 +178,17 @@ class EditorWindow(object): text.bind("<<smart-backspace>>",self.smart_backspace_event) text.bind("<<newline-and-indent>>",self.newline_and_indent_event) text.bind("<<smart-indent>>",self.smart_indent_event) - text.bind("<<indent-region>>",self.indent_region_event) - text.bind("<<dedent-region>>",self.dedent_region_event) - text.bind("<<comment-region>>",self.comment_region_event) - text.bind("<<uncomment-region>>",self.uncomment_region_event) - text.bind("<<tabify-region>>",self.tabify_region_event) - text.bind("<<untabify-region>>",self.untabify_region_event) - text.bind("<<toggle-tabs>>",self.toggle_tabs_event) - text.bind("<<change-indentwidth>>",self.change_indentwidth_event) + self.fregion = fregion = self.FormatRegion(self) + # self.fregion used in smart_indent_event to access indent_region. + text.bind("<<indent-region>>", fregion.indent_region_event) + text.bind("<<dedent-region>>", fregion.dedent_region_event) + text.bind("<<comment-region>>", fregion.comment_region_event) + text.bind("<<uncomment-region>>", fregion.uncomment_region_event) + text.bind("<<tabify-region>>", fregion.tabify_region_event) + text.bind("<<untabify-region>>", fregion.untabify_region_event) + indents = self.Indents(self) + text.bind("<<toggle-tabs>>", indents.toggle_tabs_event) + text.bind("<<change-indentwidth>>", indents.change_indentwidth_event) text.bind("<Left>", self.move_at_edge_if_selection(0)) text.bind("<Right>", self.move_at_edge_if_selection(1)) text.bind("<<del-word-left>>", self.del_word_left) @@ -195,13 +206,16 @@ class EditorWindow(object): text.bind("<<open-turtle-demo>>", self.open_turtle_demo) self.set_status_bar() + text_frame.pack(side=LEFT, fill=BOTH, expand=1) + text_frame.rowconfigure(1, weight=1) + text_frame.columnconfigure(1, weight=1) vbar['command'] = self.handle_yview - vbar.pack(side=RIGHT, fill=Y) + vbar.grid(row=1, column=2, sticky=NSEW) text['yscrollcommand'] = vbar.set text['font'] = idleConf.GetFont(self.root, 'main', 'EditorWindow') - text_frame.pack(side=LEFT, fill=BOTH, expand=1) - text.pack(side=TOP, fill=BOTH, expand=1) + text.grid(row=1, column=1, sticky=NSEW) text.focus_set() + self.set_width() # usetabs true -> literal tab characters are used by indent and # dedent cmds, possibly mixed with spaces if @@ -228,6 +242,12 @@ class EditorWindow(object): self.indentwidth = self.tabwidth self.set_notabs_indentwidth() + # Store the current value of the insertofftime now so we can restore + # it if needed. + if not hasattr(idleConf, 'blink_off_time'): + idleConf.blink_off_time = self.text['insertofftime'] + self.update_cursor_blink() + # When searching backwards for a reliable place to begin parsing, # first start num_context_lines[0] lines back, then # num_context_lines[1] lines back if that didn't work, and so on. @@ -247,6 +267,8 @@ class EditorWindow(object): self.good_load = False self.set_indentation_params(False) self.color = None # initialized below in self.ResetColorizer + self.code_context = None # optionally initialized later below + self.line_numbers = None # optionally initialized later below if filename: if os.path.exists(filename) and not os.path.isdir(filename): if io.loadfile(filename): @@ -306,29 +328,42 @@ class EditorWindow(object): text.bind("<<run-module>>", scriptbinding.run_module_event) text.bind("<<run-custom>>", scriptbinding.run_custom_event) text.bind("<<do-rstrip>>", self.Rstrip(self).do_rstrip) - ctip = self.Calltip(self) + self.ctip = ctip = self.Calltip(self) text.bind("<<try-open-calltip>>", ctip.try_open_calltip_event) #refresh-calltip must come after paren-closed to work right text.bind("<<refresh-calltip>>", ctip.refresh_calltip_event) text.bind("<<force-open-calltip>>", ctip.force_open_calltip_event) text.bind("<<zoom-height>>", self.ZoomHeight(self).zoom_height_event) - text.bind("<<toggle-code-context>>", - self.CodeContext(self).toggle_code_context_event) + if self.allow_code_context: + self.code_context = self.CodeContext(self) + text.bind("<<toggle-code-context>>", + self.code_context.toggle_code_context_event) + else: + self.update_menu_state('options', '*Code Context', 'disabled') + if self.allow_line_numbers: + self.line_numbers = self.LineNumbers(self) + if idleConf.GetOption('main', 'EditorWindow', + 'line-numbers-default', type='bool'): + self.toggle_line_numbers_event() + text.bind("<<toggle-line-numbers>>", self.toggle_line_numbers_event) + else: + self.update_menu_state('options', '*Line Numbers', 'disabled') - def _filename_to_unicode(self, filename): - """Return filename as BMP unicode so displayable in Tk.""" - # Decode bytes to unicode. - if isinstance(filename, bytes): - try: - filename = filename.decode(self.filesystemencoding) - except UnicodeDecodeError: - try: - filename = filename.decode(self.encoding) - except UnicodeDecodeError: - # byte-to-byte conversion - filename = filename.decode('iso8859-1') - # Replace non-BMP char with diamond questionmark. - return re.sub('[\U00010000-\U0010FFFF]', '\ufffd', filename) + def handle_winconfig(self, event=None): + self.set_width() + + def set_width(self): + text = self.text + inner_padding = sum(map(text.tk.getint, [text.cget('border'), + text.cget('padx')])) + pixel_width = text.winfo_width() - 2 * inner_padding + + # Divide the width of the Text widget by the font width, + # which is taken to be the width of '0' (zero). + # http://www.tcl.tk/man/tcl8.6/TkCmd/text.htm#M21 + zero_char_width = \ + Font(text, font=text.cget('font')).measure('0') + self.width = pixel_width // zero_char_width def new_callback(self, event): dirname, basename = self.io.defaultfilename() @@ -461,34 +496,26 @@ class EditorWindow(object): self.text.yview(event, *args) return 'break' - def mousescroll(self, event): - """Handle scrollwheel event. - - For wheel up, event.delta = 120*n on Windows, -1*n on darwin, - where n can be > 1 if one scrolls fast. Flicking the wheel - generates up to maybe 20 events with n up to 10 or more 1. - Macs use wheel down (delta = 1*n) to scroll up, so positive - delta means to scroll up on both systems. - - X-11 sends Control-Button-4 event instead. - """ - up = {EventType.MouseWheel: event.delta > 0, - EventType.Button: event.num == 4} - lines = -5 if up[event.type] else 5 - self.text.yview_scroll(lines, 'units') - return 'break' - rmenu = None def right_menu_event(self, event): - self.text.mark_set("insert", "@%d,%d" % (event.x, event.y)) + text = self.text + newdex = text.index(f'@{event.x},{event.y}') + try: + in_selection = (text.compare('sel.first', '<=', newdex) and + text.compare(newdex, '<=', 'sel.last')) + except TclError: + in_selection = False + if not in_selection: + text.tag_remove("sel", "1.0", "end") + text.mark_set("insert", newdex) if not self.rmenu: self.make_rmenu() rmenu = self.rmenu self.event = event iswin = sys.platform[:3] == 'win' if iswin: - self.text.config(cursor="arrow") + text.config(cursor="arrow") for item in self.rmenu_specs: try: @@ -501,7 +528,6 @@ class EditorWindow(object): state = getattr(self, verify_state)() rmenu.entryconfigure(label, state=state) - rmenu.tk_popup(event.x_root, event.y_root) if iswin: self.text.config(cursor="ibeam") @@ -653,15 +679,16 @@ class EditorWindow(object): def goto_line_event(self, event): text = self.text - lineno = tkSimpleDialog.askinteger("Goto", - "Go to line number:",parent=text) - if lineno is None: - return "break" - if lineno <= 0: - text.bell() - return "break" - text.mark_set("insert", "%d.0" % lineno) - text.see("insert") + lineno = query.Goto( + text, "Go To Line", + "Enter a positive integer\n" + "('big' = end of file):" + ).result + if lineno is not None: + text.tag_remove("sel", "1.0", "end") + text.mark_set("insert", f'{lineno}.0') + text.see("insert") + self.set_line_and_column() return "break" def open_module(self): @@ -773,6 +800,12 @@ class EditorWindow(object): self._addcolorizer() EditorWindow.color_config(self.text) + if self.code_context is not None: + self.code_context.update_highlight_colors() + + if self.line_numbers is not None: + self.line_numbers.update_colors() + IDENTCHARS = string.ascii_letters + string.digits + "_" def colorize_syntax_error(self, text, pos): @@ -786,11 +819,32 @@ class EditorWindow(object): text.mark_set("insert", pos + "+1c") text.see(pos) + def update_cursor_blink(self): + "Update the cursor blink configuration." + cursorblink = idleConf.GetOption( + 'main', 'EditorWindow', 'cursor-blink', type='bool') + if not cursorblink: + self.text['insertofftime'] = 0 + else: + # Restore the original value + self.text['insertofftime'] = idleConf.blink_off_time + def ResetFont(self): "Update the text widgets' font if it is changed" # Called from configdialog.py - self.text['font'] = idleConf.GetFont(self.root, 'main','EditorWindow') + # Update the code context widget first, since its height affects + # the height of the text widget. This avoids double re-rendering. + if self.code_context is not None: + self.code_context.update_font() + # Next, update the line numbers widget, since its width affects + # the width of the text widget. + if self.line_numbers is not None: + self.line_numbers.update_font() + # Finally, update the main text widget. + new_font = idleConf.GetFont(self.root, 'main', 'EditorWindow') + self.text['font'] = new_font + self.set_width() def RemoveKeybindings(self): "Remove the keybindings before they are changed." @@ -881,9 +935,11 @@ class EditorWindow(object): def update_recent_files_list(self, new_file=None): "Load and update the recent files list and menus" + # TODO: move to iomenu. rf_list = [] - if os.path.exists(self.recent_files_path): - with open(self.recent_files_path, 'r', + file_path = self.recent_files_path + if file_path and os.path.exists(file_path): + with open(file_path, 'r', encoding='utf_8', errors='replace') as rf_list_file: rf_list = rf_list_file.readlines() if new_file: @@ -899,29 +955,27 @@ class EditorWindow(object): rf_list = [path for path in rf_list if path not in bad_paths] ulchars = "1234567890ABCDEFGHIJK" rf_list = rf_list[0:len(ulchars)] - try: - with open(self.recent_files_path, 'w', - encoding='utf_8', errors='replace') as rf_file: - rf_file.writelines(rf_list) - except OSError as err: - if not getattr(self.root, "recentfilelist_error_displayed", False): - self.root.recentfilelist_error_displayed = True - tkMessageBox.showwarning(title='IDLE Warning', - message="Cannot update File menu Recent Files list. " - "Your operating system says:\n%s\n" - "Select OK and IDLE will continue without updating." - % self._filename_to_unicode(str(err)), - parent=self.text) + if file_path: + try: + with open(file_path, 'w', + encoding='utf_8', errors='replace') as rf_file: + rf_file.writelines(rf_list) + except OSError as err: + if not getattr(self.root, "recentfiles_message", False): + self.root.recentfiles_message = True + tkMessageBox.showwarning(title='IDLE Warning', + message="Cannot save Recent Files list to disk.\n" + f" {err}\n" + "Select OK to continue.", + parent=self.text) # for each edit window instance, construct the recent files menu for instance in self.top.instance_dict: menu = instance.recent_files_menu menu.delete(0, END) # clear, and rebuild: for i, file_name in enumerate(rf_list): file_name = file_name.rstrip() # zap \n - # make unicode string to display non-ASCII chars correctly - ufile_name = self._filename_to_unicode(file_name) callback = instance.__recent_file_callback(file_name) - menu.add_command(label=ulchars[i] + " " + ufile_name, + menu.add_command(label=ulchars[i] + " " + file_name, command=callback, underline=0) @@ -959,16 +1013,10 @@ class EditorWindow(object): def short_title(self): filename = self.io.filename - if filename: - filename = os.path.basename(filename) - else: - filename = "untitled" - # return unicode string to display non-ASCII chars correctly - return self._filename_to_unicode(filename) + return os.path.basename(filename) if filename else "untitled" def long_title(self): - # return unicode string to display non-ASCII chars correctly - return self._filename_to_unicode(self.io.filename or "") + return self.io.filename or "" def center_insert_event(self, event): self.center() @@ -1016,10 +1064,13 @@ class EditorWindow(object): return self.io.maybesave() def close(self): - reply = self.maybesave() - if str(reply) != "cancel": - self._close() - return reply + try: + reply = self.maybesave() + if str(reply) != "cancel": + self._close() + return reply + except AttributeError: # bpo-35379: close called twice + pass def _close(self): if self.io.filename: @@ -1277,11 +1328,11 @@ class EditorWindow(object): try: if first and last: if index2line(first) != index2line(last): - return self.indent_region_event(event) + return self.fregion.indent_region_event(event) text.delete(first, last) text.mark_set("insert", first) prefix = text.get("insert linestart", "insert") - raw, effective = classifyws(prefix, self.tabwidth) + raw, effective = get_line_indent(prefix, self.tabwidth) if raw == len(prefix): # only whitespace to the left self.reindent_to(effective + self.indentwidth) @@ -1300,38 +1351,51 @@ class EditorWindow(object): text.undo_block_stop() def newline_and_indent_event(self, event): + """Insert a newline and indentation after Enter keypress event. + + Properly position the cursor on the new line based on information + from the current line. This takes into account if the current line + is a shell prompt, is empty, has selected text, contains a block + opener, contains a block closer, is a continuation line, or + is inside a string. + """ text = self.text first, last = self.get_selection_indices() text.undo_block_start() - try: + try: # Close undo block and expose new line in finally clause. if first and last: text.delete(first, last) text.mark_set("insert", first) line = text.get("insert linestart", "insert") + + # Count leading whitespace for indent size. i, n = 0, len(line) while i < n and line[i] in " \t": - i = i+1 + i += 1 if i == n: - # the cursor is in or at leading indentation in a continuation - # line; just inject an empty line at the start + # The cursor is in or at leading indentation in a continuation + # line; just inject an empty line at the start. text.insert("insert linestart", '\n') return "break" indent = line[:i] - # strip whitespace before insert point unless it's in the prompt + + # Strip whitespace before insert point unless it's in the prompt. i = 0 while line and line[-1] in " \t" and line != self.prompt_last_line: line = line[:-1] - i = i+1 + i += 1 if i: text.delete("insert - %d chars" % i, "insert") - # strip whitespace after insert point + + # Strip whitespace after insert point. while text.get("insert") in " \t": text.delete("insert") - # start new line + + # Insert new line. text.insert("insert", '\n') - # adjust indentation for continuations and block - # open/close first need to find the last stmt + # Adjust indentation for continuations and block open/close. + # First need to find the last statement. lno = index2line(text.index('insert')) y = pyparse.Parser(self.indentwidth, self.tabwidth) if not self.prompt_last_line: @@ -1341,7 +1405,7 @@ class EditorWindow(object): rawtext = text.get(startatindex, "insert") y.set_code(rawtext) bod = y.find_good_parse_start( - self._build_char_in_string_func(startatindex)) + self._build_char_in_string_func(startatindex)) if bod is not None or startat == 1: break y.set_lo(bod or 0) @@ -1357,26 +1421,26 @@ class EditorWindow(object): c = y.get_continuation_type() if c != pyparse.C_NONE: - # The current stmt hasn't ended yet. + # The current statement hasn't ended yet. if c == pyparse.C_STRING_FIRST_LINE: - # after the first line of a string; do not indent at all + # After the first line of a string do not indent at all. pass elif c == pyparse.C_STRING_NEXT_LINES: - # inside a string which started before this line; - # just mimic the current indent + # Inside a string which started before this line; + # just mimic the current indent. text.insert("insert", indent) elif c == pyparse.C_BRACKET: - # line up with the first (if any) element of the + # Line up with the first (if any) element of the # last open bracket structure; else indent one # level beyond the indent of the line with the - # last open bracket + # last open bracket. self.reindent_to(y.compute_bracket_indent()) elif c == pyparse.C_BACKSLASH: - # if more than one line in this stmt already, just + # If more than one line in this statement already, just # mimic the current indent; else if initial line # has a start on an assignment stmt, indent to # beyond leftmost =; else to beyond first chunk of - # non-whitespace on initial line + # non-whitespace on initial line. if y.get_num_lines_in_stmt() > 1: text.insert("insert", indent) else: @@ -1385,9 +1449,9 @@ class EditorWindow(object): assert 0, "bogus continuation type %r" % (c,) return "break" - # This line starts a brand new stmt; indent relative to + # This line starts a brand new statement; indent relative to # indentation of initial line of closest preceding - # interesting stmt. + # interesting statement. indent = y.get_base_indent_string() text.insert("insert", indent) if y.is_block_opener(): @@ -1410,86 +1474,6 @@ class EditorWindow(object): return _icis(_startindex + "+%dc" % offset) return inner - def indent_region_event(self, event): - head, tail, chars, lines = self.get_region() - for pos in range(len(lines)): - line = lines[pos] - if line: - raw, effective = classifyws(line, self.tabwidth) - effective = effective + self.indentwidth - lines[pos] = self._make_blanks(effective) + line[raw:] - self.set_region(head, tail, chars, lines) - return "break" - - def dedent_region_event(self, event): - head, tail, chars, lines = self.get_region() - for pos in range(len(lines)): - line = lines[pos] - if line: - raw, effective = classifyws(line, self.tabwidth) - effective = max(effective - self.indentwidth, 0) - lines[pos] = self._make_blanks(effective) + line[raw:] - self.set_region(head, tail, chars, lines) - return "break" - - def comment_region_event(self, event): - head, tail, chars, lines = self.get_region() - for pos in range(len(lines) - 1): - line = lines[pos] - lines[pos] = '##' + line - self.set_region(head, tail, chars, lines) - return "break" - - def uncomment_region_event(self, event): - head, tail, chars, lines = self.get_region() - for pos in range(len(lines)): - line = lines[pos] - if not line: - continue - if line[:2] == '##': - line = line[2:] - elif line[:1] == '#': - line = line[1:] - lines[pos] = line - self.set_region(head, tail, chars, lines) - return "break" - - def tabify_region_event(self, event): - head, tail, chars, lines = self.get_region() - tabwidth = self._asktabwidth() - if tabwidth is None: return - for pos in range(len(lines)): - line = lines[pos] - if line: - raw, effective = classifyws(line, tabwidth) - ntabs, nspaces = divmod(effective, tabwidth) - lines[pos] = '\t' * ntabs + ' ' * nspaces + line[raw:] - self.set_region(head, tail, chars, lines) - return "break" - - def untabify_region_event(self, event): - head, tail, chars, lines = self.get_region() - tabwidth = self._asktabwidth() - if tabwidth is None: return - for pos in range(len(lines)): - lines[pos] = lines[pos].expandtabs(tabwidth) - self.set_region(head, tail, chars, lines) - return "break" - - def toggle_tabs_event(self, event): - if self.askyesno( - "Toggle tabs", - "Turn tabs " + ("on", "off")[self.usetabs] + - "?\nIndent width " + - ("will be", "remains at")[self.usetabs] + " 8." + - "\n Note: a tab is always 8 columns", - parent=self.text): - self.usetabs = not self.usetabs - # Try to prevent inconsistent indentation. - # User must change indent width manually after using tabs. - self.indentwidth = 8 - return "break" - # XXX this isn't bound to anything -- see tabwidth comments ## def change_tabwidth_event(self, event): ## new = self._asktabwidth() @@ -1498,45 +1482,6 @@ class EditorWindow(object): ## self.set_indentation_params(0, guess=0) ## return "break" - def change_indentwidth_event(self, event): - new = self.askinteger( - "Indent width", - "New indent width (2-16)\n(Always use 8 when using tabs)", - parent=self.text, - initialvalue=self.indentwidth, - minvalue=2, - maxvalue=16) - if new and new != self.indentwidth and not self.usetabs: - self.indentwidth = new - return "break" - - def get_region(self): - text = self.text - first, last = self.get_selection_indices() - if first and last: - head = text.index(first + " linestart") - tail = text.index(last + "-1c lineend +1c") - else: - head = text.index("insert linestart") - tail = text.index("insert lineend +1c") - chars = text.get(head, tail) - lines = chars.split("\n") - return head, tail, chars, lines - - def set_region(self, head, tail, chars, lines): - text = self.text - newchars = "\n".join(lines) - if newchars == chars: - text.bell() - return - text.tag_remove("sel", "1.0", "end") - text.mark_set("insert", head) - text.undo_block_start() - text.delete(head, tail) - text.insert(head, newchars) - text.undo_block_stop() - text.tag_add("sel", head, "insert") - # Make string that displays as n leading blanks. def _make_blanks(self, n): @@ -1558,15 +1503,6 @@ class EditorWindow(object): text.insert("insert", self._make_blanks(column)) text.undo_block_stop() - def _asktabwidth(self): - return self.askinteger( - "Tab width", - "Columns per tab? (2-16)", - parent=self.text, - initialvalue=self.indentwidth, - minvalue=2, - maxvalue=16) - # Guess indentwidth from text content. # Return guessed indentwidth. This should not be believed unless # it's in a reasonable range (e.g., it will be 0 if no indented @@ -1575,33 +1511,39 @@ class EditorWindow(object): def guess_indent(self): opener, indented = IndentSearcher(self.text, self.tabwidth).run() if opener and indented: - raw, indentsmall = classifyws(opener, self.tabwidth) - raw, indentlarge = classifyws(indented, self.tabwidth) + raw, indentsmall = get_line_indent(opener, self.tabwidth) + raw, indentlarge = get_line_indent(indented, self.tabwidth) else: indentsmall = indentlarge = 0 return indentlarge - indentsmall + def toggle_line_numbers_event(self, event=None): + if self.line_numbers is None: + return + + if self.line_numbers.is_shown: + self.line_numbers.hide_sidebar() + menu_label = "Show" + else: + self.line_numbers.show_sidebar() + menu_label = "Hide" + self.update_menu_label(menu='options', index='*Line Numbers', + label=f'{menu_label} Line Numbers') + # "line.col" -> line, as an int def index2line(index): return int(float(index)) -# Look at the leading whitespace in s. -# Return pair (# of leading ws characters, -# effective # of leading blanks after expanding -# tabs to width tabwidth) - -def classifyws(s, tabwidth): - raw = effective = 0 - for ch in s: - if ch == ' ': - raw = raw + 1 - effective = effective + 1 - elif ch == '\t': - raw = raw + 1 - effective = (effective // tabwidth + 1) * tabwidth - else: - break - return raw, effective + +_line_indent_re = re.compile(r'[ \t]*') +def get_line_indent(line, tabwidth): + """Return a line's indentation as (# chars, effective # of spaces). + + The effective # of spaces is the length after properly "expanding" + the tabs into spaces, as done by str.expandtabs(tabwidth). + """ + m = _line_indent_re.match(line) + return m.end(), len(m.group().expandtabs(tabwidth)) class IndentSearcher(object): diff --git a/lib-python/3/idlelib/format.py b/lib-python/3/idlelib/format.py new file mode 100644 index 0000000000..4b57a182c9 --- /dev/null +++ b/lib-python/3/idlelib/format.py @@ -0,0 +1,426 @@ +"""Format all or a selected region (line slice) of text. + +Region formatting options: paragraph, comment block, indent, deindent, +comment, uncomment, tabify, and untabify. + +File renamed from paragraph.py with functions added from editor.py. +""" +import re +from tkinter.messagebox import askyesno +from tkinter.simpledialog import askinteger +from idlelib.config import idleConf + + +class FormatParagraph: + """Format a paragraph, comment block, or selection to a max width. + + Does basic, standard text formatting, and also understands Python + comment blocks. Thus, for editing Python source code, this + extension is really only suitable for reformatting these comment + blocks or triple-quoted strings. + + Known problems with comment reformatting: + * If there is a selection marked, and the first line of the + selection is not complete, the block will probably not be detected + as comments, and will have the normal "text formatting" rules + applied. + * If a comment block has leading whitespace that mixes tabs and + spaces, they will not be considered part of the same block. + * Fancy comments, like this bulleted list, aren't handled :-) + """ + def __init__(self, editwin): + self.editwin = editwin + + @classmethod + def reload(cls): + cls.max_width = idleConf.GetOption('extensions', 'FormatParagraph', + 'max-width', type='int', default=72) + + def close(self): + self.editwin = None + + def format_paragraph_event(self, event, limit=None): + """Formats paragraph to a max width specified in idleConf. + + If text is selected, format_paragraph_event will start breaking lines + at the max width, starting from the beginning selection. + + If no text is selected, format_paragraph_event uses the current + cursor location to determine the paragraph (lines of text surrounded + by blank lines) and formats it. + + The length limit parameter is for testing with a known value. + """ + limit = self.max_width if limit is None else limit + text = self.editwin.text + first, last = self.editwin.get_selection_indices() + if first and last: + data = text.get(first, last) + comment_header = get_comment_header(data) + else: + first, last, comment_header, data = \ + find_paragraph(text, text.index("insert")) + if comment_header: + newdata = reformat_comment(data, limit, comment_header) + else: + newdata = reformat_paragraph(data, limit) + text.tag_remove("sel", "1.0", "end") + + if newdata != data: + text.mark_set("insert", first) + text.undo_block_start() + text.delete(first, last) + text.insert(first, newdata) + text.undo_block_stop() + else: + text.mark_set("insert", last) + text.see("insert") + return "break" + + +FormatParagraph.reload() + +def find_paragraph(text, mark): + """Returns the start/stop indices enclosing the paragraph that mark is in. + + Also returns the comment format string, if any, and paragraph of text + between the start/stop indices. + """ + lineno, col = map(int, mark.split(".")) + line = text.get("%d.0" % lineno, "%d.end" % lineno) + + # Look for start of next paragraph if the index passed in is a blank line + while text.compare("%d.0" % lineno, "<", "end") and is_all_white(line): + lineno = lineno + 1 + line = text.get("%d.0" % lineno, "%d.end" % lineno) + first_lineno = lineno + comment_header = get_comment_header(line) + comment_header_len = len(comment_header) + + # Once start line found, search for end of paragraph (a blank line) + while get_comment_header(line)==comment_header and \ + not is_all_white(line[comment_header_len:]): + lineno = lineno + 1 + line = text.get("%d.0" % lineno, "%d.end" % lineno) + last = "%d.0" % lineno + + # Search back to beginning of paragraph (first blank line before) + lineno = first_lineno - 1 + line = text.get("%d.0" % lineno, "%d.end" % lineno) + while lineno > 0 and \ + get_comment_header(line)==comment_header and \ + not is_all_white(line[comment_header_len:]): + lineno = lineno - 1 + line = text.get("%d.0" % lineno, "%d.end" % lineno) + first = "%d.0" % (lineno+1) + + return first, last, comment_header, text.get(first, last) + +# This should perhaps be replaced with textwrap.wrap +def reformat_paragraph(data, limit): + """Return data reformatted to specified width (limit).""" + lines = data.split("\n") + i = 0 + n = len(lines) + while i < n and is_all_white(lines[i]): + i = i+1 + if i >= n: + return data + indent1 = get_indent(lines[i]) + if i+1 < n and not is_all_white(lines[i+1]): + indent2 = get_indent(lines[i+1]) + else: + indent2 = indent1 + new = lines[:i] + partial = indent1 + while i < n and not is_all_white(lines[i]): + # XXX Should take double space after period (etc.) into account + words = re.split(r"(\s+)", lines[i]) + for j in range(0, len(words), 2): + word = words[j] + if not word: + continue # Can happen when line ends in whitespace + if len((partial + word).expandtabs()) > limit and \ + partial != indent1: + new.append(partial.rstrip()) + partial = indent2 + partial = partial + word + " " + if j+1 < len(words) and words[j+1] != " ": + partial = partial + " " + i = i+1 + new.append(partial.rstrip()) + # XXX Should reformat remaining paragraphs as well + new.extend(lines[i:]) + return "\n".join(new) + +def reformat_comment(data, limit, comment_header): + """Return data reformatted to specified width with comment header.""" + + # Remove header from the comment lines + lc = len(comment_header) + data = "\n".join(line[lc:] for line in data.split("\n")) + # Reformat to maxformatwidth chars or a 20 char width, + # whichever is greater. + format_width = max(limit - len(comment_header), 20) + newdata = reformat_paragraph(data, format_width) + # re-split and re-insert the comment header. + newdata = newdata.split("\n") + # If the block ends in a \n, we don't want the comment prefix + # inserted after it. (Im not sure it makes sense to reformat a + # comment block that is not made of complete lines, but whatever!) + # Can't think of a clean solution, so we hack away + block_suffix = "" + if not newdata[-1]: + block_suffix = "\n" + newdata = newdata[:-1] + return '\n'.join(comment_header+line for line in newdata) + block_suffix + +def is_all_white(line): + """Return True if line is empty or all whitespace.""" + + return re.match(r"^\s*$", line) is not None + +def get_indent(line): + """Return the initial space or tab indent of line.""" + return re.match(r"^([ \t]*)", line).group() + +def get_comment_header(line): + """Return string with leading whitespace and '#' from line or ''. + + A null return indicates that the line is not a comment line. A non- + null return, such as ' #', will be used to find the other lines of + a comment block with the same indent. + """ + m = re.match(r"^([ \t]*#*)", line) + if m is None: return "" + return m.group(1) + + +# Copied from editor.py; importing it would cause an import cycle. +_line_indent_re = re.compile(r'[ \t]*') + +def get_line_indent(line, tabwidth): + """Return a line's indentation as (# chars, effective # of spaces). + + The effective # of spaces is the length after properly "expanding" + the tabs into spaces, as done by str.expandtabs(tabwidth). + """ + m = _line_indent_re.match(line) + return m.end(), len(m.group().expandtabs(tabwidth)) + + +class FormatRegion: + "Format selected text (region)." + + def __init__(self, editwin): + self.editwin = editwin + + def get_region(self): + """Return line information about the selected text region. + + If text is selected, the first and last indices will be + for the selection. If there is no text selected, the + indices will be the current cursor location. + + Return a tuple containing (first index, last index, + string representation of text, list of text lines). + """ + text = self.editwin.text + first, last = self.editwin.get_selection_indices() + if first and last: + head = text.index(first + " linestart") + tail = text.index(last + "-1c lineend +1c") + else: + head = text.index("insert linestart") + tail = text.index("insert lineend +1c") + chars = text.get(head, tail) + lines = chars.split("\n") + return head, tail, chars, lines + + def set_region(self, head, tail, chars, lines): + """Replace the text between the given indices. + + Args: + head: Starting index of text to replace. + tail: Ending index of text to replace. + chars: Expected to be string of current text + between head and tail. + lines: List of new lines to insert between head + and tail. + """ + text = self.editwin.text + newchars = "\n".join(lines) + if newchars == chars: + text.bell() + return + text.tag_remove("sel", "1.0", "end") + text.mark_set("insert", head) + text.undo_block_start() + text.delete(head, tail) + text.insert(head, newchars) + text.undo_block_stop() + text.tag_add("sel", head, "insert") + + def indent_region_event(self, event=None): + "Indent region by indentwidth spaces." + head, tail, chars, lines = self.get_region() + for pos in range(len(lines)): + line = lines[pos] + if line: + raw, effective = get_line_indent(line, self.editwin.tabwidth) + effective = effective + self.editwin.indentwidth + lines[pos] = self.editwin._make_blanks(effective) + line[raw:] + self.set_region(head, tail, chars, lines) + return "break" + + def dedent_region_event(self, event=None): + "Dedent region by indentwidth spaces." + head, tail, chars, lines = self.get_region() + for pos in range(len(lines)): + line = lines[pos] + if line: + raw, effective = get_line_indent(line, self.editwin.tabwidth) + effective = max(effective - self.editwin.indentwidth, 0) + lines[pos] = self.editwin._make_blanks(effective) + line[raw:] + self.set_region(head, tail, chars, lines) + return "break" + + def comment_region_event(self, event=None): + """Comment out each line in region. + + ## is appended to the beginning of each line to comment it out. + """ + head, tail, chars, lines = self.get_region() + for pos in range(len(lines) - 1): + line = lines[pos] + lines[pos] = '##' + line + self.set_region(head, tail, chars, lines) + return "break" + + def uncomment_region_event(self, event=None): + """Uncomment each line in region. + + Remove ## or # in the first positions of a line. If the comment + is not in the beginning position, this command will have no effect. + """ + head, tail, chars, lines = self.get_region() + for pos in range(len(lines)): + line = lines[pos] + if not line: + continue + if line[:2] == '##': + line = line[2:] + elif line[:1] == '#': + line = line[1:] + lines[pos] = line + self.set_region(head, tail, chars, lines) + return "break" + + def tabify_region_event(self, event=None): + "Convert leading spaces to tabs for each line in selected region." + head, tail, chars, lines = self.get_region() + tabwidth = self._asktabwidth() + if tabwidth is None: + return + for pos in range(len(lines)): + line = lines[pos] + if line: + raw, effective = get_line_indent(line, tabwidth) + ntabs, nspaces = divmod(effective, tabwidth) + lines[pos] = '\t' * ntabs + ' ' * nspaces + line[raw:] + self.set_region(head, tail, chars, lines) + return "break" + + def untabify_region_event(self, event=None): + "Expand tabs to spaces for each line in region." + head, tail, chars, lines = self.get_region() + tabwidth = self._asktabwidth() + if tabwidth is None: + return + for pos in range(len(lines)): + lines[pos] = lines[pos].expandtabs(tabwidth) + self.set_region(head, tail, chars, lines) + return "break" + + def _asktabwidth(self): + "Return value for tab width." + return askinteger( + "Tab width", + "Columns per tab? (2-16)", + parent=self.editwin.text, + initialvalue=self.editwin.indentwidth, + minvalue=2, + maxvalue=16) + + +class Indents: + "Change future indents." + + def __init__(self, editwin): + self.editwin = editwin + + def toggle_tabs_event(self, event): + editwin = self.editwin + usetabs = editwin.usetabs + if askyesno( + "Toggle tabs", + "Turn tabs " + ("on", "off")[usetabs] + + "?\nIndent width " + + ("will be", "remains at")[usetabs] + " 8." + + "\n Note: a tab is always 8 columns", + parent=editwin.text): + editwin.usetabs = not usetabs + # Try to prevent inconsistent indentation. + # User must change indent width manually after using tabs. + editwin.indentwidth = 8 + return "break" + + def change_indentwidth_event(self, event): + editwin = self.editwin + new = askinteger( + "Indent width", + "New indent width (2-16)\n(Always use 8 when using tabs)", + parent=editwin.text, + initialvalue=editwin.indentwidth, + minvalue=2, + maxvalue=16) + if new and new != editwin.indentwidth and not editwin.usetabs: + editwin.indentwidth = new + return "break" + + +class Rstrip: # 'Strip Trailing Whitespace" on "Format" menu. + def __init__(self, editwin): + self.editwin = editwin + + def do_rstrip(self, event=None): + text = self.editwin.text + undo = self.editwin.undo + undo.undo_block_start() + + end_line = int(float(text.index('end'))) + for cur in range(1, end_line): + txt = text.get('%i.0' % cur, '%i.end' % cur) + raw = len(txt) + cut = len(txt.rstrip()) + # Since text.delete() marks file as changed, even if not, + # only call it when needed to actually delete something. + if cut < raw: + text.delete('%i.%i' % (cur, cut), '%i.end' % cur) + + if (text.get('end-2c') == '\n' # File ends with at least 1 newline; + and not hasattr(self.editwin, 'interp')): # & is not Shell. + # Delete extra user endlines. + while (text.index('end-1c') > '1.0' # Stop if file empty. + and text.get('end-3c') == '\n'): + text.delete('end-3c') + # Because tk indexes are slice indexes and never raise, + # a file with only newlines will be emptied. + # patchcheck.py does the same. + + undo.undo_block_stop() + + +if __name__ == "__main__": + from unittest import main + main('idlelib.idle_test.test_format', verbosity=2, exit=False) diff --git a/lib-python/3/idlelib/help.html b/lib-python/3/idlelib/help.html index 91803fd06c..424c6b50f3 100644 --- a/lib-python/3/idlelib/help.html +++ b/lib-python/3/idlelib/help.html @@ -1,12 +1,10 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> +<!DOCTYPE html> <html xmlns="http://www.w3.org/1999/xhtml"> <head> - <meta http-equiv="X-UA-Compatible" content="IE=Edge" /> - <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>IDLE — Python 3.9.0a0 documentation</title> + <meta charset="utf-8" /> + <title>IDLE — Python 3.9.0a4 documentation</title> <link rel="stylesheet" href="../_static/pydoctheme.css" type="text/css" /> <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> @@ -14,19 +12,19 @@ <script type="text/javascript" src="../_static/jquery.js"></script> <script type="text/javascript" src="../_static/underscore.js"></script> <script type="text/javascript" src="../_static/doctools.js"></script> - <script async="async" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script> + <script type="text/javascript" src="../_static/language_data.js"></script> <script type="text/javascript" src="../_static/sidebar.js"></script> <link rel="search" type="application/opensearchdescription+xml" - title="Search within Python 3.9.0a0 documentation" + title="Search within Python 3.9.0a4 documentation" href="../_static/opensearch.xml"/> <link rel="author" title="About these documents" href="../about.html" /> <link rel="index" title="Index" href="../genindex.html" /> <link rel="search" title="Search" href="../search.html" /> <link rel="copyright" title="Copyright" href="../copyright.html" /> <link rel="next" title="Other Graphical User Interface Packages" href="othergui.html" /> - <link rel="prev" title="tkinter.scrolledtext — Scrolled Text Widget" href="tkinter.scrolledtext.html" /> + <link rel="prev" title="tkinter.tix — Extension widgets for Tk" href="tkinter.tix.html" /> <link rel="canonical" href="https://docs.python.org/3/library/idle.html" /> @@ -64,7 +62,7 @@ <a href="othergui.html" title="Other Graphical User Interface Packages" accesskey="N">next</a> |</li> <li class="right" > - <a href="tkinter.scrolledtext.html" title="tkinter.scrolledtext — Scrolled Text Widget" + <a href="tkinter.tix.html" title="tkinter.tix — Extension widgets for Tk" accesskey="P">previous</a> |</li> <li><img src="../_static/py.png" alt="" @@ -73,7 +71,7 @@ <li> - <a href="../index.html">3.9.0a0 Documentation</a> » + <a href="../index.html">3.9.0a4 Documentation</a> » </li> <li class="nav-item nav-item-1"><a href="index.html" >The Python Standard Library</a> »</li> @@ -108,17 +106,17 @@ <p>IDLE is Python’s Integrated Development and Learning Environment.</p> <p>IDLE has the following features:</p> <ul class="simple"> -<li>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><code class="xref py py-mod docutils literal notranslate"><span class="pre">tkinter</span></code></a> GUI toolkit</li> -<li>cross-platform: works mostly the same on Windows, Unix, and macOS</li> -<li>Python shell window (interactive interpreter) with colorizing -of code input, output, and error messages</li> -<li>multi-window text editor with multiple undo, Python colorizing, -smart indent, call tips, auto completion, and other features</li> -<li>search within any window, replace within editor windows, and search -through multiple files (grep)</li> -<li>debugger with persistent breakpoints, stepping, and viewing -of global and local namespaces</li> -<li>configuration, browsers, and other dialogs</li> +<li><p>coded in 100% pure Python, using the <a class="reference internal" href="tkinter.html#module-tkinter" title="tkinter: Interface to Tcl/Tk for graphical user interfaces"><code class="xref py py-mod docutils literal notranslate"><span class="pre">tkinter</span></code></a> GUI toolkit</p></li> +<li><p>cross-platform: works mostly the same on Windows, Unix, and macOS</p></li> +<li><p>Python shell window (interactive interpreter) with colorizing +of code input, output, and error messages</p></li> +<li><p>multi-window text editor with multiple undo, Python colorizing, +smart indent, call tips, auto completion, and other features</p></li> +<li><p>search within any window, replace within editor windows, and search +through multiple files (grep)</p></li> +<li><p>debugger with persistent breakpoints, stepping, and viewing +of global and local namespaces</p></li> +<li><p>configuration, browsers, and other dialogs</p></li> </ul> <div class="section" id="menus"> <h2>Menus<a class="headerlink" href="#menus" title="Permalink to this headline">¶</a></h2> @@ -134,207 +132,219 @@ to the window currently selected. It has an IDLE menu, and some entries described below are moved around to conform to Apple guidelines.</p> <div class="section" id="file-menu-shell-and-editor"> <h3>File menu (Shell and Editor)<a class="headerlink" href="#file-menu-shell-and-editor" title="Permalink to this headline">¶</a></h3> -<dl class="docutils"> -<dt>New File</dt> -<dd>Create a new file editing window.</dd> -<dt>Open…</dt> -<dd>Open an existing file with an Open dialog.</dd> -<dt>Recent Files</dt> -<dd>Open a list of recent files. Click one to open it.</dd> -<dt>Open Module…</dt> -<dd>Open an existing module (searches sys.path).</dd> +<dl class="simple"> +<dt>New File</dt><dd><p>Create a new file editing window.</p> +</dd> +<dt>Open…</dt><dd><p>Open an existing file with an Open dialog.</p> +</dd> +<dt>Recent Files</dt><dd><p>Open a list of recent files. Click one to open it.</p> +</dd> +<dt>Open Module…</dt><dd><p>Open an existing module (searches sys.path).</p> +</dd> </dl> -<dl class="docutils" id="index-1"> -<dt>Class Browser</dt> -<dd>Show functions, classes, and methods in the current Editor file in a -tree structure. In the shell, open a module first.</dd> -<dt>Path Browser</dt> -<dd>Show sys.path directories, modules, functions, classes and methods in a -tree structure.</dd> -<dt>Save</dt> -<dd>Save the current window to the associated file, if there is one. Windows +<dl class="simple" id="index-1"> +<dt>Class Browser</dt><dd><p>Show functions, classes, and methods in the current Editor file in a +tree structure. In the shell, open a module first.</p> +</dd> +<dt>Path Browser</dt><dd><p>Show sys.path directories, modules, functions, classes and methods in a +tree structure.</p> +</dd> +<dt>Save</dt><dd><p>Save the current window to the associated file, if there is one. Windows that have been changed since being opened or last saved have a * before and after the window title. If there is no associated file, -do Save As instead.</dd> -<dt>Save As…</dt> -<dd>Save the current window with a Save As dialog. The file saved becomes the -new associated file for the window.</dd> -<dt>Save Copy As…</dt> -<dd>Save the current window to different file without changing the associated -file.</dd> -<dt>Print Window</dt> -<dd>Print the current window to the default printer.</dd> -<dt>Close</dt> -<dd>Close the current window (ask to save if unsaved).</dd> -<dt>Exit</dt> -<dd>Close all windows and quit IDLE (ask to save unsaved windows).</dd> +do Save As instead.</p> +</dd> +<dt>Save As…</dt><dd><p>Save the current window with a Save As dialog. The file saved becomes the +new associated file for the window.</p> +</dd> +<dt>Save Copy As…</dt><dd><p>Save the current window to different file without changing the associated +file.</p> +</dd> +<dt>Print Window</dt><dd><p>Print the current window to the default printer.</p> +</dd> +<dt>Close</dt><dd><p>Close the current window (ask to save if unsaved).</p> +</dd> +<dt>Exit</dt><dd><p>Close all windows and quit IDLE (ask to save unsaved windows).</p> +</dd> </dl> </div> <div class="section" id="edit-menu-shell-and-editor"> <h3>Edit menu (Shell and Editor)<a class="headerlink" href="#edit-menu-shell-and-editor" title="Permalink to this headline">¶</a></h3> -<dl class="docutils"> -<dt>Undo</dt> -<dd>Undo the last change to the current window. A maximum of 1000 changes may -be undone.</dd> -<dt>Redo</dt> -<dd>Redo the last undone change to the current window.</dd> -<dt>Cut</dt> -<dd>Copy selection into the system-wide clipboard; then delete the selection.</dd> -<dt>Copy</dt> -<dd>Copy selection into the system-wide clipboard.</dd> -<dt>Paste</dt> -<dd>Insert contents of the system-wide clipboard into the current window.</dd> +<dl class="simple"> +<dt>Undo</dt><dd><p>Undo the last change to the current window. A maximum of 1000 changes may +be undone.</p> +</dd> +<dt>Redo</dt><dd><p>Redo the last undone change to the current window.</p> +</dd> +<dt>Cut</dt><dd><p>Copy selection into the system-wide clipboard; then delete the selection.</p> +</dd> +<dt>Copy</dt><dd><p>Copy selection into the system-wide clipboard.</p> +</dd> +<dt>Paste</dt><dd><p>Insert contents of the system-wide clipboard into the current window.</p> +</dd> </dl> <p>The clipboard functions are also available in context menus.</p> -<dl class="docutils"> -<dt>Select All</dt> -<dd>Select the entire contents of the current window.</dd> -<dt>Find…</dt> -<dd>Open a search dialog with many options</dd> -<dt>Find Again</dt> -<dd>Repeat the last search, if there is one.</dd> -<dt>Find Selection</dt> -<dd>Search for the currently selected string, if there is one.</dd> -<dt>Find in Files…</dt> -<dd>Open a file search dialog. Put results in a new output window.</dd> -<dt>Replace…</dt> -<dd>Open a search-and-replace dialog.</dd> -<dt>Go to Line</dt> -<dd>Move cursor to the line number requested and make that line visible.</dd> -<dt>Show Completions</dt> -<dd>Open a scrollable list allowing selection of keywords and attributes. See -<a class="reference internal" href="#completions"><span class="std std-ref">Completions</span></a> in the Editing and navigation section below.</dd> -<dt>Expand Word</dt> -<dd>Expand a prefix you have typed to match a full word in the same window; -repeat to get a different expansion.</dd> -<dt>Show call tip</dt> -<dd>After an unclosed parenthesis for a function, open a small window with +<dl class="simple"> +<dt>Select All</dt><dd><p>Select the entire contents of the current window.</p> +</dd> +<dt>Find…</dt><dd><p>Open a search dialog with many options</p> +</dd> +<dt>Find Again</dt><dd><p>Repeat the last search, if there is one.</p> +</dd> +<dt>Find Selection</dt><dd><p>Search for the currently selected string, if there is one.</p> +</dd> +<dt>Find in Files…</dt><dd><p>Open a file search dialog. Put results in a new output window.</p> +</dd> +<dt>Replace…</dt><dd><p>Open a search-and-replace dialog.</p> +</dd> +<dt>Go to Line</dt><dd><p>Move the cursor to the beginning of the line requested and make that +line visible. A request past the end of the file goes to the end. +Clear any selection and update the line and column status.</p> +</dd> +<dt>Show Completions</dt><dd><p>Open a scrollable list allowing selection of keywords and attributes. See +<a class="reference internal" href="#completions"><span class="std std-ref">Completions</span></a> in the Editing and navigation section below.</p> +</dd> +<dt>Expand Word</dt><dd><p>Expand a prefix you have typed to match a full word in the same window; +repeat to get a different expansion.</p> +</dd> +<dt>Show call tip</dt><dd><p>After an unclosed parenthesis for a function, open a small window with function parameter hints. See <a class="reference internal" href="#calltips"><span class="std std-ref">Calltips</span></a> in the -Editing and navigation section below.</dd> -<dt>Show surrounding parens</dt> -<dd>Highlight the surrounding parenthesis.</dd> +Editing and navigation section below.</p> +</dd> +<dt>Show surrounding parens</dt><dd><p>Highlight the surrounding parenthesis.</p> +</dd> </dl> </div> <div class="section" id="format-menu-editor-window-only"> <span id="format-menu"></span><h3>Format menu (Editor window only)<a class="headerlink" href="#format-menu-editor-window-only" title="Permalink to this headline">¶</a></h3> -<dl class="docutils"> -<dt>Indent Region</dt> -<dd>Shift selected lines right by the indent width (default 4 spaces).</dd> -<dt>Dedent Region</dt> -<dd>Shift selected lines left by the indent width (default 4 spaces).</dd> -<dt>Comment Out Region</dt> -<dd>Insert ## in front of selected lines.</dd> -<dt>Uncomment Region</dt> -<dd>Remove leading # or ## from selected lines.</dd> -<dt>Tabify Region</dt> -<dd>Turn <em>leading</em> stretches of spaces into tabs. (Note: We recommend using -4 space blocks to indent Python code.)</dd> -<dt>Untabify Region</dt> -<dd>Turn <em>all</em> tabs into the correct number of spaces.</dd> -<dt>Toggle Tabs</dt> -<dd>Open a dialog to switch between indenting with spaces and tabs.</dd> -<dt>New Indent Width</dt> -<dd>Open a dialog to change indent width. The accepted default by the Python -community is 4 spaces.</dd> -<dt>Format Paragraph</dt> -<dd>Reformat the current blank-line-delimited paragraph in comment block or +<dl class="simple"> +<dt>Indent Region</dt><dd><p>Shift selected lines right by the indent width (default 4 spaces).</p> +</dd> +<dt>Dedent Region</dt><dd><p>Shift selected lines left by the indent width (default 4 spaces).</p> +</dd> +<dt>Comment Out Region</dt><dd><p>Insert ## in front of selected lines.</p> +</dd> +<dt>Uncomment Region</dt><dd><p>Remove leading # or ## from selected lines.</p> +</dd> +<dt>Tabify Region</dt><dd><p>Turn <em>leading</em> stretches of spaces into tabs. (Note: We recommend using +4 space blocks to indent Python code.)</p> +</dd> +<dt>Untabify Region</dt><dd><p>Turn <em>all</em> tabs into the correct number of spaces.</p> +</dd> +<dt>Toggle Tabs</dt><dd><p>Open a dialog to switch between indenting with spaces and tabs.</p> +</dd> +<dt>New Indent Width</dt><dd><p>Open a dialog to change indent width. The accepted default by the Python +community is 4 spaces.</p> +</dd> +<dt>Format Paragraph</dt><dd><p>Reformat the current blank-line-delimited paragraph in comment block or multiline string or selected line in a string. All lines in the -paragraph will be formatted to less than N columns, where N defaults to 72.</dd> -<dt>Strip trailing whitespace</dt> -<dd>Remove trailing space and other whitespace characters after the last +paragraph will be formatted to less than N columns, where N defaults to 72.</p> +</dd> +<dt>Strip trailing whitespace</dt><dd><p>Remove trailing space and other whitespace characters after the last non-whitespace character of a line by applying str.rstrip to each line, -including lines within multiline strings.</dd> +including lines within multiline strings. Except for Shell windows, +remove extra newlines at the end of the file.</p> +</dd> </dl> </div> <div class="section" id="run-menu-editor-window-only"> <span id="index-2"></span><h3>Run menu (Editor window only)<a class="headerlink" href="#run-menu-editor-window-only" title="Permalink to this headline">¶</a></h3> -<dl class="docutils" id="python-shell"> -<dt>Python Shell</dt> -<dd>Open or wake up the Python Shell window.</dd> -</dl> -<dl class="docutils" id="check-module"> -<dt>Check Module</dt> -<dd>Check the syntax of the module currently open in the Editor window. If the -module has not been saved IDLE will either prompt the user to save or -autosave, as selected in the General tab of the Idle Settings dialog. If -there is a syntax error, the approximate location is indicated in the -Editor window.</dd> -</dl> -<dl class="docutils" id="run-module"> -<dt>Run Module</dt> -<dd>Do <a class="reference internal" href="#check-module"><span class="std std-ref">Check Module</span></a>. If no error, restart the shell to clean the +<dl class="simple" id="run-module"> +<dt>Run Module</dt><dd><p>Do <a class="reference internal" href="#check-module"><span class="std std-ref">Check Module</span></a>. If no error, restart the shell to clean the environment, then execute the module. Output is displayed in the Shell window. Note that output requires use of <code class="docutils literal notranslate"><span class="pre">print</span></code> or <code class="docutils literal notranslate"><span class="pre">write</span></code>. When execution is complete, the Shell retains focus and displays a prompt. At this point, one may interactively explore the result of execution. This is similar to executing a file with <code class="docutils literal notranslate"><span class="pre">python</span> <span class="pre">-i</span> <span class="pre">file</span></code> at a command -line.</dd> +line.</p> +</dd> </dl> -<dl class="docutils" id="run-custom"> -<dt>Run… Customized</dt> -<dd>Same as <a class="reference internal" href="#run-module"><span class="std std-ref">Run Module</span></a>, but run the module with customized +<dl class="simple" id="run-custom"> +<dt>Run… Customized</dt><dd><p>Same as <a class="reference internal" href="#run-module"><span class="std std-ref">Run Module</span></a>, but run the module with customized settings. <em>Command Line Arguments</em> extend <a class="reference internal" href="sys.html#sys.argv" title="sys.argv"><code class="xref py py-data docutils literal notranslate"><span class="pre">sys.argv</span></code></a> as if passed -on a command line. The module can be run in the Shell without restarting.</dd> +on a command line. The module can be run in the Shell without restarting.</p> +</dd> +</dl> +<dl class="simple" id="check-module"> +<dt>Check Module</dt><dd><p>Check the syntax of the module currently open in the Editor window. If the +module has not been saved IDLE will either prompt the user to save or +autosave, as selected in the General tab of the Idle Settings dialog. If +there is a syntax error, the approximate location is indicated in the +Editor window.</p> +</dd> +</dl> +<dl class="simple" id="python-shell"> +<dt>Python Shell</dt><dd><p>Open or wake up the Python Shell window.</p> +</dd> </dl> </div> <div class="section" id="shell-menu-shell-window-only"> <h3>Shell menu (Shell window only)<a class="headerlink" href="#shell-menu-shell-window-only" title="Permalink to this headline">¶</a></h3> -<dl class="docutils"> -<dt>View Last Restart</dt> -<dd>Scroll the shell window to the last Shell restart.</dd> -<dt>Restart Shell</dt> -<dd>Restart the shell to clean the environment.</dd> -<dt>Previous History</dt> -<dd>Cycle through earlier commands in history which match the current entry.</dd> -<dt>Next History</dt> -<dd>Cycle through later commands in history which match the current entry.</dd> -<dt>Interrupt Execution</dt> -<dd>Stop a running program.</dd> +<dl class="simple"> +<dt>View Last Restart</dt><dd><p>Scroll the shell window to the last Shell restart.</p> +</dd> +<dt>Restart Shell</dt><dd><p>Restart the shell to clean the environment.</p> +</dd> +<dt>Previous History</dt><dd><p>Cycle through earlier commands in history which match the current entry.</p> +</dd> +<dt>Next History</dt><dd><p>Cycle through later commands in history which match the current entry.</p> +</dd> +<dt>Interrupt Execution</dt><dd><p>Stop a running program.</p> +</dd> </dl> </div> <div class="section" id="debug-menu-shell-window-only"> <h3>Debug menu (Shell window only)<a class="headerlink" href="#debug-menu-shell-window-only" title="Permalink to this headline">¶</a></h3> -<dl class="docutils"> -<dt>Go to File/Line</dt> -<dd>Look on the current line. with the cursor, and the line above for a filename +<dl class="simple"> +<dt>Go to File/Line</dt><dd><p>Look on the current line. with the cursor, and the line above for a filename and line number. If found, open the file if not already open, and show the line. Use this to view source lines referenced in an exception traceback and lines found by Find in Files. Also available in the context menu of -the Shell window and Output windows.</dd> +the Shell window and Output windows.</p> +</dd> </dl> -<dl class="docutils" id="index-3"> -<dt>Debugger (toggle)</dt> -<dd>When activated, code entered in the Shell or run from an Editor will run +<dl class="simple" id="index-3"> +<dt>Debugger (toggle)</dt><dd><p>When activated, code entered in the Shell or run from an Editor will run under the debugger. In the Editor, breakpoints can be set with the context -menu. This feature is still incomplete and somewhat experimental.</dd> -<dt>Stack Viewer</dt> -<dd>Show the stack traceback of the last exception in a tree widget, with -access to locals and globals.</dd> -<dt>Auto-open Stack Viewer</dt> -<dd>Toggle automatically opening the stack viewer on an unhandled exception.</dd> +menu. This feature is still incomplete and somewhat experimental.</p> +</dd> +<dt>Stack Viewer</dt><dd><p>Show the stack traceback of the last exception in a tree widget, with +access to locals and globals.</p> +</dd> +<dt>Auto-open Stack Viewer</dt><dd><p>Toggle automatically opening the stack viewer on an unhandled exception.</p> +</dd> </dl> </div> <div class="section" id="options-menu-shell-and-editor"> <h3>Options menu (Shell and Editor)<a class="headerlink" href="#options-menu-shell-and-editor" title="Permalink to this headline">¶</a></h3> -<dl class="docutils"> -<dt>Configure IDLE</dt> -<dd>Open a configuration dialog and change preferences for the following: +<dl class="simple"> +<dt>Configure IDLE</dt><dd><p>Open a configuration dialog and change preferences for the following: fonts, indentation, keybindings, text color themes, startup windows and -size, additional help sources, and extensions. On macOS, open the +size, additional help sources, and extensions. On macOS, open the configuration dialog by selecting Preferences in the application -menu. For more, see -<a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a> under Help and preferences.</dd> -<dt>Show/Hide Code Context (Editor Window only)</dt> -<dd>Open a pane at the top of the edit window which shows the block context +menu. For more details, see +<a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a> under Help and preferences.</p> +</dd> +</dl> +<p>Most configuration options apply to all windows or all future windows. +The option items below only apply to the active window.</p> +<dl class="simple"> +<dt>Show/Hide Code Context (Editor Window only)</dt><dd><p>Open a pane at the top of the edit window which shows the block context of the code which has scrolled above the top of the window. See -<a class="reference internal" href="#code-context"><span class="std std-ref">Code Context</span></a> in the Editing and Navigation section below.</dd> -<dt>Zoom/Restore Height</dt> -<dd>Toggles the window between normal size and maximum height. The initial size +<a class="reference internal" href="#code-context"><span class="std std-ref">Code Context</span></a> in the Editing and Navigation section +below.</p> +</dd> +<dt>Show/Hide Line Numbers (Editor Window only)</dt><dd><p>Open a column to the left of the edit window which shows the number +of each line of text. The default is off, which may be changed in the +preferences (see <a class="reference internal" href="#preferences"><span class="std std-ref">Setting preferences</span></a>).</p> +</dd> +<dt>Zoom/Restore Height</dt><dd><p>Toggles the window between normal size and maximum height. The initial size defaults to 40 lines by 80 chars unless changed on the General tab of the Configure IDLE dialog. The maximum height for a screen is determined by momentarily maximizing a window the first time one is zoomed on the screen. -Changing screen settings may invalidate the saved height. This toogle has -no effect when a window is maximized.</dd> +Changing screen settings may invalidate the saved height. This toggle has +no effect when a window is maximized.</p> +</dd> </dl> </div> <div class="section" id="window-menu-shell-and-editor"> @@ -344,17 +354,17 @@ no effect when a window is maximized.</dd> </div> <div class="section" id="help-menu-shell-and-editor"> <h3>Help menu (Shell and Editor)<a class="headerlink" href="#help-menu-shell-and-editor" title="Permalink to this headline">¶</a></h3> -<dl class="docutils"> -<dt>About IDLE</dt> -<dd>Display version, copyright, license, credits, and more.</dd> -<dt>IDLE Help</dt> -<dd>Display this IDLE document, detailing the menu options, basic editing and -navigation, and other tips.</dd> -<dt>Python Docs</dt> -<dd>Access local Python documentation, if installed, or start a web browser -and open docs.python.org showing the latest Python documentation.</dd> -<dt>Turtle Demo</dt> -<dd>Run the turtledemo module with example Python code and turtle drawings.</dd> +<dl class="simple"> +<dt>About IDLE</dt><dd><p>Display version, copyright, license, credits, and more.</p> +</dd> +<dt>IDLE Help</dt><dd><p>Display this IDLE document, detailing the menu options, basic editing and +navigation, and other tips.</p> +</dd> +<dt>Python Docs</dt><dd><p>Access local Python documentation, if installed, or start a web browser +and open docs.python.org showing the latest Python documentation.</p> +</dd> +<dt>Turtle Demo</dt><dd><p>Run the turtledemo module with example Python code and turtle drawings.</p> +</dd> </dl> <p>Additional help sources may be added here with the Configure IDLE dialog under the General tab. See the <a class="reference internal" href="#help-sources"><span class="std std-ref">Help sources</span></a> subsection below @@ -364,34 +374,35 @@ for more on Help menu choices.</p> <span id="index-4"></span><h3>Context Menus<a class="headerlink" href="#context-menus" title="Permalink to this headline">¶</a></h3> <p>Open a context menu by right-clicking in a window (Control-click on macOS). Context menus have the standard clipboard functions also on the Edit menu.</p> -<dl class="docutils"> -<dt>Cut</dt> -<dd>Copy selection into the system-wide clipboard; then delete the selection.</dd> -<dt>Copy</dt> -<dd>Copy selection into the system-wide clipboard.</dd> -<dt>Paste</dt> -<dd>Insert contents of the system-wide clipboard into the current window.</dd> +<dl class="simple"> +<dt>Cut</dt><dd><p>Copy selection into the system-wide clipboard; then delete the selection.</p> +</dd> +<dt>Copy</dt><dd><p>Copy selection into the system-wide clipboard.</p> +</dd> +<dt>Paste</dt><dd><p>Insert contents of the system-wide clipboard into the current window.</p> +</dd> </dl> <p>Editor windows also have breakpoint functions. Lines with a breakpoint set are specially marked. Breakpoints only have an effect when running under the -debugger. Breakpoints for a file are saved in the user’s .idlerc directory.</p> -<dl class="docutils"> -<dt>Set Breakpoint</dt> -<dd>Set a breakpoint on the current line.</dd> -<dt>Clear Breakpoint</dt> -<dd>Clear the breakpoint on that line.</dd> +debugger. Breakpoints for a file are saved in the user’s <code class="docutils literal notranslate"><span class="pre">.idlerc</span></code> +directory.</p> +<dl class="simple"> +<dt>Set Breakpoint</dt><dd><p>Set a breakpoint on the current line.</p> +</dd> +<dt>Clear Breakpoint</dt><dd><p>Clear the breakpoint on that line.</p> +</dd> </dl> <p>Shell and Output windows also have the following.</p> -<dl class="docutils"> -<dt>Go to file/line</dt> -<dd>Same as in Debug menu.</dd> +<dl class="simple"> +<dt>Go to file/line</dt><dd><p>Same as in Debug menu.</p> +</dd> </dl> <p>The Shell window also has an output squeezing facility explained in the <em>Python Shell window</em> subsection below.</p> -<dl class="docutils"> -<dt>Squeeze</dt> -<dd>If the cursor is over an output line, squeeze all the output between -the code above and the prompt below down to a ‘Squeezed text’ label.</dd> +<dl class="simple"> +<dt>Squeeze</dt><dd><p>If the cursor is over an output line, squeeze all the output between +the code above and the prompt below down to a ‘Squeezed text’ label.</p> +</dd> </dl> </div> </div> @@ -414,32 +425,26 @@ and that other files do not. Run Python code with the Run menu.</p> <p>In this section, ‘C’ refers to the <kbd class="kbd docutils literal notranslate">Control</kbd> key on Windows and Unix and the <kbd class="kbd docutils literal notranslate">Command</kbd> key on macOS.</p> <ul> -<li><p class="first"><kbd class="kbd docutils literal notranslate">Backspace</kbd> deletes to the left; <kbd class="kbd docutils literal notranslate">Del</kbd> deletes to the right</p> -</li> -<li><p class="first"><kbd class="kbd docutils literal notranslate">C-Backspace</kbd> delete word left; <kbd class="kbd docutils literal notranslate">C-Del</kbd> delete word to the right</p> -</li> -<li><p class="first">Arrow keys and <kbd class="kbd docutils literal notranslate">Page Up</kbd>/<kbd class="kbd docutils literal notranslate">Page Down</kbd> to move around</p> -</li> -<li><p class="first"><kbd class="kbd docutils literal notranslate">C-LeftArrow</kbd> and <kbd class="kbd docutils literal notranslate">C-RightArrow</kbd> moves by words</p> -</li> -<li><p class="first"><kbd class="kbd docutils literal notranslate">Home</kbd>/<kbd class="kbd docutils literal notranslate">End</kbd> go to begin/end of line</p> -</li> -<li><p class="first"><kbd class="kbd docutils literal notranslate">C-Home</kbd>/<kbd class="kbd docutils literal notranslate">C-End</kbd> go to begin/end of file</p> -</li> -<li><p class="first">Some useful Emacs bindings are inherited from Tcl/Tk:</p> +<li><p><kbd class="kbd docutils literal notranslate">Backspace</kbd> deletes to the left; <kbd class="kbd docutils literal notranslate">Del</kbd> deletes to the right</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-Backspace</kbd> delete word left; <kbd class="kbd docutils literal notranslate">C-Del</kbd> delete word to the right</p></li> +<li><p>Arrow keys and <kbd class="kbd docutils literal notranslate">Page Up</kbd>/<kbd class="kbd docutils literal notranslate">Page Down</kbd> to move around</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-LeftArrow</kbd> and <kbd class="kbd docutils literal notranslate">C-RightArrow</kbd> moves by words</p></li> +<li><p><kbd class="kbd docutils literal notranslate">Home</kbd>/<kbd class="kbd docutils literal notranslate">End</kbd> go to begin/end of line</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-Home</kbd>/<kbd class="kbd docutils literal notranslate">C-End</kbd> go to begin/end of file</p></li> +<li><p>Some useful Emacs bindings are inherited from Tcl/Tk:</p> <blockquote> <div><ul class="simple"> -<li><kbd class="kbd docutils literal notranslate">C-a</kbd> beginning of line</li> -<li><kbd class="kbd docutils literal notranslate">C-e</kbd> end of line</li> -<li><kbd class="kbd docutils literal notranslate">C-k</kbd> kill line (but doesn’t put it in clipboard)</li> -<li><kbd class="kbd docutils literal notranslate">C-l</kbd> center window around the insertion point</li> -<li><kbd class="kbd docutils literal notranslate">C-b</kbd> go backward one character without deleting (usually you can -also use the cursor key for this)</li> -<li><kbd class="kbd docutils literal notranslate">C-f</kbd> go forward one character without deleting (usually you can -also use the cursor key for this)</li> -<li><kbd class="kbd docutils literal notranslate">C-p</kbd> go up one line (usually you can also use the cursor key for -this)</li> -<li><kbd class="kbd docutils literal notranslate">C-d</kbd> delete next character</li> +<li><p><kbd class="kbd docutils literal notranslate">C-a</kbd> beginning of line</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-e</kbd> end of line</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-k</kbd> kill line (but doesn’t put it in clipboard)</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-l</kbd> center window around the insertion point</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-b</kbd> go backward one character without deleting (usually you can +also use the cursor key for this)</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-f</kbd> go forward one character without deleting (usually you can +also use the cursor key for this)</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-p</kbd> go up one line (usually you can also use the cursor key for +this)</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-d</kbd> delete next character</p></li> </ul> </div></blockquote> </li> @@ -542,17 +547,15 @@ If one pastes more that one statement into Shell, the result will be a <p>The editing features described in previous subsections work when entering code interactively. IDLE’s Shell window also responds to the following keys.</p> <ul> -<li><p class="first"><kbd class="kbd docutils literal notranslate">C-c</kbd> interrupts executing command</p> -</li> -<li><p class="first"><kbd class="kbd docutils literal notranslate">C-d</kbd> sends end-of-file; closes window if typed at a <code class="docutils literal notranslate"><span class="pre">>>></span></code> prompt</p> -</li> -<li><p class="first"><kbd class="kbd docutils literal notranslate">Alt-/</kbd> (Expand word) is also useful to reduce typing</p> +<li><p><kbd class="kbd docutils literal notranslate">C-c</kbd> interrupts executing command</p></li> +<li><p><kbd class="kbd docutils literal notranslate">C-d</kbd> sends end-of-file; closes window if typed at a <code class="docutils literal notranslate"><span class="pre">>>></span></code> prompt</p></li> +<li><p><kbd class="kbd docutils literal notranslate">Alt-/</kbd> (Expand word) is also useful to reduce typing</p> <p>Command history</p> <ul class="simple"> -<li><kbd class="kbd docutils literal notranslate">Alt-p</kbd> retrieves previous command matching what you have typed. On -macOS use <kbd class="kbd docutils literal notranslate">C-p</kbd>.</li> -<li><kbd class="kbd docutils literal notranslate">Alt-n</kbd> retrieves next. On macOS use <kbd class="kbd docutils literal notranslate">C-n</kbd>.</li> -<li><kbd class="kbd docutils literal notranslate">Return</kbd> while on any previous command retrieves that command</li> +<li><p><kbd class="kbd docutils literal notranslate">Alt-p</kbd> retrieves previous command matching what you have typed. On +macOS use <kbd class="kbd docutils literal notranslate">C-p</kbd>.</p></li> +<li><p><kbd class="kbd docutils literal notranslate">Alt-n</kbd> retrieves next. On macOS use <kbd class="kbd docutils literal notranslate">C-n</kbd>.</p></li> +<li><p><kbd class="kbd docutils literal notranslate">Return</kbd> while on any previous command retrieves that command</p></li> </ul> </li> </ul> @@ -602,12 +605,12 @@ functions to be used from IDLE’s Python shell.</p> </div> <p>If there are arguments:</p> <ul class="simple"> -<li>If <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">-c</span></code>, or <code class="docutils literal notranslate"><span class="pre">r</span></code> is used, all arguments are placed in +<li><p>If <code class="docutils literal notranslate"><span class="pre">-</span></code>, <code class="docutils literal notranslate"><span class="pre">-c</span></code>, or <code class="docutils literal notranslate"><span class="pre">r</span></code> is used, all arguments are placed in <code class="docutils literal notranslate"><span class="pre">sys.argv[1:...]</span></code> and <code class="docutils literal notranslate"><span class="pre">sys.argv[0]</span></code> is set to <code class="docutils literal notranslate"><span class="pre">''</span></code>, <code class="docutils literal notranslate"><span class="pre">'-c'</span></code>, or <code class="docutils literal notranslate"><span class="pre">'-r'</span></code>. No editor window is opened, even if that is the default -set in the Options dialog.</li> -<li>Otherwise, arguments are files opened for editing and -<code class="docutils literal notranslate"><span class="pre">sys.argv</span></code> reflects the arguments passed to IDLE itself.</li> +set in the Options dialog.</p></li> +<li><p>Otherwise, arguments are files opened for editing and +<code class="docutils literal notranslate"><span class="pre">sys.argv</span></code> reflects the arguments passed to IDLE itself.</p></li> </ul> </div> <div class="section" id="startup-failure"> @@ -634,17 +637,20 @@ clash, or a single installation might need admin access. If one undo the clash, or cannot or does not want to run as admin, it might be easiest to completely remove Python and start over.</p> <p>A zombie pythonw.exe process could be a problem. On Windows, use Task -Manager to detect and stop one. Sometimes a restart initiated by a program -crash or Keyboard Interrupt (control-C) may fail to connect. Dismissing -the error box or Restart Shell on the Shell menu may fix a temporary problem.</p> +Manager to check for one and stop it if there is. Sometimes a restart +initiated by a program crash or Keyboard Interrupt (control-C) may fail +to connect. Dismissing the error box or using Restart Shell on the Shell +menu may fix a temporary problem.</p> <p>When IDLE first starts, it attempts to read user configuration files in -~/.idlerc/ (~ is one’s home directory). If there is a problem, an error +<code class="docutils literal notranslate"><span class="pre">~/.idlerc/</span></code> (~ is one’s home directory). If there is a problem, an error message should be displayed. Leaving aside random disk glitches, this can -be prevented by never editing the files by hand, using the configuration -dialog, under Options, instead Options. Once it happens, the solution may -be to delete one or more of the configuration files.</p> +be prevented by never editing the files by hand. Instead, use the +configuration dialog, under Options. Once there is an error in a user +configuration file, the best solution may be to delete it and start over +with the settings dialog.</p> <p>If IDLE quits with no message, and it was not started from a console, try -starting from a console (<code class="docutils literal notranslate"><span class="pre">python</span> <span class="pre">-m</span> <span class="pre">idlelib)</span></code> and see if a message appears.</p> +starting it from a console or terminal (<code class="docutils literal notranslate"><span class="pre">python</span> <span class="pre">-m</span> <span class="pre">idlelib</span></code>) and see if +this results in an error message.</p> </div> <div class="section" id="running-user-code"> <h3>Running user code<a class="headerlink" href="#running-user-code" title="Permalink to this headline">¶</a></h3> @@ -670,6 +676,9 @@ such as multiprocessing. If such subprocess use <code class="docutils literal n or <code class="docutils literal notranslate"><span class="pre">print</span></code> or <code class="docutils literal notranslate"><span class="pre">write</span></code> to sys.stdout or sys.stderr, IDLE should be started in a command line window. The secondary subprocess will then be attached to that window for input and output.</p> +<p>The IDLE code running in the execution process adds frames to the call stack +that would not be there otherwise. IDLE wraps <code class="docutils literal notranslate"><span class="pre">sys.getrecursionlimit</span></code> and +<code class="docutils literal notranslate"><span class="pre">sys.setrecursionlimit</span></code> to reduce the effect of the additional stack frames.</p> <p>If <code class="docutils literal notranslate"><span class="pre">sys</span></code> is reset by user code, such as with <code class="docutils literal notranslate"><span class="pre">importlib.reload(sys)</span></code>, IDLE’s changes are lost and input from the keyboard and output to the screen will not work correctly.</p> @@ -772,7 +781,7 @@ re-import any specific items (e.g. from foo import baz) if the changes are to take effect. For these reasons, it is preferable to run IDLE with the default subprocess if at all possible.</p> <div class="deprecated"> -<p><span class="versionmodified">Deprecated since version 3.4.</span></p> +<p><span class="versionmodified deprecated">Deprecated since version 3.4.</span></p> </div> </div> </div> @@ -788,20 +797,20 @@ the scrollbar, or up and down arrow keys held down. Or click the TOC (Table of Contents) button and select a section header in the opened box.</p> <p>Help menu entry “Python Docs” opens the extensive sources of help, -including tutorials, available at docs.python.org/x.y, where ‘x.y’ +including tutorials, available at <code class="docutils literal notranslate"><span class="pre">docs.python.org/x.y</span></code>, where ‘x.y’ is the currently running Python version. If your system has an off-line copy of the docs (this may be an installation option), that will be opened instead.</p> <p>Selected URLs can be added or removed from the help menu at any time using the -General tab of the Configure IDLE dialog .</p> +General tab of the Configure IDLE dialog.</p> </div> <div class="section" id="setting-preferences"> <span id="preferences"></span><h3>Setting preferences<a class="headerlink" href="#setting-preferences" title="Permalink to this headline">¶</a></h3> <p>The font preferences, highlighting, keys, and general preferences can be changed via Configure IDLE on the Option menu. -Non-default user settings are saved in a .idlerc directory in the user’s +Non-default user settings are saved in a <code class="docutils literal notranslate"><span class="pre">.idlerc</span></code> directory in the user’s home directory. Problems caused by bad user configuration files are solved -by editing or deleting one or more of the files in .idlerc.</p> +by editing or deleting one or more of the files in <code class="docutils literal notranslate"><span class="pre">.idlerc</span></code>.</p> <p>On the Font tab, see the text sample for the effect of font face and size on multiple characters in multiple languages. Edit the sample to add other characters of personal interest. Use the sample to select @@ -884,8 +893,8 @@ also used for testing.</p> </ul> <h4>Previous topic</h4> - <p class="topless"><a href="tkinter.scrolledtext.html" - title="previous chapter"><code class="docutils literal notranslate"><span class="pre">tkinter.scrolledtext</span></code> — Scrolled Text Widget</a></p> + <p class="topless"><a href="tkinter.tix.html" + title="previous chapter"><code class="xref py py-mod docutils literal notranslate"><span class="pre">tkinter.tix</span></code> — Extension widgets for Tk</a></p> <h4>Next topic</h4> <p class="topless"><a href="othergui.html" title="next chapter">Other Graphical User Interface Packages</a></p> @@ -917,7 +926,7 @@ also used for testing.</p> <a href="othergui.html" title="Other Graphical User Interface Packages" >next</a> |</li> <li class="right" > - <a href="tkinter.scrolledtext.html" title="tkinter.scrolledtext — Scrolled Text Widget" + <a href="tkinter.tix.html" title="tkinter.tix — Extension widgets for Tk" >previous</a> |</li> <li><img src="../_static/py.png" alt="" @@ -926,7 +935,7 @@ also used for testing.</p> <li> - <a href="../index.html">3.9.0a0 Documentation</a> » + <a href="../index.html">3.9.0a4 Documentation</a> » </li> <li class="nav-item nav-item-1"><a href="index.html" >The Python Standard Library</a> »</li> @@ -949,7 +958,7 @@ also used for testing.</p> </ul> </div> <div class="footer"> - © <a href="../copyright.html">Copyright</a> 2001-2019, Python Software Foundation. + © <a href="../copyright.html">Copyright</a> 2001-2020, Python Software Foundation. <br /> The Python Software Foundation is a non-profit corporation. @@ -957,11 +966,11 @@ also used for testing.</p> <br /> <br /> - Last updated on Jun 17, 2019. + Last updated on Mar 07, 2020. <a href="https://docs.python.org/3/bugs.html">Found a bug</a>? <br /> - Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 1.8.1. + Created using <a href="http://sphinx.pocoo.org/">Sphinx</a> 2.1.1. </div> </body> diff --git a/lib-python/3/idlelib/help.py b/lib-python/3/idlelib/help.py index 652444a7f1..9f63ea0d39 100644 --- a/lib-python/3/idlelib/help.py +++ b/lib-python/3/idlelib/help.py @@ -50,20 +50,22 @@ class HelpParser(HTMLParser): """ def __init__(self, text): HTMLParser.__init__(self, convert_charrefs=True) - self.text = text # text widget we're rendering into - self.tags = '' # current block level text tags to apply - self.chartags = '' # current character level text tags - self.show = False # used so we exclude page navigation - self.hdrlink = False # used so we don't show header links - self.level = 0 # indentation level - self.pre = False # displaying preformatted text - self.hprefix = '' # prefix such as '25.5' to strip from headings - self.nested_dl = False # if we're in a nested <dl> - self.simplelist = False # simple list (no double spacing) - self.toc = [] # pair headers with text indexes for toc - self.header = '' # text within header tags for toc + self.text = text # Text widget we're rendering into. + self.tags = '' # Current block level text tags to apply. + self.chartags = '' # Current character level text tags. + self.show = False # Exclude html page navigation. + self.hdrlink = False # Exclude html header links. + self.level = 0 # Track indentation level. + self.pre = False # Displaying preformatted text? + self.hprefix = '' # Heading prefix (like '25.5'?) to remove. + self.nested_dl = False # In a nested <dl>? + self.simplelist = False # In a simple list (no double spacing)? + self.toc = [] # Pair headers with text indexes for toc. + self.header = '' # Text within header tags for toc. + self.prevtag = None # Previous tag info (opener?, tag). def indent(self, amt=1): + "Change indent (+1, 0, -1) and tags." self.level += amt self.tags = '' if self.level == 0 else 'l'+str(self.level) @@ -75,11 +77,14 @@ class HelpParser(HTMLParser): class_ = v s = '' if tag == 'div' and class_ == 'section': - self.show = True # start of main content + self.show = True # Start main content. elif tag == 'div' and class_ == 'sphinxsidebar': - self.show = False # end of main content - elif tag == 'p' and class_ != 'first': - s = '\n\n' + self.show = False # End main content. + elif tag == 'p' and self.prevtag and not self.prevtag[0]: + # Begin a new block for <p> tags after a closed tag. + # Avoid extra lines, e.g. after <pre> tags. + lastline = self.text.get('end-1c linestart', 'end-1c') + s = '\n\n' if lastline and not lastline.isspace() else '\n' elif tag == 'span' and class_ == 'pre': self.chartags = 'pre' elif tag == 'span' and class_ == 'versionmodified': @@ -99,7 +104,7 @@ class HelpParser(HTMLParser): elif tag == 'li': s = '\n* ' if self.simplelist else '\n\n* ' elif tag == 'dt': - s = '\n\n' if not self.nested_dl else '\n' # avoid extra line + s = '\n\n' if not self.nested_dl else '\n' # Avoid extra line. self.nested_dl = False elif tag == 'dd': self.indent() @@ -120,16 +125,18 @@ class HelpParser(HTMLParser): self.tags = tag if self.show: self.text.insert('end', s, (self.tags, self.chartags)) + self.prevtag = (True, tag) def handle_endtag(self, tag): "Handle endtags in help.html." if tag in ['h1', 'h2', 'h3']: - self.indent(0) # clear tag, reset indent + assert self.level == 0 if self.show: indent = (' ' if tag == 'h3' else ' ' if tag == 'h2' else '') self.toc.append((indent+self.header, self.text.index('insert'))) + self.tags = '' elif tag in ['span', 'em']: self.chartags = '' elif tag == 'a': @@ -138,7 +145,8 @@ class HelpParser(HTMLParser): self.pre = False self.tags = '' elif tag in ['ul', 'dd', 'ol']: - self.indent(amt=-1) + self.indent(-1) + self.prevtag = (False, tag) def handle_data(self, data): "Handle date segments in help.html." @@ -163,7 +171,7 @@ class HelpText(Text): "Configure tags and feed file to parser." uwide = idleConf.GetOption('main', 'EditorWindow', 'width', type='int') uhigh = idleConf.GetOption('main', 'EditorWindow', 'height', type='int') - uhigh = 3 * uhigh // 4 # lines average 4/3 of editor line height + uhigh = 3 * uhigh // 4 # Lines average 4/3 of editor line height. Text.__init__(self, parent, wrap='word', highlightthickness=0, padx=5, borderwidth=0, width=uwide, height=uhigh) @@ -203,7 +211,6 @@ class HelpFrame(Frame): "Display html text, scrollbar, and toc." def __init__(self, parent, filename): Frame.__init__(self, parent) - # keep references to widgets for test access. self.text = text = HelpText(self, filename) self['background'] = text['background'] self.toc = toc = self.toc_menu(text) @@ -211,7 +218,7 @@ class HelpFrame(Frame): text['yscrollcommand'] = scroll.set self.rowconfigure(0, weight=1) - self.columnconfigure(1, weight=1) # text + self.columnconfigure(1, weight=1) # Only expand the text widget. toc.grid(row=0, column=0, sticky='nw') text.grid(row=0, column=1, sticky='nsew') scroll.grid(row=0, column=2, sticky='ns') @@ -257,7 +264,7 @@ def copy_strip(): same, help.html can be backported. The internal Python version number is not displayed. If maintenance idle.rst diverges from the master version, then instead of backporting help.html from - master, repeat the proceedure above to generate a maintenance + master, repeat the procedure above to generate a maintenance version. """ src = join(abspath(dirname(dirname(dirname(__file__)))), @@ -273,7 +280,7 @@ def show_idlehelp(parent): "Create HelpWindow; called from Idle Help event handler." filename = join(abspath(dirname(__file__)), 'help.html') if not isfile(filename): - # try copy_strip, present message + # Try copy_strip, present message. return HelpWindow(parent, filename, 'IDLE Help (%s)' % python_version()) diff --git a/lib-python/3/idlelib/history.py b/lib-python/3/idlelib/history.py index 56f53a0f2f..ad44a96a9d 100644 --- a/lib-python/3/idlelib/history.py +++ b/lib-python/3/idlelib/history.py @@ -39,7 +39,7 @@ class History: return "break" def fetch(self, reverse): - '''Fetch statememt and replace current line in text widget. + '''Fetch statement and replace current line in text widget. Set prefix and pointer as needed for successive fetches. Reset them to None, None when returning to the start line. diff --git a/lib-python/3/idlelib/idle_test/htest.py b/lib-python/3/idlelib/idle_test/htest.py index 6ce8cc8a5f..1373b7642a 100644 --- a/lib-python/3/idlelib/idle_test/htest.py +++ b/lib-python/3/idlelib/idle_test/htest.py @@ -67,6 +67,7 @@ outwin.OutputWindow (indirectly being tested with grep test) import idlelib.pyshell # Set Windows DPI awareness before Tk(). from importlib import import_module +import textwrap import tkinter as tk from tkinter.ttk import Scrollbar tk.NoDefaultRoot() @@ -110,10 +111,11 @@ _color_delegator_spec = { CustomRun_spec = { 'file': 'query', - 'kwds': {'title': 'Custom Run Args', + 'kwds': {'title': 'Customize query.py Run', '_htest': True}, - 'msg': "Enter with <Return> or [Ok]. Print valid entry to Shell\n" + 'msg': "Enter with <Return> or [Run]. Print valid entry to Shell\n" "Arguments are parsed into a list\n" + "Mode is currently restart True or False\n" "Close dialog with valid entry, <Escape>, [Cancel], [X]" } @@ -204,6 +206,26 @@ _io_binding_spec = { "Check that changes were saved by opening the file elsewhere." } +_linenumbers_drag_scrolling_spec = { + 'file': 'sidebar', + 'kwds': {}, + 'msg': textwrap.dedent("""\ + 1. Click on the line numbers and drag down below the edge of the + window, moving the mouse a bit and then leaving it there for a while. + The text and line numbers should gradually scroll down, with the + selection updated continuously. + + 2. With the lines still selected, click on a line number above the + selected lines. Only the line whose number was clicked should be + selected. + + 3. Repeat step #1, dragging to above the window. The text and line + numbers should gradually scroll up, with the selection updated + continuously. + + 4. Repeat step #2, clicking a line number below the selection."""), + } + _multi_call_spec = { 'file': 'multicall', 'kwds': {}, @@ -334,7 +356,7 @@ _undo_delegator_spec = { ViewWindow_spec = { 'file': 'textview', 'kwds': {'title': 'Test textview', - 'text': 'The quick brown fox jumps over the lazy dog.\n'*35, + 'contents': 'The quick brown fox jumps over the lazy dog.\n'*35, '_htest': True}, 'msg': "Test for read-only property of text.\n" "Select text, scroll window, close" diff --git a/lib-python/3/idlelib/idle_test/mock_idle.py b/lib-python/3/idlelib/idle_test/mock_idle.py index f279a52fd5..71fa480ce4 100644 --- a/lib-python/3/idlelib/idle_test/mock_idle.py +++ b/lib-python/3/idlelib/idle_test/mock_idle.py @@ -40,8 +40,9 @@ class Func: class Editor: '''Minimally imitate editor.EditorWindow class. ''' - def __init__(self, flist=None, filename=None, key=None, root=None): - self.text = Text() + def __init__(self, flist=None, filename=None, key=None, root=None, + text=None): # Allow real Text with mock Editor. + self.text = text or Text() self.undo = UndoDelegator() def get_selection_indices(self): diff --git a/lib-python/3/idlelib/idle_test/test_autocomplete.py b/lib-python/3/idlelib/idle_test/test_autocomplete.py index 6181b29ec2..1841495fcf 100644 --- a/lib-python/3/idlelib/idle_test/test_autocomplete.py +++ b/lib-python/3/idlelib/idle_test/test_autocomplete.py @@ -1,4 +1,4 @@ -"Test autocomplete, coverage 87%." +"Test autocomplete, coverage 93%." import unittest from unittest.mock import Mock, patch @@ -45,127 +45,177 @@ class AutoCompleteTest(unittest.TestCase): def test_init(self): self.assertEqual(self.autocomplete.editwin, self.editor) + self.assertEqual(self.autocomplete.text, self.text) def test_make_autocomplete_window(self): testwin = self.autocomplete._make_autocomplete_window() self.assertIsInstance(testwin, acw.AutoCompleteWindow) def test_remove_autocomplete_window(self): - self.autocomplete.autocompletewindow = ( - self.autocomplete._make_autocomplete_window()) - self.autocomplete._remove_autocomplete_window() - self.assertIsNone(self.autocomplete.autocompletewindow) + acp = self.autocomplete + acp.autocompletewindow = m = Mock() + acp._remove_autocomplete_window() + m.hide_window.assert_called_once() + self.assertIsNone(acp.autocompletewindow) def test_force_open_completions_event(self): - # Test that force_open_completions_event calls _open_completions. - o_cs = Func() - self.autocomplete.open_completions = o_cs - self.autocomplete.force_open_completions_event('event') - self.assertEqual(o_cs.args, (True, False, True)) - - def test_try_open_completions_event(self): - Equal = self.assertEqual - autocomplete = self.autocomplete - trycompletions = self.autocomplete.try_open_completions_event - o_c_l = Func() - autocomplete._open_completions_later = o_c_l - - # _open_completions_later should not be called with no text in editor. - trycompletions('event') - Equal(o_c_l.args, None) - - # _open_completions_later should be called with COMPLETE_ATTRIBUTES (1). - self.text.insert('1.0', 're.') - trycompletions('event') - Equal(o_c_l.args, (False, False, False, 1)) - - # _open_completions_later should be called with COMPLETE_FILES (2). - self.text.delete('1.0', 'end') - self.text.insert('1.0', '"./Lib/') - trycompletions('event') - Equal(o_c_l.args, (False, False, False, 2)) + # Call _open_completions and break. + acp = self.autocomplete + open_c = Func() + acp.open_completions = open_c + self.assertEqual(acp.force_open_completions_event('event'), 'break') + self.assertEqual(open_c.args[0], ac.FORCE) def test_autocomplete_event(self): Equal = self.assertEqual - autocomplete = self.autocomplete + acp = self.autocomplete - # Test that the autocomplete event is ignored if user is pressing a - # modifier key in addition to the tab key. + # Result of autocomplete event: If modified tab, None. ev = Event(mc_state=True) - self.assertIsNone(autocomplete.autocomplete_event(ev)) + self.assertIsNone(acp.autocomplete_event(ev)) del ev.mc_state - # Test that tab after whitespace is ignored. + # If tab after whitespace, None. self.text.insert('1.0', ' """Docstring.\n ') - self.assertIsNone(autocomplete.autocomplete_event(ev)) + self.assertIsNone(acp.autocomplete_event(ev)) self.text.delete('1.0', 'end') - # If autocomplete window is open, complete() method is called. + # If active autocomplete window, complete() and 'break'. self.text.insert('1.0', 're.') - # This must call autocomplete._make_autocomplete_window(). - Equal(self.autocomplete.autocomplete_event(ev), 'break') - - # If autocomplete window is not active or does not exist, - # open_completions is called. Return depends on its return. - autocomplete._remove_autocomplete_window() - o_cs = Func() # .result = None. - autocomplete.open_completions = o_cs - Equal(self.autocomplete.autocomplete_event(ev), None) - Equal(o_cs.args, (False, True, True)) - o_cs.result = True - Equal(self.autocomplete.autocomplete_event(ev), 'break') - Equal(o_cs.args, (False, True, True)) - - def test_open_completions_later(self): - # Test that autocomplete._delayed_completion_id is set. + acp.autocompletewindow = mock = Mock() + mock.is_active = Mock(return_value=True) + Equal(acp.autocomplete_event(ev), 'break') + mock.complete.assert_called_once() + acp.autocompletewindow = None + + # If no active autocomplete window, open_completions(), None/break. + open_c = Func(result=False) + acp.open_completions = open_c + Equal(acp.autocomplete_event(ev), None) + Equal(open_c.args[0], ac.TAB) + open_c.result = True + Equal(acp.autocomplete_event(ev), 'break') + Equal(open_c.args[0], ac.TAB) + + def test_try_open_completions_event(self): + Equal = self.assertEqual + text = self.text acp = self.autocomplete + trycompletions = acp.try_open_completions_event + after = Func(result='after1') + acp.text.after = after + + # If no text or trigger, after not called. + trycompletions() + Equal(after.called, 0) + text.insert('1.0', 're') + trycompletions() + Equal(after.called, 0) + + # Attribute needed, no existing callback. + text.insert('insert', ' re.') acp._delayed_completion_id = None - acp._open_completions_later(False, False, False, ac.COMPLETE_ATTRIBUTES) + trycompletions() + Equal(acp._delayed_completion_index, text.index('insert')) + Equal(after.args, + (acp.popupwait, acp._delayed_open_completions, ac.TRY_A)) cb1 = acp._delayed_completion_id - self.assertTrue(cb1.startswith('after')) - - # Test that cb1 is cancelled and cb2 is new. - acp._open_completions_later(False, False, False, ac.COMPLETE_FILES) - self.assertNotIn(cb1, self.root.tk.call('after', 'info')) - cb2 = acp._delayed_completion_id - self.assertTrue(cb2.startswith('after') and cb2 != cb1) - self.text.after_cancel(cb2) + Equal(cb1, 'after1') + + # File needed, existing callback cancelled. + text.insert('insert', ' "./Lib/') + after.result = 'after2' + cancel = Func() + acp.text.after_cancel = cancel + trycompletions() + Equal(acp._delayed_completion_index, text.index('insert')) + Equal(cancel.args, (cb1,)) + Equal(after.args, + (acp.popupwait, acp._delayed_open_completions, ac.TRY_F)) + Equal(acp._delayed_completion_id, 'after2') def test_delayed_open_completions(self): - # Test that autocomplete._delayed_completion_id set to None - # and that open_completions is not called if the index is not - # equal to _delayed_completion_index. + Equal = self.assertEqual acp = self.autocomplete - acp.open_completions = Func() + open_c = Func() + acp.open_completions = open_c + self.text.insert('1.0', '"dict.') + + # Set autocomplete._delayed_completion_id to None. + # Text index changed, don't call open_completions. acp._delayed_completion_id = 'after' acp._delayed_completion_index = self.text.index('insert+1c') - acp._delayed_open_completions(1, 2, 3) + acp._delayed_open_completions('dummy') self.assertIsNone(acp._delayed_completion_id) - self.assertEqual(acp.open_completions.called, 0) + Equal(open_c.called, 0) - # Test that open_completions is called if indexes match. + # Text index unchanged, call open_completions. acp._delayed_completion_index = self.text.index('insert') - acp._delayed_open_completions(1, 2, 3, ac.COMPLETE_FILES) - self.assertEqual(acp.open_completions.args, (1, 2, 3, 2)) + acp._delayed_open_completions((1, 2, 3, ac.FILES)) + self.assertEqual(open_c.args[0], (1, 2, 3, ac.FILES)) + + def test_oc_cancel_comment(self): + none = self.assertIsNone + acp = self.autocomplete + + # Comment is in neither code or string. + acp._delayed_completion_id = 'after' + after = Func(result='after') + acp.text.after_cancel = after + self.text.insert(1.0, '# comment') + none(acp.open_completions(ac.TAB)) # From 'else' after 'elif'. + none(acp._delayed_completion_id) + + def test_oc_no_list(self): + acp = self.autocomplete + fetch = Func(result=([],[])) + acp.fetch_completions = fetch + self.text.insert('1.0', 'object') + self.assertIsNone(acp.open_completions(ac.TAB)) + self.text.insert('insert', '.') + self.assertIsNone(acp.open_completions(ac.TAB)) + self.assertEqual(fetch.called, 2) + + + def test_open_completions_none(self): + # Test other two None returns. + none = self.assertIsNone + acp = self.autocomplete + + # No object for attributes or need call not allowed. + self.text.insert(1.0, '.') + none(acp.open_completions(ac.TAB)) + self.text.insert('insert', ' int().') + none(acp.open_completions(ac.TAB)) + + # Blank or quote trigger 'if complete ...'. + self.text.delete(1.0, 'end') + self.assertFalse(acp.open_completions(ac.TAB)) + self.text.insert('1.0', '"') + self.assertFalse(acp.open_completions(ac.TAB)) + self.text.delete('1.0', 'end') + + class dummy_acw(): + __init__ = Func() + show_window = Func(result=False) + hide_window = Func() def test_open_completions(self): - # Test completions of files and attributes as well as non-completion - # of errors. - self.text.insert('1.0', 'pr') - self.assertTrue(self.autocomplete.open_completions(False, True, True)) + # Test completions of files and attributes. + acp = self.autocomplete + fetch = Func(result=(['tem'],['tem', '_tem'])) + acp.fetch_completions = fetch + def make_acw(): return self.dummy_acw() + acp._make_autocomplete_window = make_acw + + self.text.insert('1.0', 'int.') + acp.open_completions(ac.TAB) + self.assertIsInstance(acp.autocompletewindow, self.dummy_acw) self.text.delete('1.0', 'end') # Test files. self.text.insert('1.0', '"t') - #self.assertTrue(self.autocomplete.open_completions(False, True, True)) - self.text.delete('1.0', 'end') - - # Test with blank will fail. - self.assertFalse(self.autocomplete.open_completions(False, True, True)) - - # Test with only string quote will fail. - self.text.insert('1.0', '"') - self.assertFalse(self.autocomplete.open_completions(False, True, True)) + self.assertTrue(acp.open_completions(ac.TAB)) self.text.delete('1.0', 'end') def test_fetch_completions(self): @@ -174,21 +224,21 @@ class AutoCompleteTest(unittest.TestCase): # a small list containing non-private variables. # For file completion, a large list containing all files in the path, # and a small list containing files that do not start with '.'. - autocomplete = self.autocomplete - small, large = self.autocomplete.fetch_completions( - '', ac.COMPLETE_ATTRIBUTES) - if __main__.__file__ != ac.__file__: + acp = self.autocomplete + small, large = acp.fetch_completions( + '', ac.ATTRS) + if hasattr(__main__, '__file__') and __main__.__file__ != ac.__file__: self.assertNotIn('AutoComplete', small) # See issue 36405. # Test attributes - s, b = autocomplete.fetch_completions('', ac.COMPLETE_ATTRIBUTES) + s, b = acp.fetch_completions('', ac.ATTRS) self.assertLess(len(small), len(large)) self.assertTrue(all(filter(lambda x: x.startswith('_'), s))) self.assertTrue(any(filter(lambda x: x.startswith('_'), b))) # Test smalll should respect to __all__. with patch.dict('__main__.__dict__', {'__all__': ['a', 'b']}): - s, b = autocomplete.fetch_completions('', ac.COMPLETE_ATTRIBUTES) + s, b = acp.fetch_completions('', ac.ATTRS) self.assertEqual(s, ['a', 'b']) self.assertIn('__name__', b) # From __main__.__dict__ self.assertIn('sum', b) # From __main__.__builtins__.__dict__ @@ -197,7 +247,7 @@ class AutoCompleteTest(unittest.TestCase): mock = Mock() mock._private = Mock() with patch.dict('__main__.__dict__', {'foo': mock}): - s, b = autocomplete.fetch_completions('foo', ac.COMPLETE_ATTRIBUTES) + s, b = acp.fetch_completions('foo', ac.ATTRS) self.assertNotIn('_private', s) self.assertIn('_private', b) self.assertEqual(s, [i for i in sorted(dir(mock)) if i[:1] != '_']) @@ -211,36 +261,36 @@ class AutoCompleteTest(unittest.TestCase): return ['monty', 'python', '.hidden'] with patch.object(os, 'listdir', _listdir): - s, b = autocomplete.fetch_completions('', ac.COMPLETE_FILES) + s, b = acp.fetch_completions('', ac.FILES) self.assertEqual(s, ['bar', 'foo']) self.assertEqual(b, ['.hidden', 'bar', 'foo']) - s, b = autocomplete.fetch_completions('~', ac.COMPLETE_FILES) + s, b = acp.fetch_completions('~', ac.FILES) self.assertEqual(s, ['monty', 'python']) self.assertEqual(b, ['.hidden', 'monty', 'python']) def test_get_entity(self): # Test that a name is in the namespace of sys.modules and # __main__.__dict__. - autocomplete = self.autocomplete + acp = self.autocomplete Equal = self.assertEqual - Equal(self.autocomplete.get_entity('int'), int) + Equal(acp.get_entity('int'), int) # Test name from sys.modules. mock = Mock() with patch.dict('sys.modules', {'tempfile': mock}): - Equal(autocomplete.get_entity('tempfile'), mock) + Equal(acp.get_entity('tempfile'), mock) # Test name from __main__.__dict__. di = {'foo': 10, 'bar': 20} with patch.dict('__main__.__dict__', {'d': di}): - Equal(autocomplete.get_entity('d'), di) + Equal(acp.get_entity('d'), di) # Test name not in namespace. with patch.dict('__main__.__dict__', {}): with self.assertRaises(NameError): - autocomplete.get_entity('not_exist') + acp.get_entity('not_exist') if __name__ == '__main__': diff --git a/lib-python/3/idlelib/idle_test/test_calltip.py b/lib-python/3/idlelib/idle_test/test_calltip.py index 886959b170..d386b5cd81 100644 --- a/lib-python/3/idlelib/idle_test/test_calltip.py +++ b/lib-python/3/idlelib/idle_test/test_calltip.py @@ -219,20 +219,30 @@ bytes() -> empty bytes object''') with self.subTest(meth=meth, mtip=mtip): self.assertEqual(get_spec(meth), mtip) - def test_attribute_exception(self): + def test_buggy_getattr_class(self): class NoCall: - def __getattr__(self, name): - raise BaseException + def __getattr__(self, name): # Not invoked for class attribute. + raise IndexError # Bug. class CallA(NoCall): - def __call__(oui, a, b, c): + def __call__(self, ci): # Bug does not matter. pass class CallB(NoCall): - def __call__(self, ci): + def __call__(oui, a, b, c): # Non-standard 'self'. pass for meth, mtip in ((NoCall, default_tip), (CallA, default_tip), - (NoCall(), ''), (CallA(), '(a, b, c)'), - (CallB(), '(ci)')): + (NoCall(), ''), (CallA(), '(ci)'), + (CallB(), '(a, b, c)')): + with self.subTest(meth=meth, mtip=mtip): + self.assertEqual(get_spec(meth), mtip) + + def test_metaclass_class(self): # Failure case for issue 38689. + class Type(type): # Type() requires 3 type args, returns class. + __class__ = property({}.__getitem__, {}.__setitem__) + class Object(metaclass=Type): + __slots__ = '__class__' + for meth, mtip in ((Type, default_tip), (Object, default_tip), + (Object(), '')): with self.subTest(meth=meth, mtip=mtip): self.assertEqual(get_spec(meth), mtip) diff --git a/lib-python/3/idlelib/idle_test/test_codecontext.py b/lib-python/3/idlelib/idle_test/test_codecontext.py index 6c6893580f..9578cc731a 100644 --- a/lib-python/3/idlelib/idle_test/test_codecontext.py +++ b/lib-python/3/idlelib/idle_test/test_codecontext.py @@ -2,8 +2,9 @@ from idlelib import codecontext import unittest +import unittest.mock from test.support import requires -from tkinter import Tk, Frame, Text, TclError +from tkinter import NSEW, Tk, Frame, Text, TclError from unittest import mock import re @@ -42,6 +43,9 @@ class DummyEditwin: self.text = text self.label = '' + def getlineno(self, index): + return int(float(self.text.index(index))) + def update_menu_label(self, **kwargs): self.label = kwargs['label'] @@ -58,7 +62,7 @@ class CodeContextTest(unittest.TestCase): text.insert('1.0', code_sample) # Need to pack for creation of code context text widget. frame.pack(side='left', fill='both', expand=1) - text.pack(side='top', fill='both', expand=1) + text.grid(row=1, column=1, sticky=NSEW) cls.editor = DummyEditwin(root, frame, text) codecontext.idleConf.userCfg = testcfg @@ -73,8 +77,29 @@ class CodeContextTest(unittest.TestCase): def setUp(self): self.text.yview(0) + self.text['font'] = 'TkFixedFont' self.cc = codecontext.CodeContext(self.editor) + self.highlight_cfg = {"background": '#abcdef', + "foreground": '#123456'} + orig_idleConf_GetHighlight = codecontext.idleConf.GetHighlight + def mock_idleconf_GetHighlight(theme, element): + if element == 'context': + return self.highlight_cfg + return orig_idleConf_GetHighlight(theme, element) + GetHighlight_patcher = unittest.mock.patch.object( + codecontext.idleConf, 'GetHighlight', mock_idleconf_GetHighlight) + GetHighlight_patcher.start() + self.addCleanup(GetHighlight_patcher.stop) + + self.font_override = 'TkFixedFont' + def mock_idleconf_GetFont(root, configType, section): + return self.font_override + GetFont_patcher = unittest.mock.patch.object( + codecontext.idleConf, 'GetFont', mock_idleconf_GetFont) + GetFont_patcher.start() + self.addCleanup(GetFont_patcher.stop) + def tearDown(self): if self.cc.context: self.cc.context.destroy() @@ -89,30 +114,24 @@ class CodeContextTest(unittest.TestCase): eq(cc.editwin, ed) eq(cc.text, ed.text) - eq(cc.textfont, ed.text['font']) + eq(cc.text['font'], ed.text['font']) self.assertIsNone(cc.context) eq(cc.info, [(0, -1, '', False)]) eq(cc.topvisible, 1) - eq(self.root.tk.call('after', 'info', self.cc.t1)[1], 'timer') - eq(self.root.tk.call('after', 'info', self.cc.t2)[1], 'timer') + self.assertIsNone(self.cc.t1) def test_del(self): self.cc.__del__() - with self.assertRaises(TclError) as msg: - self.root.tk.call('after', 'info', self.cc.t1) - self.assertIn("doesn't exist", msg) - with self.assertRaises(TclError) as msg: - self.root.tk.call('after', 'info', self.cc.t2) - self.assertIn("doesn't exist", msg) - # For coverage on the except. Have to delete because the - # above Tcl error is caught by after_cancel. - del self.cc.t1, self.cc.t2 + + def test_del_with_timer(self): + timer = self.cc.t1 = self.text.after(10000, lambda: None) self.cc.__del__() + with self.assertRaises(TclError) as cm: + self.root.tk.call('after', 'info', timer) + self.assertIn("doesn't exist", str(cm.exception)) def test_reload(self): codecontext.CodeContext.reload() - self.assertEqual(self.cc.colors, {'background': 'lightgray', - 'foreground': '#000000'}) self.assertEqual(self.cc.context_depth, 15) def test_toggle_code_context_event(self): @@ -125,18 +144,31 @@ class CodeContextTest(unittest.TestCase): toggle() # Toggle on. - eq(toggle(), 'break') + toggle() self.assertIsNotNone(cc.context) - eq(cc.context['font'], cc.textfont) - eq(cc.context['fg'], cc.colors['foreground']) - eq(cc.context['bg'], cc.colors['background']) + eq(cc.context['font'], self.text['font']) + eq(cc.context['fg'], self.highlight_cfg['foreground']) + eq(cc.context['bg'], self.highlight_cfg['background']) eq(cc.context.get('1.0', 'end-1c'), '') eq(cc.editwin.label, 'Hide Code Context') + eq(self.root.tk.call('after', 'info', self.cc.t1)[1], 'timer') # Toggle off. - eq(toggle(), 'break') + toggle() self.assertIsNone(cc.context) eq(cc.editwin.label, 'Show Code Context') + self.assertIsNone(self.cc.t1) + + # Scroll down and toggle back on. + line11_context = '\n'.join(x[2] for x in cc.get_context(11)[0]) + cc.text.yview(11) + toggle() + eq(cc.context.get('1.0', 'end-1c'), line11_context) + + # Toggle off and on again. + toggle() + toggle() + eq(cc.context.get('1.0', 'end-1c'), line11_context) def test_get_context(self): eq = self.assertEqual @@ -227,7 +259,7 @@ class CodeContextTest(unittest.TestCase): (4, 4, ' def __init__(self, a, b):', 'def')]) eq(cc.topvisible, 5) eq(cc.context.get('1.0', 'end-1c'), 'class C1():\n' - ' def __init__(self, a, b):') + ' def __init__(self, a, b):') # Scroll down to line 11. Last 'def' is removed. cc.text.yview(11) @@ -239,9 +271,9 @@ class CodeContextTest(unittest.TestCase): (10, 8, ' elif a < b:', 'elif')]) eq(cc.topvisible, 12) eq(cc.context.get('1.0', 'end-1c'), 'class C1():\n' - ' def compare(self):\n' - ' if a > b:\n' - ' elif a < b:') + ' def compare(self):\n' + ' if a > b:\n' + ' elif a < b:') # No scroll. No update, even though context_depth changed. cc.update_code_context() @@ -253,9 +285,9 @@ class CodeContextTest(unittest.TestCase): (10, 8, ' elif a < b:', 'elif')]) eq(cc.topvisible, 12) eq(cc.context.get('1.0', 'end-1c'), 'class C1():\n' - ' def compare(self):\n' - ' if a > b:\n' - ' elif a < b:') + ' def compare(self):\n' + ' if a > b:\n' + ' elif a < b:') # Scroll up. cc.text.yview(5) @@ -276,7 +308,7 @@ class CodeContextTest(unittest.TestCase): cc.toggle_code_context_event() # Empty context. - cc.text.yview(f'{2}.0') + cc.text.yview('2.0') cc.update_code_context() eq(cc.topvisible, 2) cc.context.mark_set('insert', '1.5') @@ -284,7 +316,7 @@ class CodeContextTest(unittest.TestCase): eq(cc.topvisible, 1) # 4 lines of context showing. - cc.text.yview(f'{12}.0') + cc.text.yview('12.0') cc.update_code_context() eq(cc.topvisible, 12) cc.context.mark_set('insert', '3.0') @@ -293,13 +325,21 @@ class CodeContextTest(unittest.TestCase): # More context lines than limit. cc.context_depth = 2 - cc.text.yview(f'{12}.0') + cc.text.yview('12.0') cc.update_code_context() eq(cc.topvisible, 12) cc.context.mark_set('insert', '1.0') jump() eq(cc.topvisible, 8) + # Context selection stops jump. + cc.text.yview('5.0') + cc.update_code_context() + cc.context.tag_add('sel', '1.0', '2.0') + cc.context.mark_set('insert', '1.0') + jump() # Without selection, to line 2. + eq(cc.topvisible, 5) + @mock.patch.object(codecontext.CodeContext, 'update_code_context') def test_timer_event(self, mock_update): # Ensure code context is not active. @@ -313,56 +353,62 @@ class CodeContextTest(unittest.TestCase): self.cc.timer_event() mock_update.assert_called() - def test_config_timer_event(self): + def test_font(self): eq = self.assertEqual cc = self.cc - save_font = cc.text['font'] - save_colors = codecontext.CodeContext.colors - test_font = 'FakeFont' + + orig_font = cc.text['font'] + test_font = 'TkTextFont' + self.assertNotEqual(orig_font, test_font) + + # Ensure code context is not active. + if cc.context is not None: + cc.toggle_code_context_event() + + self.font_override = test_font + # Nothing breaks or changes with inactive code context. + cc.update_font() + + # Activate code context, previous font change is immediately effective. + cc.toggle_code_context_event() + eq(cc.context['font'], test_font) + + # Call the font update, change is picked up. + self.font_override = orig_font + cc.update_font() + eq(cc.context['font'], orig_font) + + def test_highlight_colors(self): + eq = self.assertEqual + cc = self.cc + + orig_colors = dict(self.highlight_cfg) test_colors = {'background': '#222222', 'foreground': '#ffff00'} + def assert_colors_are_equal(colors): + eq(cc.context['background'], colors['background']) + eq(cc.context['foreground'], colors['foreground']) + # Ensure code context is not active. if cc.context: cc.toggle_code_context_event() - # Nothing updates on inactive code context. - cc.text['font'] = test_font - codecontext.CodeContext.colors = test_colors - cc.config_timer_event() - eq(cc.textfont, save_font) - eq(cc.contextcolors, save_colors) + self.highlight_cfg = test_colors + # Nothing breaks with inactive code context. + cc.update_highlight_colors() - # Activate code context, but no change to font or color. + # Activate code context, previous colors change is immediately effective. cc.toggle_code_context_event() - cc.text['font'] = save_font - codecontext.CodeContext.colors = save_colors - cc.config_timer_event() - eq(cc.textfont, save_font) - eq(cc.contextcolors, save_colors) - eq(cc.context['font'], save_font) - eq(cc.context['background'], save_colors['background']) - eq(cc.context['foreground'], save_colors['foreground']) - - # Active code context, change font. - cc.text['font'] = test_font - cc.config_timer_event() - eq(cc.textfont, test_font) - eq(cc.contextcolors, save_colors) - eq(cc.context['font'], test_font) - eq(cc.context['background'], save_colors['background']) - eq(cc.context['foreground'], save_colors['foreground']) - - # Active code context, change color. - cc.text['font'] = save_font - codecontext.CodeContext.colors = test_colors - cc.config_timer_event() - eq(cc.textfont, save_font) - eq(cc.contextcolors, test_colors) - eq(cc.context['font'], save_font) - eq(cc.context['background'], test_colors['background']) - eq(cc.context['foreground'], test_colors['foreground']) - codecontext.CodeContext.colors = save_colors - cc.config_timer_event() + assert_colors_are_equal(test_colors) + + # Call colors update with no change to the configured colors. + cc.update_highlight_colors() + assert_colors_are_equal(test_colors) + + # Call the colors update with code context active, change is picked up. + self.highlight_cfg = orig_colors + cc.update_highlight_colors() + assert_colors_are_equal(orig_colors) class HelperFunctionText(unittest.TestCase): diff --git a/lib-python/3/idlelib/idle_test/test_config.py b/lib-python/3/idlelib/idle_test/test_config.py index 255210df7d..697fda5279 100644 --- a/lib-python/3/idlelib/idle_test/test_config.py +++ b/lib-python/3/idlelib/idle_test/test_config.py @@ -159,19 +159,6 @@ class IdleUserConfParserTest(unittest.TestCase): self.assertFalse(parser.IsEmpty()) self.assertCountEqual(parser.sections(), ['Foo']) - def test_remove_file(self): - with tempfile.TemporaryDirectory() as tdir: - path = os.path.join(tdir, 'test.cfg') - parser = self.new_parser(path) - parser.RemoveFile() # Should not raise exception. - - parser.AddSection('Foo') - parser.SetOption('Foo', 'bar', 'true') - parser.Save() - self.assertTrue(os.path.exists(path)) - parser.RemoveFile() - self.assertFalse(os.path.exists(path)) - def test_save(self): with tempfile.TemporaryDirectory() as tdir: path = os.path.join(tdir, 'test.cfg') @@ -233,7 +220,7 @@ class IdleConfTest(unittest.TestCase): @unittest.skipIf(sys.platform.startswith('win'), 'this is test for unix system') def test_get_user_cfg_dir_unix(self): - "Test to get user config directory under unix" + # Test to get user config directory under unix. conf = self.new_config(_utest=True) # Check normal way should success @@ -256,7 +243,7 @@ class IdleConfTest(unittest.TestCase): @unittest.skipIf(not sys.platform.startswith('win'), 'this is test for Windows system') def test_get_user_cfg_dir_windows(self): - "Test to get user config directory under Windows" + # Test to get user config directory under Windows. conf = self.new_config(_utest=True) # Check normal way should success @@ -297,12 +284,12 @@ class IdleConfTest(unittest.TestCase): self.assertIsInstance(user_parser, config.IdleUserConfParser) # Check config path are correct - for config_type, parser in conf.defaultCfg.items(): + for cfg_type, parser in conf.defaultCfg.items(): self.assertEqual(parser.file, - os.path.join(idle_dir, 'config-%s.def' % config_type)) - for config_type, parser in conf.userCfg.items(): + os.path.join(idle_dir, f'config-{cfg_type}.def')) + for cfg_type, parser in conf.userCfg.items(): self.assertEqual(parser.file, - os.path.join(conf.userdir, 'config-%s.cfg' % config_type)) + os.path.join(conf.userdir or '#', f'config-{cfg_type}.cfg')) def test_load_cfg_files(self): conf = self.new_config(_utest=True) @@ -386,7 +373,7 @@ class IdleConfTest(unittest.TestCase): 'background': '#171717'}) def test_get_theme_dict(self): - "XXX: NOT YET DONE" + # TODO: finish. conf = self.mock_config() # These two should be the same diff --git a/lib-python/3/idlelib/idle_test/test_configdialog.py b/lib-python/3/idlelib/idle_test/test_configdialog.py index 37e83439c4..1fea6d41df 100644 --- a/lib-python/3/idlelib/idle_test/test_configdialog.py +++ b/lib-python/3/idlelib/idle_test/test_configdialog.py @@ -8,7 +8,7 @@ requires('gui') import unittest from unittest import mock from idlelib.idle_test.mock_idle import Func -from tkinter import Tk, StringVar, IntVar, BooleanVar, DISABLED, NORMAL +from tkinter import (Tk, StringVar, IntVar, BooleanVar, DISABLED, NORMAL) from idlelib import config from idlelib.configdialog import idleConf, changes, tracers @@ -30,6 +30,7 @@ highpage = changes['highlight'] keyspage = changes['keys'] extpage = changes['extensions'] + def setUpModule(): global root, dialog idleConf.userCfg = testcfg @@ -37,6 +38,7 @@ def setUpModule(): # root.withdraw() # Comment out, see issue 30870 dialog = configdialog.ConfigDialog(root, 'Test', _utest=True) + def tearDownModule(): global root, dialog idleConf.userCfg = usercfg @@ -48,6 +50,58 @@ def tearDownModule(): root = dialog = None +class ConfigDialogTest(unittest.TestCase): + + def test_deactivate_current_config(self): + pass + + def activate_config_changes(self): + pass + + +class ButtonTest(unittest.TestCase): + + def test_click_ok(self): + d = dialog + apply = d.apply = mock.Mock() + destroy = d.destroy = mock.Mock() + d.buttons['Ok'].invoke() + apply.assert_called_once() + destroy.assert_called_once() + del d.destroy, d.apply + + def test_click_apply(self): + d = dialog + deactivate = d.deactivate_current_config = mock.Mock() + save_ext = d.save_all_changed_extensions = mock.Mock() + activate = d.activate_config_changes = mock.Mock() + d.buttons['Apply'].invoke() + deactivate.assert_called_once() + save_ext.assert_called_once() + activate.assert_called_once() + del d.save_all_changed_extensions + del d.activate_config_changes, d.deactivate_current_config + + def test_click_cancel(self): + d = dialog + d.destroy = Func() + changes['main']['something'] = 1 + d.buttons['Cancel'].invoke() + self.assertEqual(changes['main'], {}) + self.assertEqual(d.destroy.called, 1) + del d.destroy + + def test_click_help(self): + dialog.note.select(dialog.keyspage) + with mock.patch.object(configdialog, 'view_text', + new_callable=Func) as view: + dialog.buttons['Help'].invoke() + title, contents = view.kwds['title'], view.kwds['contents'] + self.assertEqual(title, 'Help for IDLE preferences') + self.assertTrue(contents.startswith('When you click') and + contents.endswith('a different name.\n')) + + class FontPageTest(unittest.TestCase): """Test that font widgets enable users to make font changes. @@ -420,6 +474,48 @@ class HighPageTest(unittest.TestCase): eq(d.highlight_target.get(), elem[tag]) eq(d.set_highlight_target.called, count) + def test_highlight_sample_double_click(self): + # Test double click on highlight_sample. + eq = self.assertEqual + d = self.page + + hs = d.highlight_sample + hs.focus_force() + hs.see(1.0) + hs.update_idletasks() + + # Test binding from configdialog. + hs.event_generate('<Enter>', x=0, y=0) + hs.event_generate('<Motion>', x=0, y=0) + # Double click is a sequence of two clicks in a row. + for _ in range(2): + hs.event_generate('<ButtonPress-1>', x=0, y=0) + hs.event_generate('<ButtonRelease-1>', x=0, y=0) + + eq(hs.tag_ranges('sel'), ()) + + def test_highlight_sample_b1_motion(self): + # Test button motion on highlight_sample. + eq = self.assertEqual + d = self.page + + hs = d.highlight_sample + hs.focus_force() + hs.see(1.0) + hs.update_idletasks() + + x, y, dx, dy, offset = hs.dlineinfo('1.0') + + # Test binding from configdialog. + hs.event_generate('<Leave>') + hs.event_generate('<Enter>') + hs.event_generate('<Motion>', x=x, y=y) + hs.event_generate('<ButtonPress-1>', x=x, y=y) + hs.event_generate('<B1-Motion>', x=dx, y=dy) + hs.event_generate('<ButtonRelease-1>', x=dx, y=dy) + + eq(hs.tag_ranges('sel'), ()) + def test_set_theme_type(self): eq = self.assertEqual d = self.page @@ -648,8 +744,13 @@ class HighPageTest(unittest.TestCase): idleConf.userCfg['highlight'].SetOption(theme_name, 'name', 'value') highpage[theme_name] = {'option': 'True'} + theme_name2 = 'other theme' + idleConf.userCfg['highlight'].SetOption(theme_name2, 'name', 'value') + highpage[theme_name2] = {'option': 'False'} + # Force custom theme. - d.theme_source.set(False) + d.custom_theme_on.state(('!disabled',)) + d.custom_theme_on.invoke() d.custom_name.set(theme_name) # Cancel deletion. @@ -657,7 +758,7 @@ class HighPageTest(unittest.TestCase): d.button_delete_custom.invoke() eq(yesno.called, 1) eq(highpage[theme_name], {'option': 'True'}) - eq(idleConf.GetSectionList('user', 'highlight'), ['spam theme']) + eq(idleConf.GetSectionList('user', 'highlight'), [theme_name, theme_name2]) eq(dialog.deactivate_current_config.called, 0) eq(dialog.activate_config_changes.called, 0) eq(d.set_theme_type.called, 0) @@ -667,13 +768,26 @@ class HighPageTest(unittest.TestCase): d.button_delete_custom.invoke() eq(yesno.called, 2) self.assertNotIn(theme_name, highpage) - eq(idleConf.GetSectionList('user', 'highlight'), []) - eq(d.custom_theme_on.state(), ('disabled',)) - eq(d.custom_name.get(), '- no custom themes -') + eq(idleConf.GetSectionList('user', 'highlight'), [theme_name2]) + eq(d.custom_theme_on.state(), ()) + eq(d.custom_name.get(), theme_name2) eq(dialog.deactivate_current_config.called, 1) eq(dialog.activate_config_changes.called, 1) eq(d.set_theme_type.called, 1) + # Confirm deletion of second theme - empties list. + d.custom_name.set(theme_name2) + yesno.result = True + d.button_delete_custom.invoke() + eq(yesno.called, 3) + self.assertNotIn(theme_name, highpage) + eq(idleConf.GetSectionList('user', 'highlight'), []) + eq(d.custom_theme_on.state(), ('disabled',)) + eq(d.custom_name.get(), '- no custom themes -') + eq(dialog.deactivate_current_config.called, 2) + eq(dialog.activate_config_changes.called, 2) + eq(d.set_theme_type.called, 2) + del dialog.activate_config_changes, dialog.deactivate_current_config del d.askyesno @@ -1041,8 +1155,13 @@ class KeysPageTest(unittest.TestCase): idleConf.userCfg['keys'].SetOption(keyset_name, 'name', 'value') keyspage[keyset_name] = {'option': 'True'} + keyset_name2 = 'other key set' + idleConf.userCfg['keys'].SetOption(keyset_name2, 'name', 'value') + keyspage[keyset_name2] = {'option': 'False'} + # Force custom keyset. - d.keyset_source.set(False) + d.custom_keyset_on.state(('!disabled',)) + d.custom_keyset_on.invoke() d.custom_name.set(keyset_name) # Cancel deletion. @@ -1050,7 +1169,7 @@ class KeysPageTest(unittest.TestCase): d.button_delete_custom_keys.invoke() eq(yesno.called, 1) eq(keyspage[keyset_name], {'option': 'True'}) - eq(idleConf.GetSectionList('user', 'keys'), ['spam key set']) + eq(idleConf.GetSectionList('user', 'keys'), [keyset_name, keyset_name2]) eq(dialog.deactivate_current_config.called, 0) eq(dialog.activate_config_changes.called, 0) eq(d.set_keys_type.called, 0) @@ -1060,13 +1179,26 @@ class KeysPageTest(unittest.TestCase): d.button_delete_custom_keys.invoke() eq(yesno.called, 2) self.assertNotIn(keyset_name, keyspage) - eq(idleConf.GetSectionList('user', 'keys'), []) - eq(d.custom_keyset_on.state(), ('disabled',)) - eq(d.custom_name.get(), '- no custom keys -') + eq(idleConf.GetSectionList('user', 'keys'), [keyset_name2]) + eq(d.custom_keyset_on.state(), ()) + eq(d.custom_name.get(), keyset_name2) eq(dialog.deactivate_current_config.called, 1) eq(dialog.activate_config_changes.called, 1) eq(d.set_keys_type.called, 1) + # Confirm deletion of second keyset - empties list. + d.custom_name.set(keyset_name2) + yesno.result = True + d.button_delete_custom_keys.invoke() + eq(yesno.called, 3) + self.assertNotIn(keyset_name, keyspage) + eq(idleConf.GetSectionList('user', 'keys'), []) + eq(d.custom_keyset_on.state(), ('disabled',)) + eq(d.custom_name.get(), '- no custom keys -') + eq(dialog.deactivate_current_config.called, 2) + eq(dialog.activate_config_changes.called, 2) + eq(d.set_keys_type.called, 2) + del dialog.activate_config_changes, dialog.deactivate_current_config del d.askyesno @@ -1135,6 +1267,10 @@ class GenPageTest(unittest.TestCase): d.win_width_int.insert(0, '11') self.assertEqual(mainpage, {'EditorWindow': {'width': '11'}}) + def test_cursor_blink(self): + self.page.cursor_blink_bool.invoke() + self.assertEqual(mainpage, {'EditorWindow': {'cursor-blink': 'False'}}) + def test_autocomplete_wait(self): self.page.auto_wait_int.delete(0, 'end') self.page.auto_wait_int.insert(0, '11') diff --git a/lib-python/3/idlelib/idle_test/test_editor.py b/lib-python/3/idlelib/idle_test/test_editor.py index 12bc847366..443dcf0216 100644 --- a/lib-python/3/idlelib/idle_test/test_editor.py +++ b/lib-python/3/idlelib/idle_test/test_editor.py @@ -2,8 +2,10 @@ from idlelib import editor import unittest +from collections import namedtuple from test.support import requires from tkinter import Tk +from idlelib.idle_test.mock_idle import Func Editor = editor.EditorWindow @@ -30,16 +32,188 @@ class EditorWindowTest(unittest.TestCase): e._close() -class EditorFunctionTest(unittest.TestCase): +class TestGetLineIndent(unittest.TestCase): + def test_empty_lines(self): + for tabwidth in [1, 2, 4, 6, 8]: + for line in ['', '\n']: + with self.subTest(line=line, tabwidth=tabwidth): + self.assertEqual( + editor.get_line_indent(line, tabwidth=tabwidth), + (0, 0), + ) - def test_filename_to_unicode(self): - func = Editor._filename_to_unicode - class dummy(): - filesystemencoding = 'utf-8' - pairs = (('abc', 'abc'), ('a\U00011111c', 'a\ufffdc'), - (b'abc', 'abc'), (b'a\xf0\x91\x84\x91c', 'a\ufffdc')) - for inp, out in pairs: - self.assertEqual(func(dummy, inp), out) + def test_tabwidth_4(self): + # (line, (raw, effective)) + tests = (('no spaces', (0, 0)), + # Internal space isn't counted. + (' space test', (4, 4)), + ('\ttab test', (1, 4)), + ('\t\tdouble tabs test', (2, 8)), + # Different results when mixing tabs and spaces. + (' \tmixed test', (5, 8)), + (' \t mixed test', (5, 6)), + ('\t mixed test', (5, 8)), + # Spaces not divisible by tabwidth. + (' \tmixed test', (3, 4)), + (' \t mixed test', (3, 5)), + ('\t mixed test', (3, 6)), + # Only checks spaces and tabs. + ('\nnewline test', (0, 0))) + + for line, expected in tests: + with self.subTest(line=line): + self.assertEqual( + editor.get_line_indent(line, tabwidth=4), + expected, + ) + + def test_tabwidth_8(self): + # (line, (raw, effective)) + tests = (('no spaces', (0, 0)), + # Internal space isn't counted. + (' space test', (8, 8)), + ('\ttab test', (1, 8)), + ('\t\tdouble tabs test', (2, 16)), + # Different results when mixing tabs and spaces. + (' \tmixed test', (9, 16)), + (' \t mixed test', (9, 10)), + ('\t mixed test', (9, 16)), + # Spaces not divisible by tabwidth. + (' \tmixed test', (3, 8)), + (' \t mixed test', (3, 9)), + ('\t mixed test', (3, 10)), + # Only checks spaces and tabs. + ('\nnewline test', (0, 0))) + + for line, expected in tests: + with self.subTest(line=line): + self.assertEqual( + editor.get_line_indent(line, tabwidth=8), + expected, + ) + + +def insert(text, string): + text.delete('1.0', 'end') + text.insert('end', string) + text.update() # Force update for colorizer to finish. + + +class IndentAndNewlineTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.window = Editor(root=cls.root) + cls.window.indentwidth = 2 + cls.window.tabwidth = 2 + + @classmethod + def tearDownClass(cls): + cls.window._close() + del cls.window + cls.root.update_idletasks() + for id in cls.root.tk.call('after', 'info'): + cls.root.after_cancel(id) + cls.root.destroy() + del cls.root + + def test_indent_and_newline_event(self): + eq = self.assertEqual + w = self.window + text = w.text + get = text.get + nl = w.newline_and_indent_event + + TestInfo = namedtuple('Tests', ['label', 'text', 'expected', 'mark']) + + tests = (TestInfo('Empty line inserts with no indent.', + ' \n def __init__(self):', + '\n \n def __init__(self):\n', + '1.end'), + TestInfo('Inside bracket before space, deletes space.', + ' def f1(self, a, b):', + ' def f1(self,\n a, b):\n', + '1.14'), + TestInfo('Inside bracket after space, deletes space.', + ' def f1(self, a, b):', + ' def f1(self,\n a, b):\n', + '1.15'), + TestInfo('Inside string with one line - no indent.', + ' """Docstring."""', + ' """Docstring.\n"""\n', + '1.15'), + TestInfo('Inside string with more than one line.', + ' """Docstring.\n Docstring Line 2"""', + ' """Docstring.\n Docstring Line 2\n """\n', + '2.18'), + TestInfo('Backslash with one line.', + 'a =\\', + 'a =\\\n \n', + '1.end'), + TestInfo('Backslash with more than one line.', + 'a =\\\n multiline\\', + 'a =\\\n multiline\\\n \n', + '2.end'), + TestInfo('Block opener - indents +1 level.', + ' def f1(self):\n pass', + ' def f1(self):\n \n pass\n', + '1.end'), + TestInfo('Block closer - dedents -1 level.', + ' def f1(self):\n pass', + ' def f1(self):\n pass\n \n', + '2.end'), + ) + + w.prompt_last_line = '' + for test in tests: + with self.subTest(label=test.label): + insert(text, test.text) + text.mark_set('insert', test.mark) + nl(event=None) + eq(get('1.0', 'end'), test.expected) + + # Selected text. + insert(text, ' def f1(self, a, b):\n return a + b') + text.tag_add('sel', '1.17', '1.end') + nl(None) + # Deletes selected text before adding new line. + eq(get('1.0', 'end'), ' def f1(self, a,\n \n return a + b\n') + + # Preserves the whitespace in shell prompt. + w.prompt_last_line = '>>> ' + insert(text, '>>> \t\ta =') + text.mark_set('insert', '1.5') + nl(None) + eq(get('1.0', 'end'), '>>> \na =\n') + + +class RMenuTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.window = Editor(root=cls.root) + + @classmethod + def tearDownClass(cls): + cls.window._close() + del cls.window + cls.root.update_idletasks() + for id in cls.root.tk.call('after', 'info'): + cls.root.after_cancel(id) + cls.root.destroy() + del cls.root + + class DummyRMenu: + def tk_popup(x, y): pass + + def test_rclick(self): + pass if __name__ == '__main__': diff --git a/lib-python/3/idlelib/idle_test/test_paragraph.py b/lib-python/3/idlelib/idle_test/test_format.py index 0cb966fb96..a79bb51508 100644 --- a/lib-python/3/idlelib/idle_test/test_paragraph.py +++ b/lib-python/3/idlelib/idle_test/test_format.py @@ -1,10 +1,12 @@ -"Test paragraph, coverage 76%." +"Test format, coverage 99%." -from idlelib import paragraph as pg +from idlelib import format as ft import unittest +from unittest import mock from test.support import requires from tkinter import Tk, Text from idlelib.editor import EditorWindow +from idlelib.idle_test.mock_idle import Editor as MockEditor class Is_Get_Test(unittest.TestCase): @@ -16,26 +18,26 @@ class Is_Get_Test(unittest.TestCase): leadingws_nocomment = ' This is not a comment' def test_is_all_white(self): - self.assertTrue(pg.is_all_white('')) - self.assertTrue(pg.is_all_white('\t\n\r\f\v')) - self.assertFalse(pg.is_all_white(self.test_comment)) + self.assertTrue(ft.is_all_white('')) + self.assertTrue(ft.is_all_white('\t\n\r\f\v')) + self.assertFalse(ft.is_all_white(self.test_comment)) def test_get_indent(self): Equal = self.assertEqual - Equal(pg.get_indent(self.test_comment), '') - Equal(pg.get_indent(self.trailingws_comment), '') - Equal(pg.get_indent(self.leadingws_comment), ' ') - Equal(pg.get_indent(self.leadingws_nocomment), ' ') + Equal(ft.get_indent(self.test_comment), '') + Equal(ft.get_indent(self.trailingws_comment), '') + Equal(ft.get_indent(self.leadingws_comment), ' ') + Equal(ft.get_indent(self.leadingws_nocomment), ' ') def test_get_comment_header(self): Equal = self.assertEqual # Test comment strings - Equal(pg.get_comment_header(self.test_comment), '#') - Equal(pg.get_comment_header(self.trailingws_comment), '#') - Equal(pg.get_comment_header(self.leadingws_comment), ' #') + Equal(ft.get_comment_header(self.test_comment), '#') + Equal(ft.get_comment_header(self.trailingws_comment), '#') + Equal(ft.get_comment_header(self.leadingws_comment), ' #') # Test non-comment strings - Equal(pg.get_comment_header(self.leadingws_nocomment), ' ') - Equal(pg.get_comment_header(self.test_nocomment), '') + Equal(ft.get_comment_header(self.leadingws_nocomment), ' ') + Equal(ft.get_comment_header(self.test_nocomment), '') class FindTest(unittest.TestCase): @@ -63,7 +65,7 @@ class FindTest(unittest.TestCase): linelength = int(text.index("%d.end" % line).split('.')[1]) for col in (0, linelength//2, linelength): tempindex = "%d.%d" % (line, col) - self.assertEqual(pg.find_paragraph(text, tempindex), expected) + self.assertEqual(ft.find_paragraph(text, tempindex), expected) text.delete('1.0', 'end') def test_find_comment(self): @@ -162,7 +164,7 @@ class ReformatFunctionTest(unittest.TestCase): def test_reformat_paragraph(self): Equal = self.assertEqual - reform = pg.reformat_paragraph + reform = ft.reformat_paragraph hw = "O hello world" Equal(reform(' ', 1), ' ') Equal(reform("Hello world", 20), "Hello world") @@ -193,7 +195,7 @@ class ReformatCommentTest(unittest.TestCase): test_string = ( " \"\"\"this is a test of a reformat for a triple quoted string" " will it reformat to less than 70 characters for me?\"\"\"") - result = pg.reformat_comment(test_string, 70, " ") + result = ft.reformat_comment(test_string, 70, " ") expected = ( " \"\"\"this is a test of a reformat for a triple quoted string will it\n" " reformat to less than 70 characters for me?\"\"\"") @@ -202,7 +204,7 @@ class ReformatCommentTest(unittest.TestCase): test_comment = ( "# this is a test of a reformat for a triple quoted string will " "it reformat to less than 70 characters for me?") - result = pg.reformat_comment(test_comment, 70, "#") + result = ft.reformat_comment(test_comment, 70, "#") expected = ( "# this is a test of a reformat for a triple quoted string will it\n" "# reformat to less than 70 characters for me?") @@ -211,7 +213,7 @@ class ReformatCommentTest(unittest.TestCase): class FormatClassTest(unittest.TestCase): def test_init_close(self): - instance = pg.FormatParagraph('editor') + instance = ft.FormatParagraph('editor') self.assertEqual(instance.editwin, 'editor') instance.close() self.assertEqual(instance.editwin, None) @@ -273,7 +275,7 @@ class FormatEventTest(unittest.TestCase): cls.root.withdraw() editor = Editor(root=cls.root) cls.text = editor.text.text # Test code does not need the wrapper. - cls.formatter = pg.FormatParagraph(editor).format_paragraph_event + cls.formatter = ft.FormatParagraph(editor).format_paragraph_event # Sets the insert mark just after the re-wrapped and inserted text. @classmethod @@ -375,5 +377,292 @@ class FormatEventTest(unittest.TestCase): ## text.delete('1.0', 'end') +class DummyEditwin: + def __init__(self, root, text): + self.root = root + self.text = text + self.indentwidth = 4 + self.tabwidth = 4 + self.usetabs = False + self.context_use_ps1 = True + + _make_blanks = EditorWindow._make_blanks + get_selection_indices = EditorWindow.get_selection_indices + + +class FormatRegionTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.text = Text(cls.root) + cls.text.undo_block_start = mock.Mock() + cls.text.undo_block_stop = mock.Mock() + cls.editor = DummyEditwin(cls.root, cls.text) + cls.formatter = ft.FormatRegion(cls.editor) + + @classmethod + def tearDownClass(cls): + del cls.text, cls.formatter, cls.editor + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def setUp(self): + self.text.insert('1.0', self.code_sample) + + def tearDown(self): + self.text.delete('1.0', 'end') + + code_sample = """\ +# WS line needed for test. +class C1(): + # Class comment. + def __init__(self, a, b): + self.a = a + self.b = b + + def compare(self): + if a > b: + return a + elif a < b: + return b + else: + return None +""" + + def test_get_region(self): + get = self.formatter.get_region + text = self.text + eq = self.assertEqual + + # Add selection. + text.tag_add('sel', '7.0', '10.0') + expected_lines = ['', + ' def compare(self):', + ' if a > b:', + ''] + eq(get(), ('7.0', '10.0', '\n'.join(expected_lines), expected_lines)) + + # Remove selection. + text.tag_remove('sel', '1.0', 'end') + eq(get(), ('15.0', '16.0', '\n', ['', ''])) + + def test_set_region(self): + set_ = self.formatter.set_region + text = self.text + eq = self.assertEqual + + save_bell = text.bell + text.bell = mock.Mock() + line6 = self.code_sample.splitlines()[5] + line10 = self.code_sample.splitlines()[9] + + text.tag_add('sel', '6.0', '11.0') + head, tail, chars, lines = self.formatter.get_region() + + # No changes. + set_(head, tail, chars, lines) + text.bell.assert_called_once() + eq(text.get('6.0', '11.0'), chars) + eq(text.get('sel.first', 'sel.last'), chars) + text.tag_remove('sel', '1.0', 'end') + + # Alter selected lines by changing lines and adding a newline. + newstring = 'added line 1\n\n\n\n' + newlines = newstring.split('\n') + set_('7.0', '10.0', chars, newlines) + # Selection changed. + eq(text.get('sel.first', 'sel.last'), newstring) + # Additional line added, so last index is changed. + eq(text.get('7.0', '11.0'), newstring) + # Before and after lines unchanged. + eq(text.get('6.0', '7.0-1c'), line6) + eq(text.get('11.0', '12.0-1c'), line10) + text.tag_remove('sel', '1.0', 'end') + + text.bell = save_bell + + def test_indent_region_event(self): + indent = self.formatter.indent_region_event + text = self.text + eq = self.assertEqual + + text.tag_add('sel', '7.0', '10.0') + indent() + # Blank lines aren't affected by indent. + eq(text.get('7.0', '10.0'), ('\n def compare(self):\n if a > b:\n')) + + def test_dedent_region_event(self): + dedent = self.formatter.dedent_region_event + text = self.text + eq = self.assertEqual + + text.tag_add('sel', '7.0', '10.0') + dedent() + # Blank lines aren't affected by dedent. + eq(text.get('7.0', '10.0'), ('\ndef compare(self):\n if a > b:\n')) + + def test_comment_region_event(self): + comment = self.formatter.comment_region_event + text = self.text + eq = self.assertEqual + + text.tag_add('sel', '7.0', '10.0') + comment() + eq(text.get('7.0', '10.0'), ('##\n## def compare(self):\n## if a > b:\n')) + + def test_uncomment_region_event(self): + comment = self.formatter.comment_region_event + uncomment = self.formatter.uncomment_region_event + text = self.text + eq = self.assertEqual + + text.tag_add('sel', '7.0', '10.0') + comment() + uncomment() + eq(text.get('7.0', '10.0'), ('\n def compare(self):\n if a > b:\n')) + + # Only remove comments at the beginning of a line. + text.tag_remove('sel', '1.0', 'end') + text.tag_add('sel', '3.0', '4.0') + uncomment() + eq(text.get('3.0', '3.end'), (' # Class comment.')) + + self.formatter.set_region('3.0', '4.0', '', ['# Class comment.', '']) + uncomment() + eq(text.get('3.0', '3.end'), (' Class comment.')) + + @mock.patch.object(ft.FormatRegion, "_asktabwidth") + def test_tabify_region_event(self, _asktabwidth): + tabify = self.formatter.tabify_region_event + text = self.text + eq = self.assertEqual + + text.tag_add('sel', '7.0', '10.0') + # No tabwidth selected. + _asktabwidth.return_value = None + self.assertIsNone(tabify()) + + _asktabwidth.return_value = 3 + self.assertIsNotNone(tabify()) + eq(text.get('7.0', '10.0'), ('\n\t def compare(self):\n\t\t if a > b:\n')) + + @mock.patch.object(ft.FormatRegion, "_asktabwidth") + def test_untabify_region_event(self, _asktabwidth): + untabify = self.formatter.untabify_region_event + text = self.text + eq = self.assertEqual + + text.tag_add('sel', '7.0', '10.0') + # No tabwidth selected. + _asktabwidth.return_value = None + self.assertIsNone(untabify()) + + _asktabwidth.return_value = 2 + self.formatter.tabify_region_event() + _asktabwidth.return_value = 3 + self.assertIsNotNone(untabify()) + eq(text.get('7.0', '10.0'), ('\n def compare(self):\n if a > b:\n')) + + @mock.patch.object(ft, "askinteger") + def test_ask_tabwidth(self, askinteger): + ask = self.formatter._asktabwidth + askinteger.return_value = 10 + self.assertEqual(ask(), 10) + + +class IndentsTest(unittest.TestCase): + + @mock.patch.object(ft, "askyesno") + def test_toggle_tabs(self, askyesno): + editor = DummyEditwin(None, None) # usetabs == False. + indents = ft.Indents(editor) + askyesno.return_value = True + + indents.toggle_tabs_event(None) + self.assertEqual(editor.usetabs, True) + self.assertEqual(editor.indentwidth, 8) + + indents.toggle_tabs_event(None) + self.assertEqual(editor.usetabs, False) + self.assertEqual(editor.indentwidth, 8) + + @mock.patch.object(ft, "askinteger") + def test_change_indentwidth(self, askinteger): + editor = DummyEditwin(None, None) # indentwidth == 4. + indents = ft.Indents(editor) + + askinteger.return_value = None + indents.change_indentwidth_event(None) + self.assertEqual(editor.indentwidth, 4) + + askinteger.return_value = 3 + indents.change_indentwidth_event(None) + self.assertEqual(editor.indentwidth, 3) + + askinteger.return_value = 5 + editor.usetabs = True + indents.change_indentwidth_event(None) + self.assertEqual(editor.indentwidth, 3) + + +class RstripTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + cls.root.withdraw() + cls.text = Text(cls.root) + cls.editor = MockEditor(text=cls.text) + cls.do_rstrip = ft.Rstrip(cls.editor).do_rstrip + + @classmethod + def tearDownClass(cls): + del cls.text, cls.do_rstrip, cls.editor + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def tearDown(self): + self.text.delete('1.0', 'end-1c') + + def test_rstrip_lines(self): + original = ( + "Line with an ending tab \n" + "Line ending in 5 spaces \n" + "Linewithnospaces\n" + " indented line\n" + " indented line with trailing space \n" + " \n") + stripped = ( + "Line with an ending tab\n" + "Line ending in 5 spaces\n" + "Linewithnospaces\n" + " indented line\n" + " indented line with trailing space\n") + + self.text.insert('1.0', original) + self.do_rstrip() + self.assertEqual(self.text.get('1.0', 'insert'), stripped) + + def test_rstrip_end(self): + text = self.text + for code in ('', '\n', '\n\n\n'): + with self.subTest(code=code): + text.insert('1.0', code) + self.do_rstrip() + self.assertEqual(text.get('1.0','end-1c'), '') + for code in ('a\n', 'a\n\n', 'a\n\n\n'): + with self.subTest(code=code): + text.delete('1.0', 'end-1c') + text.insert('1.0', code) + self.do_rstrip() + self.assertEqual(text.get('1.0','end-1c'), 'a\n') + + if __name__ == '__main__': unittest.main(verbosity=2, exit=2) diff --git a/lib-python/3/idlelib/idle_test/test_iomenu.py b/lib-python/3/idlelib/idle_test/test_iomenu.py index 743a05b3c3..99f4048796 100644 --- a/lib-python/3/idlelib/idle_test/test_iomenu.py +++ b/lib-python/3/idlelib/idle_test/test_iomenu.py @@ -1,14 +1,13 @@ -"Test , coverage 16%." +"Test , coverage 17%." from idlelib import iomenu import unittest from test.support import requires from tkinter import Tk - from idlelib.editor import EditorWindow -class IOBindigTest(unittest.TestCase): +class IOBindingTest(unittest.TestCase): @classmethod def setUpClass(cls): @@ -16,9 +15,11 @@ class IOBindigTest(unittest.TestCase): cls.root = Tk() cls.root.withdraw() cls.editwin = EditorWindow(root=cls.root) + cls.io = iomenu.IOBinding(cls.editwin) @classmethod def tearDownClass(cls): + cls.io.close() cls.editwin._close() del cls.editwin cls.root.update_idletasks() @@ -28,9 +29,20 @@ class IOBindigTest(unittest.TestCase): del cls.root def test_init(self): - io = iomenu.IOBinding(self.editwin) - self.assertIs(io.editwin, self.editwin) - io.close + self.assertIs(self.io.editwin, self.editwin) + + def test_fixnewlines_end(self): + eq = self.assertEqual + io = self.io + fix = io.fixnewlines + text = io.editwin.text + self.editwin.interp = None + eq(fix(), '') + del self.editwin.interp + text.insert(1.0, 'a') + eq(fix(), 'a'+io.eol_convention) + eq(text.get('1.0', 'end-1c'), 'a\n') + eq(fix(), 'a'+io.eol_convention) if __name__ == '__main__': diff --git a/lib-python/3/idlelib/idle_test/test_multicall.py b/lib-python/3/idlelib/idle_test/test_multicall.py index 68156a743d..ba582bb3ca 100644 --- a/lib-python/3/idlelib/idle_test/test_multicall.py +++ b/lib-python/3/idlelib/idle_test/test_multicall.py @@ -35,6 +35,14 @@ class MultiCallTest(unittest.TestCase): mctext = self.mc(self.root) self.assertIsInstance(mctext._MultiCall__binders, list) + def test_yview(self): + # Added for tree.wheel_event + # (it depends on yview to not be overriden) + mc = self.mc + self.assertIs(mc.yview, Text.yview) + mctext = self.mc(self.root) + self.assertIs(mctext.yview.__func__, Text.yview) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/lib-python/3/idlelib/idle_test/test_pyparse.py b/lib-python/3/idlelib/idle_test/test_pyparse.py index 479b84a216..f21baf7534 100644 --- a/lib-python/3/idlelib/idle_test/test_pyparse.py +++ b/lib-python/3/idlelib/idle_test/test_pyparse.py @@ -18,7 +18,7 @@ class ParseMapTest(unittest.TestCase): # trans is the production instance of ParseMap, used in _study1 parser = pyparse.Parser(4, 4) self.assertEqual('\t a([{b}])b"c\'d\n'.translate(pyparse.trans), - 'xxx(((x)))x"x\'x\n') + 'xxx(((x)))x"x\'x\n') class PyParseTest(unittest.TestCase): @@ -58,17 +58,32 @@ class PyParseTest(unittest.TestCase): p = self.parser setcode = p.set_code start = p.find_good_parse_start + def char_in_string_false(index): return False + + # First line starts with 'def' and ends with ':', then 0 is the pos. + setcode('def spam():\n') + eq(start(char_in_string_false), 0) + + # First line begins with a keyword in the list and ends + # with an open brace, then 0 is the pos. This is how + # hyperparser calls this function as the newline is not added + # in the editor, but rather on the call to setcode. + setcode('class spam( ' + ' \n') + eq(start(char_in_string_false), 0) # Split def across lines. setcode('"""This is a module docstring"""\n' - 'class C():\n' - ' def __init__(self, a,\n' - ' b=True):\n' - ' pass\n' - ) + 'class C():\n' + ' def __init__(self, a,\n' + ' b=True):\n' + ' pass\n' + ) - # No value sent for is_char_in_string(). - self.assertIsNone(start()) + # Passing no value or non-callable should fail (issue 32989). + with self.assertRaises(TypeError): + start() + with self.assertRaises(TypeError): + start(False) # Make text look like a string. This returns pos as the start # position, but it's set to None. @@ -76,7 +91,7 @@ class PyParseTest(unittest.TestCase): # Make all text look like it's not in a string. This means that it # found a good start position. - eq(start(is_char_in_string=lambda index: False), 44) + eq(start(char_in_string_false), 44) # If the beginning of the def line is not in a string, then it # returns that as the index. @@ -91,11 +106,11 @@ class PyParseTest(unittest.TestCase): # Code without extra line break in def line - mostly returns the same # values. setcode('"""This is a module docstring"""\n' - 'class C():\n' - ' def __init__(self, a, b=True):\n' - ' pass\n' - ) - eq(start(is_char_in_string=lambda index: False), 44) + 'class C():\n' + ' def __init__(self, a, b=True):\n' + ' pass\n' + ) + eq(start(char_in_string_false), 44) eq(start(is_char_in_string=lambda index: index > 44), 44) eq(start(is_char_in_string=lambda index: index >= 44), 33) # When the def line isn't split, this returns which doesn't match the @@ -206,8 +221,8 @@ class PyParseTest(unittest.TestCase): 'openbracket', 'bracketing']) tests = ( TestInfo('', 0, 0, '', None, ((0, 0),)), - TestInfo("'''This is a multiline continutation docstring.\n\n", - 0, 49, "'", None, ((0, 0), (0, 1), (49, 0))), + TestInfo("'''This is a multiline continuation docstring.\n\n", + 0, 48, "'", None, ((0, 0), (0, 1), (48, 0))), TestInfo(' # Comment\\\n', 0, 12, '', None, ((0, 0), (1, 1), (12, 0))), # A comment without a space is a special case diff --git a/lib-python/3/idlelib/idle_test/test_pyshell.py b/lib-python/3/idlelib/idle_test/test_pyshell.py index 581444ca5e..4a096676f2 100644 --- a/lib-python/3/idlelib/idle_test/test_pyshell.py +++ b/lib-python/3/idlelib/idle_test/test_pyshell.py @@ -7,6 +7,28 @@ from test.support import requires from tkinter import Tk +class FunctionTest(unittest.TestCase): + # Test stand-alone module level non-gui functions. + + def test_restart_line_wide(self): + eq = self.assertEqual + for file, mul, extra in (('', 22, ''), ('finame', 21, '=')): + width = 60 + bar = mul * '=' + with self.subTest(file=file, bar=bar): + file = file or 'Shell' + line = pyshell.restart_line(width, file) + eq(len(line), width) + eq(line, f"{bar+extra} RESTART: {file} {bar}") + + def test_restart_line_narrow(self): + expect, taglen = "= RESTART: Shell", 16 + for width in (taglen-1, taglen, taglen+1): + with self.subTest(width=width): + self.assertEqual(pyshell.restart_line(width, ''), expect) + self.assertEqual(pyshell.restart_line(taglen+2, ''), expect+' =') + + class PyShellFileListTest(unittest.TestCase): @classmethod diff --git a/lib-python/3/idlelib/idle_test/test_query.py b/lib-python/3/idlelib/idle_test/test_query.py index 3b444de15d..6d026cb532 100644 --- a/lib-python/3/idlelib/idle_test/test_query.py +++ b/lib-python/3/idlelib/idle_test/test_query.py @@ -12,7 +12,7 @@ HelpSource htests. These are run by running query.py. from idlelib import query import unittest from test.support import requires -from tkinter import Tk +from tkinter import Tk, END import sys from unittest import mock @@ -138,6 +138,33 @@ class ModuleNameTest(unittest.TestCase): self.assertEqual(dialog.entry_error['text'], '') +class GotoTest(unittest.TestCase): + "Test Goto subclass of Query." + + class Dummy_ModuleName: + entry_ok = query.Goto.entry_ok # Function being tested. + def __init__(self, dummy_entry): + self.entry = Var(value=dummy_entry) + self.entry_error = {'text': ''} + def showerror(self, message): + self.entry_error['text'] = message + + def test_bogus_goto(self): + dialog = self.Dummy_ModuleName('a') + self.assertEqual(dialog.entry_ok(), None) + self.assertIn('not a base 10 integer', dialog.entry_error['text']) + + def test_bad_goto(self): + dialog = self.Dummy_ModuleName('0') + self.assertEqual(dialog.entry_ok(), None) + self.assertIn('not a positive integer', dialog.entry_error['text']) + + def test_good_goto(self): + dialog = self.Dummy_ModuleName('1') + self.assertEqual(dialog.entry_ok(), 1) + self.assertEqual(dialog.entry_error['text'], '') + + # 3 HelpSource test classes each test one method. class HelpsourceBrowsefileTest(unittest.TestCase): @@ -363,6 +390,22 @@ class ModulenameGuiTest(unittest.TestCase): root.destroy() +class GotoGuiTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + + def test_click_module_name(self): + root = Tk() + root.withdraw() + dialog = query.Goto(root, 'T', 't', _utest=True) + dialog.entry.insert(0, '22') + dialog.button_ok.invoke() + self.assertEqual(dialog.result, 22) + root.destroy() + + class HelpsourceGuiTest(unittest.TestCase): @classmethod @@ -392,10 +435,12 @@ class CustomRunGuiTest(unittest.TestCase): def test_click_args(self): root = Tk() root.withdraw() - dialog = query.CustomRun(root, 'Title', _utest=True) - dialog.entry.insert(0, 'okay') + dialog = query.CustomRun(root, 'Title', + cli_args=['a', 'b=1'], _utest=True) + self.assertEqual(dialog.entry.get(), 'a b=1') + dialog.entry.insert(END, ' c') dialog.button_ok.invoke() - self.assertEqual(dialog.result, (['okay'], True)) + self.assertEqual(dialog.result, (['a', 'b=1', 'c'], True)) root.destroy() diff --git a/lib-python/3/idlelib/idle_test/test_rstrip.py b/lib-python/3/idlelib/idle_test/test_rstrip.py deleted file mode 100644 index 2bc7c6f035..0000000000 --- a/lib-python/3/idlelib/idle_test/test_rstrip.py +++ /dev/null @@ -1,53 +0,0 @@ -"Test rstrip, coverage 100%." - -from idlelib import rstrip -import unittest -from idlelib.idle_test.mock_idle import Editor - -class rstripTest(unittest.TestCase): - - def test_rstrip_line(self): - editor = Editor() - text = editor.text - do_rstrip = rstrip.Rstrip(editor).do_rstrip - - do_rstrip() - self.assertEqual(text.get('1.0', 'insert'), '') - text.insert('1.0', ' ') - do_rstrip() - self.assertEqual(text.get('1.0', 'insert'), '') - text.insert('1.0', ' \n') - do_rstrip() - self.assertEqual(text.get('1.0', 'insert'), '\n') - - def test_rstrip_multiple(self): - editor = Editor() - # Comment above, uncomment 3 below to test with real Editor & Text. - #from idlelib.editor import EditorWindow as Editor - #from tkinter import Tk - #editor = Editor(root=Tk()) - text = editor.text - do_rstrip = rstrip.Rstrip(editor).do_rstrip - - original = ( - "Line with an ending tab \n" - "Line ending in 5 spaces \n" - "Linewithnospaces\n" - " indented line\n" - " indented line with trailing space \n" - " ") - stripped = ( - "Line with an ending tab\n" - "Line ending in 5 spaces\n" - "Linewithnospaces\n" - " indented line\n" - " indented line with trailing space\n") - - text.insert('1.0', original) - do_rstrip() - self.assertEqual(text.get('1.0', 'insert'), stripped) - - - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/lib-python/3/idlelib/idle_test/test_run.py b/lib-python/3/idlelib/idle_test/test_run.py index 46f0235fbf..9995dbe2ec 100644 --- a/lib-python/3/idlelib/idle_test/test_run.py +++ b/lib-python/3/idlelib/idle_test/test_run.py @@ -6,6 +6,8 @@ from unittest import mock from test.support import captured_stderr import io +import sys + class RunTest(unittest.TestCase): @@ -34,7 +36,7 @@ class RunTest(unittest.TestCase): self.assertIn('UnhashableException: ex1', tb[10]) -# PseudoFile tests. +# StdioFile tests. class S(str): def __str__(self): @@ -66,14 +68,14 @@ class MockShell: self.lines = list(lines)[::-1] -class PseudeInputFilesTest(unittest.TestCase): +class StdInputFilesTest(unittest.TestCase): def test_misc(self): shell = MockShell() - f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + f = run.StdInputFile(shell, 'stdin') self.assertIsInstance(f, io.TextIOBase) self.assertEqual(f.encoding, 'utf-8') - self.assertIsNone(f.errors) + self.assertEqual(f.errors, 'strict') self.assertIsNone(f.newlines) self.assertEqual(f.name, '<stdin>') self.assertFalse(f.closed) @@ -84,7 +86,7 @@ class PseudeInputFilesTest(unittest.TestCase): def test_unsupported(self): shell = MockShell() - f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + f = run.StdInputFile(shell, 'stdin') self.assertRaises(OSError, f.fileno) self.assertRaises(OSError, f.tell) self.assertRaises(OSError, f.seek, 0) @@ -93,7 +95,7 @@ class PseudeInputFilesTest(unittest.TestCase): def test_read(self): shell = MockShell() - f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + f = run.StdInputFile(shell, 'stdin') shell.push(['one\n', 'two\n', '']) self.assertEqual(f.read(), 'one\ntwo\n') shell.push(['one\n', 'two\n', '']) @@ -113,7 +115,7 @@ class PseudeInputFilesTest(unittest.TestCase): def test_readline(self): shell = MockShell() - f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + f = run.StdInputFile(shell, 'stdin') shell.push(['one\n', 'two\n', 'three\n', 'four\n']) self.assertEqual(f.readline(), 'one\n') self.assertEqual(f.readline(-1), 'two\n') @@ -138,7 +140,7 @@ class PseudeInputFilesTest(unittest.TestCase): def test_readlines(self): shell = MockShell() - f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + f = run.StdInputFile(shell, 'stdin') shell.push(['one\n', 'two\n', '']) self.assertEqual(f.readlines(), ['one\n', 'two\n']) shell.push(['one\n', 'two\n', '']) @@ -159,7 +161,7 @@ class PseudeInputFilesTest(unittest.TestCase): def test_close(self): shell = MockShell() - f = run.PseudoInputFile(shell, 'stdin', 'utf-8') + f = run.StdInputFile(shell, 'stdin') shell.push(['one\n', 'two\n', '']) self.assertFalse(f.closed) self.assertEqual(f.readline(), 'one\n') @@ -169,14 +171,14 @@ class PseudeInputFilesTest(unittest.TestCase): self.assertRaises(TypeError, f.close, 1) -class PseudeOutputFilesTest(unittest.TestCase): +class StdOutputFilesTest(unittest.TestCase): def test_misc(self): shell = MockShell() - f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') + f = run.StdOutputFile(shell, 'stdout') self.assertIsInstance(f, io.TextIOBase) self.assertEqual(f.encoding, 'utf-8') - self.assertIsNone(f.errors) + self.assertEqual(f.errors, 'strict') self.assertIsNone(f.newlines) self.assertEqual(f.name, '<stdout>') self.assertFalse(f.closed) @@ -187,7 +189,7 @@ class PseudeOutputFilesTest(unittest.TestCase): def test_unsupported(self): shell = MockShell() - f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') + f = run.StdOutputFile(shell, 'stdout') self.assertRaises(OSError, f.fileno) self.assertRaises(OSError, f.tell) self.assertRaises(OSError, f.seek, 0) @@ -196,16 +198,36 @@ class PseudeOutputFilesTest(unittest.TestCase): def test_write(self): shell = MockShell() - f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') + f = run.StdOutputFile(shell, 'stdout') f.write('test') self.assertEqual(shell.written, [('test', 'stdout')]) shell.reset() - f.write('t\xe8st') - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + f.write('t\xe8\u015b\U0001d599') + self.assertEqual(shell.written, [('t\xe8\u015b\U0001d599', 'stdout')]) shell.reset() - f.write(S('t\xe8st')) - self.assertEqual(shell.written, [('t\xe8st', 'stdout')]) + f.write(S('t\xe8\u015b\U0001d599')) + self.assertEqual(shell.written, [('t\xe8\u015b\U0001d599', 'stdout')]) + self.assertEqual(type(shell.written[0][0]), str) + shell.reset() + + self.assertRaises(TypeError, f.write) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.write, b'test') + self.assertRaises(TypeError, f.write, 123) + self.assertEqual(shell.written, []) + self.assertRaises(TypeError, f.write, 'test', 'spam') + self.assertEqual(shell.written, []) + + def test_write_stderr_nonencodable(self): + shell = MockShell() + f = run.StdOutputFile(shell, 'stderr', 'iso-8859-15', 'backslashreplace') + f.write('t\xe8\u015b\U0001d599\xa4') + self.assertEqual(shell.written, [('t\xe8\\u015b\\U0001d599\\xa4', 'stderr')]) + shell.reset() + + f.write(S('t\xe8\u015b\U0001d599\xa4')) + self.assertEqual(shell.written, [('t\xe8\\u015b\\U0001d599\\xa4', 'stderr')]) self.assertEqual(type(shell.written[0][0]), str) shell.reset() @@ -219,7 +241,7 @@ class PseudeOutputFilesTest(unittest.TestCase): def test_writelines(self): shell = MockShell() - f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') + f = run.StdOutputFile(shell, 'stdout') f.writelines([]) self.assertEqual(shell.written, []) shell.reset() @@ -249,7 +271,7 @@ class PseudeOutputFilesTest(unittest.TestCase): def test_close(self): shell = MockShell() - f = run.PseudoOutputFile(shell, 'stdout', 'utf-8') + f = run.StdOutputFile(shell, 'stdout') self.assertFalse(f.closed) f.write('test') f.close() @@ -260,5 +282,44 @@ class PseudeOutputFilesTest(unittest.TestCase): self.assertRaises(TypeError, f.close, 1) +class TestSysRecursionLimitWrappers(unittest.TestCase): + + def test_bad_setrecursionlimit_calls(self): + run.install_recursionlimit_wrappers() + self.addCleanup(run.uninstall_recursionlimit_wrappers) + f = sys.setrecursionlimit + self.assertRaises(TypeError, f, limit=100) + self.assertRaises(TypeError, f, 100, 1000) + self.assertRaises(ValueError, f, 0) + + def test_roundtrip(self): + run.install_recursionlimit_wrappers() + self.addCleanup(run.uninstall_recursionlimit_wrappers) + + # check that setting the recursion limit works + orig_reclimit = sys.getrecursionlimit() + self.addCleanup(sys.setrecursionlimit, orig_reclimit) + sys.setrecursionlimit(orig_reclimit + 3) + + # check that the new limit is returned by sys.getrecursionlimit() + new_reclimit = sys.getrecursionlimit() + self.assertEqual(new_reclimit, orig_reclimit + 3) + + def test_default_recursion_limit_preserved(self): + orig_reclimit = sys.getrecursionlimit() + run.install_recursionlimit_wrappers() + self.addCleanup(run.uninstall_recursionlimit_wrappers) + new_reclimit = sys.getrecursionlimit() + self.assertEqual(new_reclimit, orig_reclimit) + + def test_fixdoc(self): + def func(): "docstring" + run.fixdoc(func, "more") + self.assertEqual(func.__doc__, "docstring\n\nmore") + func.__doc__ = None + run.fixdoc(func, "more") + self.assertEqual(func.__doc__, "more") + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/lib-python/3/idlelib/idle_test/test_sidebar.py b/lib-python/3/idlelib/idle_test/test_sidebar.py new file mode 100644 index 0000000000..2974a9a7b0 --- /dev/null +++ b/lib-python/3/idlelib/idle_test/test_sidebar.py @@ -0,0 +1,374 @@ +"""Test sidebar, coverage 93%""" +import idlelib.sidebar +from itertools import chain +import unittest +import unittest.mock +from test.support import requires +import tkinter as tk + +from idlelib.delegator import Delegator +from idlelib.percolator import Percolator + + +class Dummy_editwin: + def __init__(self, text): + self.text = text + self.text_frame = self.text.master + self.per = Percolator(text) + self.undo = Delegator() + self.per.insertfilter(self.undo) + + def setvar(self, name, value): + pass + + def getlineno(self, index): + return int(float(self.text.index(index))) + + +class LineNumbersTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = tk.Tk() + + cls.text_frame = tk.Frame(cls.root) + cls.text_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True) + cls.text_frame.rowconfigure(1, weight=1) + cls.text_frame.columnconfigure(1, weight=1) + + cls.text = tk.Text(cls.text_frame, width=80, height=24, wrap=tk.NONE) + cls.text.grid(row=1, column=1, sticky=tk.NSEW) + + cls.editwin = Dummy_editwin(cls.text) + cls.editwin.vbar = tk.Scrollbar(cls.text_frame) + + @classmethod + def tearDownClass(cls): + cls.editwin.per.close() + cls.root.update() + cls.root.destroy() + del cls.text, cls.text_frame, cls.editwin, cls.root + + def setUp(self): + self.linenumber = idlelib.sidebar.LineNumbers(self.editwin) + + self.highlight_cfg = {"background": '#abcdef', + "foreground": '#123456'} + orig_idleConf_GetHighlight = idlelib.sidebar.idleConf.GetHighlight + def mock_idleconf_GetHighlight(theme, element): + if element == 'linenumber': + return self.highlight_cfg + return orig_idleConf_GetHighlight(theme, element) + GetHighlight_patcher = unittest.mock.patch.object( + idlelib.sidebar.idleConf, 'GetHighlight', mock_idleconf_GetHighlight) + GetHighlight_patcher.start() + self.addCleanup(GetHighlight_patcher.stop) + + self.font_override = 'TkFixedFont' + def mock_idleconf_GetFont(root, configType, section): + return self.font_override + GetFont_patcher = unittest.mock.patch.object( + idlelib.sidebar.idleConf, 'GetFont', mock_idleconf_GetFont) + GetFont_patcher.start() + self.addCleanup(GetFont_patcher.stop) + + def tearDown(self): + self.text.delete('1.0', 'end') + + def get_selection(self): + return tuple(map(str, self.text.tag_ranges('sel'))) + + def get_line_screen_position(self, line): + bbox = self.linenumber.sidebar_text.bbox(f'{line}.end -1c') + x = bbox[0] + 2 + y = bbox[1] + 2 + return x, y + + def assert_state_disabled(self): + state = self.linenumber.sidebar_text.config()['state'] + self.assertEqual(state[-1], tk.DISABLED) + + def get_sidebar_text_contents(self): + return self.linenumber.sidebar_text.get('1.0', tk.END) + + def assert_sidebar_n_lines(self, n_lines): + expected = '\n'.join(chain(map(str, range(1, n_lines + 1)), [''])) + self.assertEqual(self.get_sidebar_text_contents(), expected) + + def assert_text_equals(self, expected): + return self.assertEqual(self.text.get('1.0', 'end'), expected) + + def test_init_empty(self): + self.assert_sidebar_n_lines(1) + + def test_init_not_empty(self): + self.text.insert('insert', 'foo bar\n'*3) + self.assert_text_equals('foo bar\n'*3 + '\n') + self.assert_sidebar_n_lines(4) + + def test_toggle_linenumbering(self): + self.assertEqual(self.linenumber.is_shown, False) + self.linenumber.show_sidebar() + self.assertEqual(self.linenumber.is_shown, True) + self.linenumber.hide_sidebar() + self.assertEqual(self.linenumber.is_shown, False) + self.linenumber.hide_sidebar() + self.assertEqual(self.linenumber.is_shown, False) + self.linenumber.show_sidebar() + self.assertEqual(self.linenumber.is_shown, True) + self.linenumber.show_sidebar() + self.assertEqual(self.linenumber.is_shown, True) + + def test_insert(self): + self.text.insert('insert', 'foobar') + self.assert_text_equals('foobar\n') + self.assert_sidebar_n_lines(1) + self.assert_state_disabled() + + self.text.insert('insert', '\nfoo') + self.assert_text_equals('foobar\nfoo\n') + self.assert_sidebar_n_lines(2) + self.assert_state_disabled() + + self.text.insert('insert', 'hello\n'*2) + self.assert_text_equals('foobar\nfoohello\nhello\n\n') + self.assert_sidebar_n_lines(4) + self.assert_state_disabled() + + self.text.insert('insert', '\nworld') + self.assert_text_equals('foobar\nfoohello\nhello\n\nworld\n') + self.assert_sidebar_n_lines(5) + self.assert_state_disabled() + + def test_delete(self): + self.text.insert('insert', 'foobar') + self.assert_text_equals('foobar\n') + self.text.delete('1.1', '1.3') + self.assert_text_equals('fbar\n') + self.assert_sidebar_n_lines(1) + self.assert_state_disabled() + + self.text.insert('insert', 'foo\n'*2) + self.assert_text_equals('fbarfoo\nfoo\n\n') + self.assert_sidebar_n_lines(3) + self.assert_state_disabled() + + # Note: deleting up to "2.end" doesn't delete the final newline. + self.text.delete('2.0', '2.end') + self.assert_text_equals('fbarfoo\n\n\n') + self.assert_sidebar_n_lines(3) + self.assert_state_disabled() + + self.text.delete('1.3', 'end') + self.assert_text_equals('fba\n') + self.assert_sidebar_n_lines(1) + self.assert_state_disabled() + + # Note: Text widgets always keep a single '\n' character at the end. + self.text.delete('1.0', 'end') + self.assert_text_equals('\n') + self.assert_sidebar_n_lines(1) + self.assert_state_disabled() + + def test_sidebar_text_width(self): + """ + Test that linenumber text widget is always at the minimum + width + """ + def get_width(): + return self.linenumber.sidebar_text.config()['width'][-1] + + self.assert_sidebar_n_lines(1) + self.assertEqual(get_width(), 1) + + self.text.insert('insert', 'foo') + self.assert_sidebar_n_lines(1) + self.assertEqual(get_width(), 1) + + self.text.insert('insert', 'foo\n'*8) + self.assert_sidebar_n_lines(9) + self.assertEqual(get_width(), 1) + + self.text.insert('insert', 'foo\n') + self.assert_sidebar_n_lines(10) + self.assertEqual(get_width(), 2) + + self.text.insert('insert', 'foo\n') + self.assert_sidebar_n_lines(11) + self.assertEqual(get_width(), 2) + + self.text.delete('insert -1l linestart', 'insert linestart') + self.assert_sidebar_n_lines(10) + self.assertEqual(get_width(), 2) + + self.text.delete('insert -1l linestart', 'insert linestart') + self.assert_sidebar_n_lines(9) + self.assertEqual(get_width(), 1) + + self.text.insert('insert', 'foo\n'*90) + self.assert_sidebar_n_lines(99) + self.assertEqual(get_width(), 2) + + self.text.insert('insert', 'foo\n') + self.assert_sidebar_n_lines(100) + self.assertEqual(get_width(), 3) + + self.text.insert('insert', 'foo\n') + self.assert_sidebar_n_lines(101) + self.assertEqual(get_width(), 3) + + self.text.delete('insert -1l linestart', 'insert linestart') + self.assert_sidebar_n_lines(100) + self.assertEqual(get_width(), 3) + + self.text.delete('insert -1l linestart', 'insert linestart') + self.assert_sidebar_n_lines(99) + self.assertEqual(get_width(), 2) + + self.text.delete('50.0 -1c', 'end -1c') + self.assert_sidebar_n_lines(49) + self.assertEqual(get_width(), 2) + + self.text.delete('5.0 -1c', 'end -1c') + self.assert_sidebar_n_lines(4) + self.assertEqual(get_width(), 1) + + # Note: Text widgets always keep a single '\n' character at the end. + self.text.delete('1.0', 'end -1c') + self.assert_sidebar_n_lines(1) + self.assertEqual(get_width(), 1) + + def test_click_selection(self): + self.linenumber.show_sidebar() + self.text.insert('1.0', 'one\ntwo\nthree\nfour\n') + self.root.update() + + # Click on the second line. + x, y = self.get_line_screen_position(2) + self.linenumber.sidebar_text.event_generate('<Button-1>', x=x, y=y) + self.linenumber.sidebar_text.update() + self.root.update() + + self.assertEqual(self.get_selection(), ('2.0', '3.0')) + + def simulate_drag(self, start_line, end_line): + start_x, start_y = self.get_line_screen_position(start_line) + end_x, end_y = self.get_line_screen_position(end_line) + + self.linenumber.sidebar_text.event_generate('<Button-1>', + x=start_x, y=start_y) + self.root.update() + + def lerp(a, b, steps): + """linearly interpolate from a to b (inclusive) in equal steps""" + last_step = steps - 1 + for i in range(steps): + yield ((last_step - i) / last_step) * a + (i / last_step) * b + + for x, y in zip( + map(int, lerp(start_x, end_x, steps=11)), + map(int, lerp(start_y, end_y, steps=11)), + ): + self.linenumber.sidebar_text.event_generate('<B1-Motion>', x=x, y=y) + self.root.update() + + self.linenumber.sidebar_text.event_generate('<ButtonRelease-1>', + x=end_x, y=end_y) + self.root.update() + + def test_drag_selection_down(self): + self.linenumber.show_sidebar() + self.text.insert('1.0', 'one\ntwo\nthree\nfour\nfive\n') + self.root.update() + + # Drag from the second line to the fourth line. + self.simulate_drag(2, 4) + self.assertEqual(self.get_selection(), ('2.0', '5.0')) + + def test_drag_selection_up(self): + self.linenumber.show_sidebar() + self.text.insert('1.0', 'one\ntwo\nthree\nfour\nfive\n') + self.root.update() + + # Drag from the fourth line to the second line. + self.simulate_drag(4, 2) + self.assertEqual(self.get_selection(), ('2.0', '5.0')) + + def test_scroll(self): + self.linenumber.show_sidebar() + self.text.insert('1.0', 'line\n' * 100) + self.root.update() + + # Scroll down 10 lines. + self.text.yview_scroll(10, 'unit') + self.root.update() + self.assertEqual(self.text.index('@0,0'), '11.0') + self.assertEqual(self.linenumber.sidebar_text.index('@0,0'), '11.0') + + # Generate a mouse-wheel event and make sure it scrolled up or down. + # The meaning of the "delta" is OS-dependant, so this just checks for + # any change. + self.linenumber.sidebar_text.event_generate('<MouseWheel>', + x=0, y=0, + delta=10) + self.root.update() + self.assertNotEqual(self.text.index('@0,0'), '11.0') + self.assertNotEqual(self.linenumber.sidebar_text.index('@0,0'), '11.0') + + def test_font(self): + ln = self.linenumber + + orig_font = ln.sidebar_text['font'] + test_font = 'TkTextFont' + self.assertNotEqual(orig_font, test_font) + + # Ensure line numbers aren't shown. + ln.hide_sidebar() + + self.font_override = test_font + # Nothing breaks when line numbers aren't shown. + ln.update_font() + + # Activate line numbers, previous font change is immediately effective. + ln.show_sidebar() + self.assertEqual(ln.sidebar_text['font'], test_font) + + # Call the font update with line numbers shown, change is picked up. + self.font_override = orig_font + ln.update_font() + self.assertEqual(ln.sidebar_text['font'], orig_font) + + def test_highlight_colors(self): + ln = self.linenumber + + orig_colors = dict(self.highlight_cfg) + test_colors = {'background': '#222222', 'foreground': '#ffff00'} + + def assert_colors_are_equal(colors): + self.assertEqual(ln.sidebar_text['background'], colors['background']) + self.assertEqual(ln.sidebar_text['foreground'], colors['foreground']) + + # Ensure line numbers aren't shown. + ln.hide_sidebar() + + self.highlight_cfg = test_colors + # Nothing breaks with inactive code context. + ln.update_colors() + + # Show line numbers, previous colors change is immediately effective. + ln.show_sidebar() + assert_colors_are_equal(test_colors) + + # Call colors update with no change to the configured colors. + ln.update_colors() + assert_colors_are_equal(test_colors) + + # Call the colors update with line numbers shown, change is picked up. + self.highlight_cfg = orig_colors + ln.update_colors() + assert_colors_are_equal(orig_colors) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/lib-python/3/idlelib/idle_test/test_squeezer.py b/lib-python/3/idlelib/idle_test/test_squeezer.py index 4e3da030a3..e3912f4bbb 100644 --- a/lib-python/3/idlelib/idle_test/test_squeezer.py +++ b/lib-python/3/idlelib/idle_test/test_squeezer.py @@ -1,6 +1,5 @@ "Test squeezer, coverage 95%" -from collections import namedtuple from textwrap import dedent from tkinter import Text, Tk import unittest @@ -82,18 +81,10 @@ class CountLinesTest(unittest.TestCase): class SqueezerTest(unittest.TestCase): """Tests for the Squeezer class.""" - def tearDown(self): - # Clean up the Squeezer class's reference to its instance, - # to avoid side-effects from one test case upon another. - if Squeezer._instance_weakref is not None: - Squeezer._instance_weakref = None - def make_mock_editor_window(self, with_text_widget=False): """Create a mock EditorWindow instance.""" editwin = NonCallableMagicMock() - # isinstance(editwin, PyShell) must be true for Squeezer to enable - # auto-squeezing; in practice this will always be true. - editwin.__class__ = PyShell + editwin.width = 80 if with_text_widget: editwin.root = get_test_tk_root(self) @@ -107,7 +98,6 @@ class SqueezerTest(unittest.TestCase): if editor_window is None: editor_window = self.make_mock_editor_window() squeezer = Squeezer(editor_window) - squeezer.get_line_width = Mock(return_value=80) return squeezer def make_text_widget(self, root=None): @@ -143,8 +133,8 @@ class SqueezerTest(unittest.TestCase): line_width=line_width, expected=expected): text = eval(text_code) - squeezer.get_line_width.return_value = line_width - self.assertEqual(squeezer.count_lines(text), expected) + with patch.object(editwin, 'width', line_width): + self.assertEqual(squeezer.count_lines(text), expected) def test_init(self): """Test the creation of Squeezer instances.""" @@ -294,7 +284,6 @@ class SqueezerTest(unittest.TestCase): """Test the reload() class-method.""" editwin = self.make_mock_editor_window(with_text_widget=True) squeezer = self.make_squeezer_instance(editwin) - squeezer.load_font = Mock() orig_auto_squeeze_min_lines = squeezer.auto_squeeze_min_lines @@ -307,7 +296,6 @@ class SqueezerTest(unittest.TestCase): Squeezer.reload() self.assertEqual(squeezer.auto_squeeze_min_lines, new_auto_squeeze_min_lines) - squeezer.load_font.assert_called() def test_reload_no_squeezer_instances(self): """Test that Squeezer.reload() runs without any instances existing.""" diff --git a/lib-python/3/idlelib/idle_test/test_textview.py b/lib-python/3/idlelib/idle_test/test_textview.py index 6f0c193051..7189378ab3 100644 --- a/lib-python/3/idlelib/idle_test/test_textview.py +++ b/lib-python/3/idlelib/idle_test/test_textview.py @@ -6,12 +6,12 @@ Using mock Text would not change this. Other mocks are used to retrieve information about calls. """ from idlelib import textview as tv -import unittest from test.support import requires requires('gui') import os -from tkinter import Tk +import unittest +from tkinter import Tk, TclError, CHAR, NONE, WORD from tkinter.ttk import Button from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Mbox_func @@ -69,13 +69,65 @@ class ViewWindowTest(unittest.TestCase): view.destroy() -class TextFrameTest(unittest.TestCase): +class AutoHideScrollbarTest(unittest.TestCase): + # Method set is tested in ScrollableTextFrameTest + def test_forbidden_geometry(self): + scroll = tv.AutoHideScrollbar(root) + self.assertRaises(TclError, scroll.pack) + self.assertRaises(TclError, scroll.place) + + +class ScrollableTextFrameTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.root = root = Tk() + root.withdraw() + + @classmethod + def tearDownClass(cls): + cls.root.update_idletasks() + cls.root.destroy() + del cls.root + + def make_frame(self, wrap=NONE, **kwargs): + frame = tv.ScrollableTextFrame(self.root, wrap=wrap, **kwargs) + def cleanup_frame(): + frame.update_idletasks() + frame.destroy() + self.addCleanup(cleanup_frame) + return frame + + def test_line1(self): + frame = self.make_frame() + frame.text.insert('1.0', 'test text') + self.assertEqual(frame.text.get('1.0', '1.end'), 'test text') + + def test_horiz_scrollbar(self): + # The horizontal scrollbar should be shown/hidden according to + # the 'wrap' setting: It should only be shown when 'wrap' is + # set to NONE. + + # wrap = NONE -> with horizontal scrolling + frame = self.make_frame(wrap=NONE) + self.assertEqual(frame.text.cget('wrap'), NONE) + self.assertIsNotNone(frame.xscroll) + + # wrap != NONE -> no horizontal scrolling + for wrap in [CHAR, WORD]: + with self.subTest(wrap=wrap): + frame = self.make_frame(wrap=wrap) + self.assertEqual(frame.text.cget('wrap'), wrap) + self.assertIsNone(frame.xscroll) + + +class ViewFrameTest(unittest.TestCase): @classmethod def setUpClass(cls): cls.root = root = Tk() root.withdraw() - cls.frame = tv.TextFrame(root, 'test text') + cls.frame = tv.ViewFrame(root, 'test text') @classmethod def tearDownClass(cls): diff --git a/lib-python/3/idlelib/idle_test/test_tooltip.py b/lib-python/3/idlelib/idle_test/test_tooltip.py index 44ea1110e1..c616d4fde3 100644 --- a/lib-python/3/idlelib/idle_test/test_tooltip.py +++ b/lib-python/3/idlelib/idle_test/test_tooltip.py @@ -1,3 +1,10 @@ +"""Test tooltip, coverage 100%. + +Coverage is 100% after excluding 6 lines with "# pragma: no cover". +They involve TclErrors that either should or should not happen in a +particular situation, and which are 'pass'ed if they do. +""" + from idlelib.tooltip import TooltipBase, Hovertip from test.support import requires requires('gui') @@ -12,16 +19,13 @@ def setUpModule(): global root root = Tk() -def root_update(): - global root - root.update() - def tearDownModule(): global root root.update_idletasks() root.destroy() del root + def add_call_counting(func): @wraps(func) def wrapped_func(*args, **kwargs): @@ -65,22 +69,25 @@ class HovertipTest(unittest.TestCase): def setUp(self): self.top, self.button = _make_top_and_button(self) + def is_tipwindow_shown(self, tooltip): + return tooltip.tipwindow and tooltip.tipwindow.winfo_viewable() + def test_showtip(self): tooltip = Hovertip(self.button, 'ToolTip text') self.addCleanup(tooltip.hidetip) - self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertFalse(self.is_tipwindow_shown(tooltip)) tooltip.showtip() - self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertTrue(self.is_tipwindow_shown(tooltip)) def test_showtip_twice(self): tooltip = Hovertip(self.button, 'ToolTip text') self.addCleanup(tooltip.hidetip) - self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertFalse(self.is_tipwindow_shown(tooltip)) tooltip.showtip() - self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertTrue(self.is_tipwindow_shown(tooltip)) orig_tipwindow = tooltip.tipwindow tooltip.showtip() - self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertTrue(self.is_tipwindow_shown(tooltip)) self.assertIs(tooltip.tipwindow, orig_tipwindow) def test_hidetip(self): @@ -88,59 +95,67 @@ class HovertipTest(unittest.TestCase): self.addCleanup(tooltip.hidetip) tooltip.showtip() tooltip.hidetip() - self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + self.assertFalse(self.is_tipwindow_shown(tooltip)) def test_showtip_on_mouse_enter_no_delay(self): tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=None) self.addCleanup(tooltip.hidetip) tooltip.showtip = add_call_counting(tooltip.showtip) - root_update() - self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + root.update() + self.assertFalse(self.is_tipwindow_shown(tooltip)) self.button.event_generate('<Enter>', x=0, y=0) - root_update() - self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + root.update() + self.assertTrue(self.is_tipwindow_shown(tooltip)) self.assertGreater(len(tooltip.showtip.call_args_list), 0) - def test_showtip_on_mouse_enter_hover_delay(self): - tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=50) - self.addCleanup(tooltip.hidetip) - tooltip.showtip = add_call_counting(tooltip.showtip) - root_update() - self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + def test_hover_with_delay(self): + # Run multiple tests requiring an actual delay simultaneously. + + # Test #1: A hover tip with a non-zero delay appears after the delay. + tooltip1 = Hovertip(self.button, 'ToolTip text', hover_delay=100) + self.addCleanup(tooltip1.hidetip) + tooltip1.showtip = add_call_counting(tooltip1.showtip) + root.update() + self.assertFalse(self.is_tipwindow_shown(tooltip1)) self.button.event_generate('<Enter>', x=0, y=0) - root_update() - self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) - time.sleep(0.1) - root_update() - self.assertTrue(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) - self.assertGreater(len(tooltip.showtip.call_args_list), 0) + root.update() + self.assertFalse(self.is_tipwindow_shown(tooltip1)) + + # Test #2: A hover tip with a non-zero delay doesn't appear when + # the mouse stops hovering over the base widget before the delay + # expires. + tooltip2 = Hovertip(self.button, 'ToolTip text', hover_delay=100) + self.addCleanup(tooltip2.hidetip) + tooltip2.showtip = add_call_counting(tooltip2.showtip) + root.update() + self.button.event_generate('<Enter>', x=0, y=0) + root.update() + self.button.event_generate('<Leave>', x=0, y=0) + root.update() + + time.sleep(0.15) + root.update() + + # Test #1 assertions. + self.assertTrue(self.is_tipwindow_shown(tooltip1)) + self.assertGreater(len(tooltip1.showtip.call_args_list), 0) + + # Test #2 assertions. + self.assertFalse(self.is_tipwindow_shown(tooltip2)) + self.assertEqual(tooltip2.showtip.call_args_list, []) def test_hidetip_on_mouse_leave(self): tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=None) self.addCleanup(tooltip.hidetip) tooltip.showtip = add_call_counting(tooltip.showtip) - root_update() + root.update() self.button.event_generate('<Enter>', x=0, y=0) - root_update() + root.update() self.button.event_generate('<Leave>', x=0, y=0) - root_update() - self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) + root.update() + self.assertFalse(self.is_tipwindow_shown(tooltip)) self.assertGreater(len(tooltip.showtip.call_args_list), 0) - def test_dont_show_on_mouse_leave_before_delay(self): - tooltip = Hovertip(self.button, 'ToolTip text', hover_delay=50) - self.addCleanup(tooltip.hidetip) - tooltip.showtip = add_call_counting(tooltip.showtip) - root_update() - self.button.event_generate('<Enter>', x=0, y=0) - root_update() - self.button.event_generate('<Leave>', x=0, y=0) - root_update() - time.sleep(0.1) - root_update() - self.assertFalse(tooltip.tipwindow and tooltip.tipwindow.winfo_viewable()) - self.assertEqual(tooltip.showtip.call_args_list, []) - if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/lib-python/3/idlelib/idle_test/test_tree.py b/lib-python/3/idlelib/idle_test/test_tree.py index 9be9abee36..b3e4c10cf9 100644 --- a/lib-python/3/idlelib/idle_test/test_tree.py +++ b/lib-python/3/idlelib/idle_test/test_tree.py @@ -4,7 +4,7 @@ from idlelib import tree import unittest from test.support import requires requires('gui') -from tkinter import Tk +from tkinter import Tk, EventType, SCROLL class TreeTest(unittest.TestCase): @@ -29,5 +29,32 @@ class TreeTest(unittest.TestCase): node.expand() +class TestScrollEvent(unittest.TestCase): + + def test_wheel_event(self): + # Fake widget class containing `yview` only. + class _Widget: + def __init__(widget, *expected): + widget.expected = expected + def yview(widget, *args): + self.assertTupleEqual(widget.expected, args) + # Fake event class + class _Event: + pass + # (type, delta, num, amount) + tests = ((EventType.MouseWheel, 120, -1, -5), + (EventType.MouseWheel, -120, -1, 5), + (EventType.ButtonPress, -1, 4, -5), + (EventType.ButtonPress, -1, 5, 5)) + + event = _Event() + for ty, delta, num, amount in tests: + event.type = ty + event.delta = delta + event.num = num + res = tree.wheel_event(event, _Widget(SCROLL, amount, "units")) + self.assertEqual(res, "break") + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/lib-python/3/idlelib/iomenu.py b/lib-python/3/idlelib/iomenu.py index b9e813be06..4b2833b8ca 100644 --- a/lib-python/3/idlelib/iomenu.py +++ b/lib-python/3/idlelib/iomenu.py @@ -15,6 +15,7 @@ from idlelib.config import idleConf if idlelib.testing: # Set True by test.test_idle to avoid setlocale. encoding = 'utf-8' + errors = 'surrogateescape' else: # Try setting the locale, so that we can find out # what encoding to use @@ -24,15 +25,9 @@ else: except (ImportError, locale.Error): pass - locale_decode = 'ascii' if sys.platform == 'win32': - # On Windows, we could use "mbcs". However, to give the user - # a portable encoding name, we need to find the code page - try: - locale_encoding = locale.getdefaultlocale()[1] - codecs.lookup(locale_encoding) - except LookupError: - pass + encoding = 'utf-8' + errors = 'surrogateescape' else: try: # Different things can fail here: the locale module may not be @@ -40,30 +35,30 @@ else: # resulting codeset may be unknown to Python. We ignore all # these problems, falling back to ASCII locale_encoding = locale.nl_langinfo(locale.CODESET) - if locale_encoding is None or locale_encoding == '': - # situation occurs on macOS - locale_encoding = 'ascii' - codecs.lookup(locale_encoding) + if locale_encoding: + codecs.lookup(locale_encoding) except (NameError, AttributeError, LookupError): # Try getdefaultlocale: it parses environment variables, # which may give a clue. Unfortunately, getdefaultlocale has # bugs that can cause ValueError. try: locale_encoding = locale.getdefaultlocale()[1] - if locale_encoding is None or locale_encoding == '': - # situation occurs on macOS - locale_encoding = 'ascii' - codecs.lookup(locale_encoding) + if locale_encoding: + codecs.lookup(locale_encoding) except (ValueError, LookupError): pass - locale_encoding = locale_encoding.lower() - - encoding = locale_encoding - # Encoding is used in multiple files; locale_encoding nowhere. - # The only use of 'encoding' below is in _decode as initial value - # of deprecated block asking user for encoding. - # Perhaps use elsewhere should be reviewed. + if locale_encoding: + encoding = locale_encoding.lower() + errors = 'strict' + else: + # POSIX locale or macOS + encoding = 'ascii' + errors = 'surrogateescape' + # Encoding is used in multiple files; locale_encoding nowhere. + # The only use of 'encoding' below is in _decode as initial value + # of deprecated block asking user for encoding. + # Perhaps use elsewhere should be reviewed. coding_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) @@ -376,10 +371,7 @@ class IOBinding: return "break" def writefile(self, filename): - self.fixlastline() - text = self.text.get("1.0", "end-1c") - if self.eol_convention != "\n": - text = text.replace("\n", self.eol_convention) + text = self.fixnewlines() chars = self.encode(text) try: with open(filename, "wb") as f: @@ -392,6 +384,16 @@ class IOBinding: parent=self.text) return False + def fixnewlines(self): + "Return text with final \n if needed and os eols." + if (self.text.get("end-2c") != '\n' + and not hasattr(self.editwin, "interp")): # Not shell. + self.text.insert("end-1c", "\n") + text = self.text.get("1.0", "end-1c") + if self.eol_convention != "\n": + text = text.replace("\n", self.eol_convention) + return text + def encode(self, chars): if isinstance(chars, bytes): # This is either plain ASCII, or Tk was returning mixed-encoding @@ -431,11 +433,6 @@ class IOBinding: # declared encoding return BOM_UTF8 + chars.encode("utf-8") - def fixlastline(self): - c = self.text.get("end-2c") - if c != '\n': - self.text.insert("end-1c", "\n") - def print_window(self, event): confirm = tkMessageBox.askokcancel( title="Print", diff --git a/lib-python/3/idlelib/mainmenu.py b/lib-python/3/idlelib/mainmenu.py index 1b8dc47565..74edce2348 100644 --- a/lib-python/3/idlelib/mainmenu.py +++ b/lib-python/3/idlelib/mainmenu.py @@ -60,6 +60,7 @@ menudefs = [ ]), ('format', [ + ('F_ormat Paragraph', '<<format-paragraph>>'), ('_Indent Region', '<<indent-region>>'), ('_Dedent Region', '<<dedent-region>>'), ('Comment _Out Region', '<<comment-region>>'), @@ -68,15 +69,14 @@ menudefs = [ ('Untabify Region', '<<untabify-region>>'), ('Toggle Tabs', '<<toggle-tabs>>'), ('New Indent Width', '<<change-indentwidth>>'), - ('F_ormat Paragraph', '<<format-paragraph>>'), ('S_trip Trailing Whitespace', '<<do-rstrip>>'), ]), ('run', [ - ('Python Shell', '<<open-python-shell>>'), - ('C_heck Module', '<<check-module>>'), ('R_un Module', '<<run-module>>'), ('Run... _Customized', '<<run-custom>>'), + ('C_heck Module', '<<check-module>>'), + ('Python Shell', '<<open-python-shell>>'), ]), ('shell', [ @@ -100,7 +100,8 @@ menudefs = [ ('Configure _IDLE', '<<open-config-dialog>>'), None, ('Show _Code Context', '<<toggle-code-context>>'), - ('Zoom Height', '<<zoom-height>>'), + ('Show _Line Numbers', '<<toggle-line-numbers>>'), + ('_Zoom Height', '<<zoom-height>>'), ]), ('window', [ diff --git a/lib-python/3/idlelib/outwin.py b/lib-python/3/idlelib/outwin.py index ecc53ef019..90272b6feb 100644 --- a/lib-python/3/idlelib/outwin.py +++ b/lib-python/3/idlelib/outwin.py @@ -74,11 +74,11 @@ class OutputWindow(EditorWindow): ("Go to file/line", "<<goto-file-line>>", None), ] + allow_code_context = False + def __init__(self, *args): EditorWindow.__init__(self, *args) self.text.bind("<<goto-file-line>>", self.goto_file_line) - self.text.unbind("<<toggle-code-context>>") - self.update_menu_state('options', '*Code Context', 'disabled') # Customize EditorWindow def ispythonsource(self, filename): diff --git a/lib-python/3/idlelib/paragraph.py b/lib-python/3/idlelib/paragraph.py deleted file mode 100644 index 81422571fa..0000000000 --- a/lib-python/3/idlelib/paragraph.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Format a paragraph, comment block, or selection to a max width. - -Does basic, standard text formatting, and also understands Python -comment blocks. Thus, for editing Python source code, this -extension is really only suitable for reformatting these comment -blocks or triple-quoted strings. - -Known problems with comment reformatting: -* If there is a selection marked, and the first line of the - selection is not complete, the block will probably not be detected - as comments, and will have the normal "text formatting" rules - applied. -* If a comment block has leading whitespace that mixes tabs and - spaces, they will not be considered part of the same block. -* Fancy comments, like this bulleted list, aren't handled :-) -""" -import re - -from idlelib.config import idleConf - - -class FormatParagraph: - - def __init__(self, editwin): - self.editwin = editwin - - @classmethod - def reload(cls): - cls.max_width = idleConf.GetOption('extensions', 'FormatParagraph', - 'max-width', type='int', default=72) - - def close(self): - self.editwin = None - - def format_paragraph_event(self, event, limit=None): - """Formats paragraph to a max width specified in idleConf. - - If text is selected, format_paragraph_event will start breaking lines - at the max width, starting from the beginning selection. - - If no text is selected, format_paragraph_event uses the current - cursor location to determine the paragraph (lines of text surrounded - by blank lines) and formats it. - - The length limit parameter is for testing with a known value. - """ - limit = self.max_width if limit is None else limit - text = self.editwin.text - first, last = self.editwin.get_selection_indices() - if first and last: - data = text.get(first, last) - comment_header = get_comment_header(data) - else: - first, last, comment_header, data = \ - find_paragraph(text, text.index("insert")) - if comment_header: - newdata = reformat_comment(data, limit, comment_header) - else: - newdata = reformat_paragraph(data, limit) - text.tag_remove("sel", "1.0", "end") - - if newdata != data: - text.mark_set("insert", first) - text.undo_block_start() - text.delete(first, last) - text.insert(first, newdata) - text.undo_block_stop() - else: - text.mark_set("insert", last) - text.see("insert") - return "break" - - -FormatParagraph.reload() - -def find_paragraph(text, mark): - """Returns the start/stop indices enclosing the paragraph that mark is in. - - Also returns the comment format string, if any, and paragraph of text - between the start/stop indices. - """ - lineno, col = map(int, mark.split(".")) - line = text.get("%d.0" % lineno, "%d.end" % lineno) - - # Look for start of next paragraph if the index passed in is a blank line - while text.compare("%d.0" % lineno, "<", "end") and is_all_white(line): - lineno = lineno + 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) - first_lineno = lineno - comment_header = get_comment_header(line) - comment_header_len = len(comment_header) - - # Once start line found, search for end of paragraph (a blank line) - while get_comment_header(line)==comment_header and \ - not is_all_white(line[comment_header_len:]): - lineno = lineno + 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) - last = "%d.0" % lineno - - # Search back to beginning of paragraph (first blank line before) - lineno = first_lineno - 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) - while lineno > 0 and \ - get_comment_header(line)==comment_header and \ - not is_all_white(line[comment_header_len:]): - lineno = lineno - 1 - line = text.get("%d.0" % lineno, "%d.end" % lineno) - first = "%d.0" % (lineno+1) - - return first, last, comment_header, text.get(first, last) - -# This should perhaps be replaced with textwrap.wrap -def reformat_paragraph(data, limit): - """Return data reformatted to specified width (limit).""" - lines = data.split("\n") - i = 0 - n = len(lines) - while i < n and is_all_white(lines[i]): - i = i+1 - if i >= n: - return data - indent1 = get_indent(lines[i]) - if i+1 < n and not is_all_white(lines[i+1]): - indent2 = get_indent(lines[i+1]) - else: - indent2 = indent1 - new = lines[:i] - partial = indent1 - while i < n and not is_all_white(lines[i]): - # XXX Should take double space after period (etc.) into account - words = re.split(r"(\s+)", lines[i]) - for j in range(0, len(words), 2): - word = words[j] - if not word: - continue # Can happen when line ends in whitespace - if len((partial + word).expandtabs()) > limit and \ - partial != indent1: - new.append(partial.rstrip()) - partial = indent2 - partial = partial + word + " " - if j+1 < len(words) and words[j+1] != " ": - partial = partial + " " - i = i+1 - new.append(partial.rstrip()) - # XXX Should reformat remaining paragraphs as well - new.extend(lines[i:]) - return "\n".join(new) - -def reformat_comment(data, limit, comment_header): - """Return data reformatted to specified width with comment header.""" - - # Remove header from the comment lines - lc = len(comment_header) - data = "\n".join(line[lc:] for line in data.split("\n")) - # Reformat to maxformatwidth chars or a 20 char width, - # whichever is greater. - format_width = max(limit - len(comment_header), 20) - newdata = reformat_paragraph(data, format_width) - # re-split and re-insert the comment header. - newdata = newdata.split("\n") - # If the block ends in a \n, we don't want the comment prefix - # inserted after it. (Im not sure it makes sense to reformat a - # comment block that is not made of complete lines, but whatever!) - # Can't think of a clean solution, so we hack away - block_suffix = "" - if not newdata[-1]: - block_suffix = "\n" - newdata = newdata[:-1] - return '\n'.join(comment_header+line for line in newdata) + block_suffix - -def is_all_white(line): - """Return True if line is empty or all whitespace.""" - - return re.match(r"^\s*$", line) is not None - -def get_indent(line): - """Return the initial space or tab indent of line.""" - return re.match(r"^([ \t]*)", line).group() - -def get_comment_header(line): - """Return string with leading whitespace and '#' from line or ''. - - A null return indicates that the line is not a comment line. A non- - null return, such as ' #', will be used to find the other lines of - a comment block with the same indent. - """ - m = re.match(r"^([ \t]*#*)", line) - if m is None: return "" - return m.group(1) - - -if __name__ == "__main__": - from unittest import main - main('idlelib.idle_test.test_paragraph', verbosity=2, exit=False) diff --git a/lib-python/3/idlelib/pyparse.py b/lib-python/3/idlelib/pyparse.py index 81e7f53980..d34872b439 100644 --- a/lib-python/3/idlelib/pyparse.py +++ b/lib-python/3/idlelib/pyparse.py @@ -133,8 +133,7 @@ class Parser: self.code = s self.study_level = 0 - def find_good_parse_start(self, is_char_in_string=None, - _synchre=_synchre): + def find_good_parse_start(self, is_char_in_string): """ Return index of a good place to begin parsing, as close to the end of the string as possible. This will be the start of some @@ -149,10 +148,6 @@ class Parser: """ code, pos = self.code, None - if not is_char_in_string: - # no clue -- make the caller pass everything - return None - # Peek back from the end for a good place to start, # but don't try too often; pos will be left None, or # bumped to a legitimate synch point. @@ -575,7 +570,7 @@ class Parser: return code[i:j] def is_block_opener(self): - "Return True if the last interesting statemtent opens a block." + "Return True if the last interesting statement opens a block." self._study2() return self.lastch == ':' diff --git a/lib-python/3/idlelib/pyshell.py b/lib-python/3/idlelib/pyshell.py index 7ad5a76c3b..66ae0f7435 100755 --- a/lib-python/3/idlelib/pyshell.py +++ b/lib-python/3/idlelib/pyshell.py @@ -16,7 +16,7 @@ except ImportError: if sys.platform == 'win32': try: import ctypes - PROCESS_SYSTEM_DPI_AWARE = 1 + PROCESS_SYSTEM_DPI_AWARE = 1 # Int required. ctypes.OleDLL('shcore').SetProcessDpiAwareness(PROCESS_SYSTEM_DPI_AWARE) except (ImportError, AttributeError, OSError): pass @@ -54,7 +54,7 @@ from idlelib.editor import EditorWindow, fixwordbreaks from idlelib.filelist import FileList from idlelib.outwin import OutputWindow from idlelib import rpc -from idlelib.run import idle_formatwarning, PseudoInputFile, PseudoOutputFile +from idlelib.run import idle_formatwarning, StdInputFile, StdOutputFile from idlelib.undo import UndoDelegator HOST = '127.0.0.1' # python execution server on localhost loopback @@ -133,6 +133,7 @@ class PyShellEditorWindow(EditorWindow): self.text.bind("<<clear-breakpoint-here>>", self.clear_breakpoint_here) self.text.bind("<<open-python-shell>>", self.flist.open_shell) + #TODO: don't read/write this from/to .idlerc when testing self.breakpointPath = os.path.join( idleConf.userdir, 'breakpoints.lst') # whenever a file is changed, restore breakpoints @@ -387,6 +388,19 @@ class MyRPCClient(rpc.RPCClient): "Override the base class - just re-raise EOFError" raise EOFError +def restart_line(width, filename): # See bpo-38141. + """Return width long restart line formatted with filename. + + Fill line with balanced '='s, with any extras and at least one at + the beginning. Do not end with a trailing space. + """ + tag = f"= RESTART: {filename or 'Shell'} =" + if width >= len(tag): + div, mod = divmod((width -len(tag)), 2) + return f"{(div+mod)*'='}{tag}{div*'='}" + else: + return tag[:-2] # Remove ' ='. + class ModifiedInterpreter(InteractiveInterpreter): @@ -394,7 +408,6 @@ class ModifiedInterpreter(InteractiveInterpreter): self.tkconsole = tkconsole locals = sys.modules['__main__'].__dict__ InteractiveInterpreter.__init__(self, locals=locals) - self.save_warnings_filters = None self.restarting = False self.subprocess_arglist = None self.port = PORT @@ -492,9 +505,8 @@ class ModifiedInterpreter(InteractiveInterpreter): console.stop_readline() # annotate restart in shell window and mark it console.text.delete("iomark", "end-1c") - tag = 'RESTART: ' + (filename if filename else 'Shell') - halfbar = ((int(console.width) -len(tag) - 4) // 2) * '=' - console.write("\n{0} {1} {0}".format(halfbar, tag)) + console.write('\n') + console.write(restart_line(console.width, filename)) console.text.mark_set("restart", "end-1c") console.text.mark_gravity("restart", "left") if not filename: @@ -664,27 +676,11 @@ class ModifiedInterpreter(InteractiveInterpreter): def runsource(self, source): "Extend base class method: Stuff the source in the line cache first" filename = self.stuffsource(source) - self.more = 0 - self.save_warnings_filters = warnings.filters[:] - warnings.filterwarnings(action="error", category=SyntaxWarning) # at the moment, InteractiveInterpreter expects str assert isinstance(source, str) - #if isinstance(source, str): - # from idlelib import iomenu - # try: - # source = source.encode(iomenu.encoding) - # except UnicodeError: - # self.tkconsole.resetoutput() - # self.write("Unsupported characters in input\n") - # return - try: - # InteractiveInterpreter.runsource() calls its runcode() method, - # which is overridden (see below) - return InteractiveInterpreter.runsource(self, source, filename) - finally: - if self.save_warnings_filters is not None: - warnings.filters[:] = self.save_warnings_filters - self.save_warnings_filters = None + # InteractiveInterpreter.runsource() calls its runcode() method, + # which is overridden (see below) + return InteractiveInterpreter.runsource(self, source, filename) def stuffsource(self, source): "Stuff source in the filename cache" @@ -763,9 +759,6 @@ class ModifiedInterpreter(InteractiveInterpreter): if self.tkconsole.executing: self.interp.restart_subprocess() self.checklinecache() - if self.save_warnings_filters is not None: - warnings.filters[:] = self.save_warnings_filters - self.save_warnings_filters = None debugger = self.debugger try: self.tkconsole.beginexecuting() @@ -861,6 +854,8 @@ class PyShell(OutputWindow): ("Squeeze", "<<squeeze-current-text>>"), ] + allow_line_numbers = False + # New classes from idlelib.history import History @@ -906,10 +901,14 @@ class PyShell(OutputWindow): self.save_stderr = sys.stderr self.save_stdin = sys.stdin from idlelib import iomenu - self.stdin = PseudoInputFile(self, "stdin", iomenu.encoding) - self.stdout = PseudoOutputFile(self, "stdout", iomenu.encoding) - self.stderr = PseudoOutputFile(self, "stderr", iomenu.encoding) - self.console = PseudoOutputFile(self, "console", iomenu.encoding) + self.stdin = StdInputFile(self, "stdin", + iomenu.encoding, iomenu.errors) + self.stdout = StdOutputFile(self, "stdout", + iomenu.encoding, iomenu.errors) + self.stderr = StdOutputFile(self, "stderr", + iomenu.encoding, "backslashreplace") + self.console = StdOutputFile(self, "console", + iomenu.encoding, iomenu.errors) if not use_subprocess: sys.stdout = self.stdout sys.stderr = self.stderr @@ -993,12 +992,12 @@ class PyShell(OutputWindow): def beginexecuting(self): "Helper for ModifiedInterpreter" self.resetoutput() - self.executing = 1 + self.executing = True def endexecuting(self): "Helper for ModifiedInterpreter" - self.executing = 0 - self.canceled = 0 + self.executing = False + self.canceled = False self.showprompt() def close(self): @@ -1075,7 +1074,7 @@ class PyShell(OutputWindow): def readline(self): save = self.reading try: - self.reading = 1 + self.reading = True self.top.mainloop() # nested mainloop() finally: self.reading = save @@ -1087,11 +1086,11 @@ class PyShell(OutputWindow): line = "\n" self.resetoutput() if self.canceled: - self.canceled = 0 + self.canceled = False if not use_subprocess: raise KeyboardInterrupt if self.endoffile: - self.endoffile = 0 + self.endoffile = False line = "" return line @@ -1109,8 +1108,8 @@ class PyShell(OutputWindow): self.interp.write("KeyboardInterrupt\n") self.showprompt() return "break" - self.endoffile = 0 - self.canceled = 1 + self.endoffile = False + self.canceled = True if (self.executing and self.interp.rpcclt): if self.interp.getdebugger(): self.interp.restart_subprocess() @@ -1130,8 +1129,8 @@ class PyShell(OutputWindow): self.resetoutput() self.close() else: - self.canceled = 0 - self.endoffile = 1 + self.canceled = False + self.endoffile = True self.top.quit() return "break" @@ -1292,18 +1291,9 @@ class PyShell(OutputWindow): self.text.insert("end-1c", "\n") self.text.mark_set("iomark", "end-1c") self.set_line_and_column() + self.ctip.remove_calltip_window() def write(self, s, tags=()): - if isinstance(s, str) and len(s) and max(s) > '\uffff': - # Tk doesn't support outputting non-BMP characters - # Let's assume what printed string is not very long, - # find first non-BMP character and construct informative - # UnicodeEncodeError exception. - for start, char in enumerate(s): - if char > '\uffff': - break - raise UnicodeEncodeError("UCS-2", char, start, start+1, - 'Non-BMP character not supported in Tk') try: self.text.mark_gravity("iomark", "right") count = OutputWindow.write(self, s, tags, "iomark") @@ -1312,7 +1302,7 @@ class PyShell(OutputWindow): raise ###pass # ### 11Aug07 KBK if we are expecting exceptions # let's find out what they are and be specific. if self.canceled: - self.canceled = 0 + self.canceled = False if not use_subprocess: raise KeyboardInterrupt return count @@ -1495,9 +1485,14 @@ def main(): iconfile = os.path.join(icondir, 'idle.ico') root.wm_iconbitmap(default=iconfile) elif not macosx.isAquaTk(): - ext = '.png' if TkVersion >= 8.6 else '.gif' + if TkVersion >= 8.6: + ext = '.png' + sizes = (16, 32, 48, 256) + else: + ext = '.gif' + sizes = (16, 32, 48) iconfiles = [os.path.join(icondir, 'idle_%d%s' % (size, ext)) - for size in (16, 32, 48)] + for size in sizes] icons = [PhotoImage(master=root, file=iconfile) for iconfile in iconfiles] root.wm_iconphoto(True, *icons) diff --git a/lib-python/3/idlelib/query.py b/lib-python/3/idlelib/query.py index 9b3bb1d186..2a88530b4d 100644 --- a/lib-python/3/idlelib/query.py +++ b/lib-python/3/idlelib/query.py @@ -36,10 +36,10 @@ class Query(Toplevel): """ def __init__(self, parent, title, message, *, text0='', used_names={}, _htest=False, _utest=False): - """Create popup, do not return until tk widget destroyed. + """Create modal popup, return when destroyed. - Additional subclass init must be done before calling this - unless _utest=True is passed to suppress wait_window(). + Additional subclass init must be done before this unless + _utest=True is passed to suppress wait_window(). title - string, title of popup dialog message - string, informational message to display @@ -48,15 +48,17 @@ class Query(Toplevel): _htest - bool, change box location when running htest _utest - bool, leave window hidden and not modal """ - Toplevel.__init__(self, parent) - self.withdraw() # Hide while configuring, especially geometry. - self.parent = parent - self.title(title) + self.parent = parent # Needed for Font call. self.message = message self.text0 = text0 self.used_names = used_names + + Toplevel.__init__(self, parent) + self.withdraw() # Hide while configuring, especially geometry. + self.title(title) self.transient(parent) self.grab_set() + windowingsystem = self.tk.call('tk', 'windowingsystem') if windowingsystem == 'aqua': try: @@ -69,9 +71,9 @@ class Query(Toplevel): self.protocol("WM_DELETE_WINDOW", self.cancel) self.bind('<Key-Return>', self.ok) self.bind("<KP_Enter>", self.ok) - self.resizable(height=False, width=False) + self.create_widgets() - self.update_idletasks() # Needed here for winfo_reqwidth below. + self.update_idletasks() # Need here for winfo_reqwidth below. self.geometry( # Center dialog over parent (or below htest box). "+%d+%d" % ( parent.winfo_rootx() + @@ -80,12 +82,19 @@ class Query(Toplevel): ((parent.winfo_height()/2 - self.winfo_reqheight()/2) if not _htest else 150) ) ) + self.resizable(height=False, width=False) + if not _utest: self.deiconify() # Unhide now that geometry set. self.wait_window() - def create_widgets(self, ok_text='OK'): # Call from override, if any. - # Bind to self widgets needed for entry_ok or unittest. + def create_widgets(self, ok_text='OK'): # Do not replace. + """Create entry (rows, extras, buttons. + + Entry stuff on rows 0-2, spanning cols 0-2. + Buttons on row 99, cols 1, 2. + """ + # Bind to self the widgets needed for entry_ok or unittest. self.frame = frame = Frame(self, padding=10) frame.grid(column=0, row=0, sticky='news') frame.grid_columnconfigure(0, weight=1) @@ -99,26 +108,31 @@ class Query(Toplevel): exists=True, root=self.parent) self.entry_error = Label(frame, text=' ', foreground='red', font=self.error_font) - self.button_ok = Button( - frame, text=ok_text, default='active', command=self.ok) - self.button_cancel = Button( - frame, text='Cancel', command=self.cancel) - + # Display or blank error by setting ['text'] =. entrylabel.grid(column=0, row=0, columnspan=3, padx=5, sticky=W) self.entry.grid(column=0, row=1, columnspan=3, padx=5, sticky=W+E, pady=[10,0]) self.entry_error.grid(column=0, row=2, columnspan=3, padx=5, sticky=W+E) + + self.create_extra() + + self.button_ok = Button( + frame, text=ok_text, default='active', command=self.ok) + self.button_cancel = Button( + frame, text='Cancel', command=self.cancel) + self.button_ok.grid(column=1, row=99, padx=5) self.button_cancel.grid(column=2, row=99, padx=5) + def create_extra(self): pass # Override to add widgets. + def showerror(self, message, widget=None): #self.bell(displayof=self) (widget or self.entry_error)['text'] = 'ERROR: ' + message def entry_ok(self): # Example: usually replace. "Return non-blank entry or None." - self.entry_error['text'] = '' entry = self.entry.get().strip() if not entry: self.showerror('blank line.') @@ -130,6 +144,7 @@ class Query(Toplevel): Otherwise leave dialog open for user to correct entry or cancel. ''' + self.entry_error['text'] = '' entry = self.entry_ok() if entry is not None: self.result = entry @@ -159,7 +174,6 @@ class SectionName(Query): def entry_ok(self): "Return sensible ConfigParser section name or None." - self.entry_error['text'] = '' name = self.entry.get().strip() if not name: self.showerror('no name specified.') @@ -184,7 +198,6 @@ class ModuleName(Query): def entry_ok(self): "Return entered module name as file path or None." - self.entry_error['text'] = '' name = self.entry.get().strip() if not name: self.showerror('no name specified.') @@ -210,6 +223,22 @@ class ModuleName(Query): return file_path +class Goto(Query): + "Get a positive line number for editor Go To Line." + # Used in editor.EditorWindow.goto_line_event. + + def entry_ok(self): + try: + lineno = int(self.entry.get()) + except ValueError: + self.showerror('not a base 10 integer.') + return None + if lineno <= 0: + self.showerror('not a positive integer.') + return None + return lineno + + class HelpSource(Query): "Get menu name and help source for Help menu." # Used in ConfigDialog.HelpListItemAdd/Edit, (941/9) @@ -227,8 +256,8 @@ class HelpSource(Query): parent, title, message, text0=menuitem, used_names=used_names, _htest=_htest, _utest=_utest) - def create_widgets(self): - super().create_widgets() + def create_extra(self): + "Add path widjets to rows 10-12." frame = self.frame pathlabel = Label(frame, anchor='w', justify='left', text='Help File Path: Enter URL or browse for file') @@ -297,7 +326,6 @@ class HelpSource(Query): def entry_ok(self): "Return apparently valid (name, path) or None" - self.entry_error['text'] = '' self.path_error['text'] = '' name = self.item_ok() path = self.path_ok() @@ -311,16 +339,20 @@ class CustomRun(Query): """ # Used in runscript.run_custom_event - def __init__(self, parent, title, *, cli_args='', + def __init__(self, parent, title, *, cli_args=[], _htest=False, _utest=False): - # TODO Use cli_args to pre-populate entry. + """cli_args is a list of strings. + + The list is assigned to the default Entry StringVar. + The strings are displayed joined by ' ' for display. + """ message = 'Command Line Arguments for sys.argv:' super().__init__( parent, title, message, text0=cli_args, _htest=_htest, _utest=_utest) - def create_widgets(self): - super().create_widgets(ok_text='Run') + def create_extra(self): + "Add run mode on rows 10-12." frame = self.frame self.restartvar = BooleanVar(self, value=True) restart = Checkbutton(frame, variable=self.restartvar, onvalue=True, @@ -328,7 +360,7 @@ class CustomRun(Query): self.args_error = Label(frame, text=' ', foreground='red', font=self.error_font) - restart.grid(column=0, row=4, columnspan=3, padx=5, sticky='w') + restart.grid(column=0, row=10, columnspan=3, padx=5, sticky='w') self.args_error.grid(column=0, row=12, columnspan=3, padx=5, sticky='we') @@ -344,7 +376,6 @@ class CustomRun(Query): def entry_ok(self): "Return apparently valid (cli_args, restart) or None" - self.entry_error['text'] = '' cli_args = self.cli_args_ok() restart = self.restartvar.get() return None if cli_args is None else (cli_args, restart) diff --git a/lib-python/3/idlelib/rstrip.py b/lib-python/3/idlelib/rstrip.py deleted file mode 100644 index f93b5e8fc2..0000000000 --- a/lib-python/3/idlelib/rstrip.py +++ /dev/null @@ -1,29 +0,0 @@ -'Provides "Strip trailing whitespace" under the "Format" menu.' - -class Rstrip: - - def __init__(self, editwin): - self.editwin = editwin - - def do_rstrip(self, event=None): - - text = self.editwin.text - undo = self.editwin.undo - - undo.undo_block_start() - - end_line = int(float(text.index('end'))) - for cur in range(1, end_line): - txt = text.get('%i.0' % cur, '%i.end' % cur) - raw = len(txt) - cut = len(txt.rstrip()) - # Since text.delete() marks file as changed, even if not, - # only call it when needed to actually delete something. - if cut < raw: - text.delete('%i.%i' % (cur, cut), '%i.end' % cur) - - undo.undo_block_stop() - -if __name__ == "__main__": - from unittest import main - main('idlelib.idle_test.test_rstrip', verbosity=2,) diff --git a/lib-python/3/idlelib/run.py b/lib-python/3/idlelib/run.py index 6b3928b7bf..5bd84aadcd 100644 --- a/lib-python/3/idlelib/run.py +++ b/lib-python/3/idlelib/run.py @@ -4,10 +4,12 @@ Simplified, pyshell.ModifiedInterpreter spawns a subprocess with f'''{sys.executable} -c "__import__('idlelib.run').run.main()"''' '.run' is needed because __import__ returns idlelib, not idlelib.run. """ +import functools import io import linecache import queue import sys +import textwrap import time import traceback import _thread as thread @@ -305,6 +307,67 @@ def fix_scaling(root): font['size'] = round(-0.75*size) +def fixdoc(fun, text): + tem = (fun.__doc__ + '\n\n') if fun.__doc__ is not None else '' + fun.__doc__ = tem + textwrap.fill(textwrap.dedent(text)) + +RECURSIONLIMIT_DELTA = 30 + +def install_recursionlimit_wrappers(): + """Install wrappers to always add 30 to the recursion limit.""" + # see: bpo-26806 + + @functools.wraps(sys.setrecursionlimit) + def setrecursionlimit(*args, **kwargs): + # mimic the original sys.setrecursionlimit()'s input handling + if kwargs: + raise TypeError( + "setrecursionlimit() takes no keyword arguments") + try: + limit, = args + except ValueError: + raise TypeError(f"setrecursionlimit() takes exactly one " + f"argument ({len(args)} given)") + if not limit > 0: + raise ValueError( + "recursion limit must be greater or equal than 1") + + return setrecursionlimit.__wrapped__(limit + RECURSIONLIMIT_DELTA) + + fixdoc(setrecursionlimit, f"""\ + This IDLE wrapper adds {RECURSIONLIMIT_DELTA} to prevent possible + uninterruptible loops.""") + + @functools.wraps(sys.getrecursionlimit) + def getrecursionlimit(): + return getrecursionlimit.__wrapped__() - RECURSIONLIMIT_DELTA + + fixdoc(getrecursionlimit, f"""\ + This IDLE wrapper subtracts {RECURSIONLIMIT_DELTA} to compensate + for the {RECURSIONLIMIT_DELTA} IDLE adds when setting the limit.""") + + # add the delta to the default recursion limit, to compensate + sys.setrecursionlimit(sys.getrecursionlimit() + RECURSIONLIMIT_DELTA) + + sys.setrecursionlimit = setrecursionlimit + sys.getrecursionlimit = getrecursionlimit + + +def uninstall_recursionlimit_wrappers(): + """Uninstall the recursion limit wrappers from the sys module. + + IDLE only uses this for tests. Users can import run and call + this to remove the wrapping. + """ + if ( + getattr(sys.setrecursionlimit, '__wrapped__', None) and + getattr(sys.getrecursionlimit, '__wrapped__', None) + ): + sys.setrecursionlimit = sys.setrecursionlimit.__wrapped__ + sys.getrecursionlimit = sys.getrecursionlimit.__wrapped__ + sys.setrecursionlimit(sys.getrecursionlimit() - RECURSIONLIMIT_DELTA) + + class MyRPCServer(rpc.RPCServer): def handle_error(self, request, client_address): @@ -338,18 +401,23 @@ class MyRPCServer(rpc.RPCServer): # Pseudofiles for shell-remote communication (also used in pyshell) -class PseudoFile(io.TextIOBase): +class StdioFile(io.TextIOBase): - def __init__(self, shell, tags, encoding=None): + def __init__(self, shell, tags, encoding='utf-8', errors='strict'): self.shell = shell self.tags = tags self._encoding = encoding + self._errors = errors @property def encoding(self): return self._encoding @property + def errors(self): + return self._errors + + @property def name(self): return '<%s>' % self.tags @@ -357,7 +425,7 @@ class PseudoFile(io.TextIOBase): return True -class PseudoOutputFile(PseudoFile): +class StdOutputFile(StdioFile): def writable(self): return True @@ -365,19 +433,12 @@ class PseudoOutputFile(PseudoFile): def write(self, s): if self.closed: raise ValueError("write to closed file") - if type(s) is not str: - if not isinstance(s, str): - raise TypeError('must be str, not ' + type(s).__name__) - # See issue #19481 - s = str.__str__(s) + s = str.encode(s, self.encoding, self.errors).decode(self.encoding, self.errors) return self.shell.write(s, self.tags) -class PseudoInputFile(PseudoFile): - - def __init__(self, shell, tags, encoding=None): - PseudoFile.__init__(self, shell, tags, encoding) - self._line_buffer = '' +class StdInputFile(StdioFile): + _line_buffer = '' def readable(self): return True @@ -432,12 +493,12 @@ class MyHandler(rpc.RPCHandler): executive = Executive(self) self.register("exec", executive) self.console = self.get_remote_proxy("console") - sys.stdin = PseudoInputFile(self.console, "stdin", - iomenu.encoding) - sys.stdout = PseudoOutputFile(self.console, "stdout", - iomenu.encoding) - sys.stderr = PseudoOutputFile(self.console, "stderr", - iomenu.encoding) + sys.stdin = StdInputFile(self.console, "stdin", + iomenu.encoding, iomenu.errors) + sys.stdout = StdOutputFile(self.console, "stdout", + iomenu.encoding, iomenu.errors) + sys.stderr = StdOutputFile(self.console, "stderr", + iomenu.encoding, "backslashreplace") sys.displayhook = rpc.displayhook # page help() text to shell. @@ -448,6 +509,8 @@ class MyHandler(rpc.RPCHandler): # sys.stdin gets changed from within IDLE's shell. See issue17838. self._keep_stdin = sys.stdin + install_recursionlimit_wrappers() + self.interp = self.get_remote_proxy("interp") rpc.RPCHandler.getresponse(self, myseq=None, wait=0.05) diff --git a/lib-python/3/idlelib/runscript.py b/lib-python/3/idlelib/runscript.py index b041e56fb8..a54108794a 100644 --- a/lib-python/3/idlelib/runscript.py +++ b/lib-python/3/idlelib/runscript.py @@ -19,6 +19,7 @@ from idlelib.config import idleConf from idlelib import macosx from idlelib import pyshell from idlelib.query import CustomRun +from idlelib import outwin indent_message = """Error: Inconsistent indentation detected! @@ -39,11 +40,16 @@ class ScriptBinding: # XXX This should be done differently self.flist = self.editwin.flist self.root = self.editwin.root + # cli_args is list of strings that extends sys.argv + self.cli_args = [] if macosx.isCocoaTk(): self.editwin.text_frame.bind('<<run-module-event-2>>', self._run_module_event) def check_module_event(self, event): + if isinstance(self.editwin, outwin.OutputWindow): + self.editwin.text.bell() + return 'break' filename = self.getfilename() if not filename: return 'break' @@ -127,6 +133,9 @@ class ScriptBinding: module being executed and also add that directory to its sys.path if not already included. """ + if isinstance(self.editwin, outwin.OutputWindow): + self.editwin.text.bell() + return 'break' filename = self.getfilename() if not filename: return 'break' @@ -137,19 +146,19 @@ class ScriptBinding: return 'break' if customize: title = f"Customize {self.editwin.short_title()} Run" - run_args = CustomRun(self.shell.text, title).result + run_args = CustomRun(self.shell.text, title, + cli_args=self.cli_args).result if not run_args: # User cancelled. return 'break' - cli_args, restart = run_args if customize else ([], True) + self.cli_args, restart = run_args if customize else ([], True) interp = self.shell.interp if pyshell.use_subprocess and restart: interp.restart_subprocess( - with_cwd=False, filename= - self.editwin._filename_to_unicode(filename)) + with_cwd=False, filename=filename) dirname = os.path.dirname(filename) argv = [filename] - if cli_args: - argv += cli_args + if self.cli_args: + argv += self.cli_args interp.runcommand(f"""if 1: __file__ = {filename!r} import sys as _sys @@ -161,7 +170,7 @@ class ScriptBinding: _sys.argv = argv import os as _os _os.chdir({dirname!r}) - del _sys, _basename, _os + del _sys, argv, _basename, _os \n""") interp.prepend_syspath(filename) # XXX KBK 03Jul04 When run w/o subprocess, runtime warnings still diff --git a/lib-python/3/idlelib/sidebar.py b/lib-python/3/idlelib/sidebar.py new file mode 100644 index 0000000000..41c09684a2 --- /dev/null +++ b/lib-python/3/idlelib/sidebar.py @@ -0,0 +1,341 @@ +"""Line numbering implementation for IDLE as an extension. +Includes BaseSideBar which can be extended for other sidebar based extensions +""" +import functools +import itertools + +import tkinter as tk +from idlelib.config import idleConf +from idlelib.delegator import Delegator + + +def get_end_linenumber(text): + """Utility to get the last line's number in a Tk text widget.""" + return int(float(text.index('end-1c'))) + + +def get_widget_padding(widget): + """Get the total padding of a Tk widget, including its border.""" + # TODO: use also in codecontext.py + manager = widget.winfo_manager() + if manager == 'pack': + info = widget.pack_info() + elif manager == 'grid': + info = widget.grid_info() + else: + raise ValueError(f"Unsupported geometry manager: {manager}") + + # All values are passed through getint(), since some + # values may be pixel objects, which can't simply be added to ints. + padx = sum(map(widget.tk.getint, [ + info['padx'], + widget.cget('padx'), + widget.cget('border'), + ])) + pady = sum(map(widget.tk.getint, [ + info['pady'], + widget.cget('pady'), + widget.cget('border'), + ])) + return padx, pady + + +class BaseSideBar: + """ + The base class for extensions which require a sidebar. + """ + def __init__(self, editwin): + self.editwin = editwin + self.parent = editwin.text_frame + self.text = editwin.text + + _padx, pady = get_widget_padding(self.text) + self.sidebar_text = tk.Text(self.parent, width=1, wrap=tk.NONE, + padx=2, pady=pady, + borderwidth=0, highlightthickness=0) + self.sidebar_text.config(state=tk.DISABLED) + self.text['yscrollcommand'] = self.redirect_yscroll_event + self.update_font() + self.update_colors() + + self.is_shown = False + + def update_font(self): + """Update the sidebar text font, usually after config changes.""" + font = idleConf.GetFont(self.text, 'main', 'EditorWindow') + self._update_font(font) + + def _update_font(self, font): + self.sidebar_text['font'] = font + + def update_colors(self): + """Update the sidebar text colors, usually after config changes.""" + colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'normal') + self._update_colors(foreground=colors['foreground'], + background=colors['background']) + + def _update_colors(self, foreground, background): + self.sidebar_text.config( + fg=foreground, bg=background, + selectforeground=foreground, selectbackground=background, + inactiveselectbackground=background, + ) + + def show_sidebar(self): + if not self.is_shown: + self.sidebar_text.grid(row=1, column=0, sticky=tk.NSEW) + self.is_shown = True + + def hide_sidebar(self): + if self.is_shown: + self.sidebar_text.grid_forget() + self.is_shown = False + + def redirect_yscroll_event(self, *args, **kwargs): + """Redirect vertical scrolling to the main editor text widget. + + The scroll bar is also updated. + """ + self.editwin.vbar.set(*args) + self.sidebar_text.yview_moveto(args[0]) + return 'break' + + def redirect_focusin_event(self, event): + """Redirect focus-in events to the main editor text widget.""" + self.text.focus_set() + return 'break' + + def redirect_mousebutton_event(self, event, event_name): + """Redirect mouse button events to the main editor text widget.""" + self.text.focus_set() + self.text.event_generate(event_name, x=0, y=event.y) + return 'break' + + def redirect_mousewheel_event(self, event): + """Redirect mouse wheel events to the editwin text widget.""" + self.text.event_generate('<MouseWheel>', + x=0, y=event.y, delta=event.delta) + return 'break' + + +class EndLineDelegator(Delegator): + """Generate callbacks with the current end line number after + insert or delete operations""" + def __init__(self, changed_callback): + """ + changed_callback - Callable, will be called after insert + or delete operations with the current + end line number. + """ + Delegator.__init__(self) + self.changed_callback = changed_callback + + def insert(self, index, chars, tags=None): + self.delegate.insert(index, chars, tags) + self.changed_callback(get_end_linenumber(self.delegate)) + + def delete(self, index1, index2=None): + self.delegate.delete(index1, index2) + self.changed_callback(get_end_linenumber(self.delegate)) + + +class LineNumbers(BaseSideBar): + """Line numbers support for editor windows.""" + def __init__(self, editwin): + BaseSideBar.__init__(self, editwin) + self.prev_end = 1 + self._sidebar_width_type = type(self.sidebar_text['width']) + self.sidebar_text.config(state=tk.NORMAL) + self.sidebar_text.insert('insert', '1', 'linenumber') + self.sidebar_text.config(state=tk.DISABLED) + self.sidebar_text.config(takefocus=False, exportselection=False) + self.sidebar_text.tag_config('linenumber', justify=tk.RIGHT) + + self.bind_events() + + end = get_end_linenumber(self.text) + self.update_sidebar_text(end) + + end_line_delegator = EndLineDelegator(self.update_sidebar_text) + # Insert the delegator after the undo delegator, so that line numbers + # are properly updated after undo and redo actions. + end_line_delegator.setdelegate(self.editwin.undo.delegate) + self.editwin.undo.setdelegate(end_line_delegator) + # Reset the delegator caches of the delegators "above" the + # end line delegator we just inserted. + delegator = self.editwin.per.top + while delegator is not end_line_delegator: + delegator.resetcache() + delegator = delegator.delegate + + self.is_shown = False + + def bind_events(self): + # Ensure focus is always redirected to the main editor text widget. + self.sidebar_text.bind('<FocusIn>', self.redirect_focusin_event) + + # Redirect mouse scrolling to the main editor text widget. + # + # Note that without this, scrolling with the mouse only scrolls + # the line numbers. + self.sidebar_text.bind('<MouseWheel>', self.redirect_mousewheel_event) + + # Redirect mouse button events to the main editor text widget, + # except for the left mouse button (1). + # + # Note: X-11 sends Button-4 and Button-5 events for the scroll wheel. + def bind_mouse_event(event_name, target_event_name): + handler = functools.partial(self.redirect_mousebutton_event, + event_name=target_event_name) + self.sidebar_text.bind(event_name, handler) + + for button in [2, 3, 4, 5]: + for event_name in (f'<Button-{button}>', + f'<ButtonRelease-{button}>', + f'<B{button}-Motion>', + ): + bind_mouse_event(event_name, target_event_name=event_name) + + # Convert double- and triple-click events to normal click events, + # since event_generate() doesn't allow generating such events. + for event_name in (f'<Double-Button-{button}>', + f'<Triple-Button-{button}>', + ): + bind_mouse_event(event_name, + target_event_name=f'<Button-{button}>') + + # This is set by b1_mousedown_handler() and read by + # drag_update_selection_and_insert_mark(), to know where dragging + # began. + start_line = None + # These are set by b1_motion_handler() and read by selection_handler(). + # last_y is passed this way since the mouse Y-coordinate is not + # available on selection event objects. last_yview is passed this way + # to recognize scrolling while the mouse isn't moving. + last_y = last_yview = None + + def b1_mousedown_handler(event): + # select the entire line + lineno = int(float(self.sidebar_text.index(f"@0,{event.y}"))) + self.text.tag_remove("sel", "1.0", "end") + self.text.tag_add("sel", f"{lineno}.0", f"{lineno+1}.0") + self.text.mark_set("insert", f"{lineno+1}.0") + + # remember this line in case this is the beginning of dragging + nonlocal start_line + start_line = lineno + self.sidebar_text.bind('<Button-1>', b1_mousedown_handler) + + def b1_mouseup_handler(event): + # On mouse up, we're no longer dragging. Set the shared persistent + # variables to None to represent this. + nonlocal start_line + nonlocal last_y + nonlocal last_yview + start_line = None + last_y = None + last_yview = None + self.sidebar_text.bind('<ButtonRelease-1>', b1_mouseup_handler) + + def drag_update_selection_and_insert_mark(y_coord): + """Helper function for drag and selection event handlers.""" + lineno = int(float(self.sidebar_text.index(f"@0,{y_coord}"))) + a, b = sorted([start_line, lineno]) + self.text.tag_remove("sel", "1.0", "end") + self.text.tag_add("sel", f"{a}.0", f"{b+1}.0") + self.text.mark_set("insert", + f"{lineno if lineno == a else lineno + 1}.0") + + # Special handling of dragging with mouse button 1. In "normal" text + # widgets this selects text, but the line numbers text widget has + # selection disabled. Still, dragging triggers some selection-related + # functionality under the hood. Specifically, dragging to above or + # below the text widget triggers scrolling, in a way that bypasses the + # other scrolling synchronization mechanisms.i + def b1_drag_handler(event, *args): + nonlocal last_y + nonlocal last_yview + last_y = event.y + last_yview = self.sidebar_text.yview() + if not 0 <= last_y <= self.sidebar_text.winfo_height(): + self.text.yview_moveto(last_yview[0]) + drag_update_selection_and_insert_mark(event.y) + self.sidebar_text.bind('<B1-Motion>', b1_drag_handler) + + # With mouse-drag scrolling fixed by the above, there is still an edge- + # case we need to handle: When drag-scrolling, scrolling can continue + # while the mouse isn't moving, leading to the above fix not scrolling + # properly. + def selection_handler(event): + if last_yview is None: + # This logic is only needed while dragging. + return + yview = self.sidebar_text.yview() + if yview != last_yview: + self.text.yview_moveto(yview[0]) + drag_update_selection_and_insert_mark(last_y) + self.sidebar_text.bind('<<Selection>>', selection_handler) + + def update_colors(self): + """Update the sidebar text colors, usually after config changes.""" + colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'linenumber') + self._update_colors(foreground=colors['foreground'], + background=colors['background']) + + def update_sidebar_text(self, end): + """ + Perform the following action: + Each line sidebar_text contains the linenumber for that line + Synchronize with editwin.text so that both sidebar_text and + editwin.text contain the same number of lines""" + if end == self.prev_end: + return + + width_difference = len(str(end)) - len(str(self.prev_end)) + if width_difference: + cur_width = int(float(self.sidebar_text['width'])) + new_width = cur_width + width_difference + self.sidebar_text['width'] = self._sidebar_width_type(new_width) + + self.sidebar_text.config(state=tk.NORMAL) + if end > self.prev_end: + new_text = '\n'.join(itertools.chain( + [''], + map(str, range(self.prev_end + 1, end + 1)), + )) + self.sidebar_text.insert(f'end -1c', new_text, 'linenumber') + else: + self.sidebar_text.delete(f'{end+1}.0 -1c', 'end -1c') + self.sidebar_text.config(state=tk.DISABLED) + + self.prev_end = end + + +def _linenumbers_drag_scrolling(parent): # htest # + from idlelib.idle_test.test_sidebar import Dummy_editwin + + toplevel = tk.Toplevel(parent) + text_frame = tk.Frame(toplevel) + text_frame.pack(side=tk.LEFT, fill=tk.BOTH, expand=True) + text_frame.rowconfigure(1, weight=1) + text_frame.columnconfigure(1, weight=1) + + font = idleConf.GetFont(toplevel, 'main', 'EditorWindow') + text = tk.Text(text_frame, width=80, height=24, wrap=tk.NONE, font=font) + text.grid(row=1, column=1, sticky=tk.NSEW) + + editwin = Dummy_editwin(text) + editwin.vbar = tk.Scrollbar(text_frame) + + linenumbers = LineNumbers(editwin) + linenumbers.show_sidebar() + + text.insert('1.0', '\n'.join('a'*i for i in range(1, 101))) + + +if __name__ == '__main__': + from unittest import main + main('idlelib.idle_test.test_sidebar', verbosity=2, exit=False) + + from idlelib.idle_test.htest import run + run(_linenumbers_drag_scrolling) diff --git a/lib-python/3/idlelib/squeezer.py b/lib-python/3/idlelib/squeezer.py index 032401f2ab..be1538a25f 100644 --- a/lib-python/3/idlelib/squeezer.py +++ b/lib-python/3/idlelib/squeezer.py @@ -15,10 +15,8 @@ output written to the standard error stream ("stderr"), such as exception messages and their tracebacks. """ import re -import weakref import tkinter as tk -from tkinter.font import Font import tkinter.messagebox as tkMessageBox from idlelib.config import idleConf @@ -203,8 +201,6 @@ class Squeezer: This avoids IDLE's shell slowing down considerably, and even becoming completely unresponsive, when very long outputs are written. """ - _instance_weakref = None - @classmethod def reload(cls): """Load class variables from config.""" @@ -213,14 +209,6 @@ class Squeezer: type="int", default=50, ) - # Loading the font info requires a Tk root. IDLE doesn't rely - # on Tkinter's "default root", so the instance will reload - # font info using its editor windows's Tk root. - if cls._instance_weakref is not None: - instance = cls._instance_weakref() - if instance is not None: - instance.load_font() - def __init__(self, editwin): """Initialize settings for Squeezer. @@ -241,9 +229,6 @@ class Squeezer: # however, needs to make such changes. self.base_text = editwin.per.bottom - Squeezer._instance_weakref = weakref.ref(self) - self.load_font() - # Twice the text widget's border width and internal padding; # pre-calculated here for the get_line_width() method. self.window_width_delta = 2 * ( @@ -298,24 +283,7 @@ class Squeezer: Tabs are considered tabwidth characters long. """ - linewidth = self.get_line_width() - return count_lines_with_wrapping(s, linewidth) - - def get_line_width(self): - # The maximum line length in pixels: The width of the text - # widget, minus twice the border width and internal padding. - linewidth_pixels = \ - self.base_text.winfo_width() - self.window_width_delta - - # Divide the width of the Text widget by the font width, - # which is taken to be the width of '0' (zero). - # http://www.tcl.tk/man/tcl8.6/TkCmd/text.htm#M21 - return linewidth_pixels // self.zero_char_width - - def load_font(self): - text = self.base_text - self.zero_char_width = \ - Font(text, font=text.cget('font')).measure('0') + return count_lines_with_wrapping(s, self.editwin.width) def squeeze_current_text_event(self, event): """squeeze-current-text event handler diff --git a/lib-python/3/idlelib/textview.py b/lib-python/3/idlelib/textview.py index 4867a80db1..a66c1a4309 100644 --- a/lib-python/3/idlelib/textview.py +++ b/lib-python/3/idlelib/textview.py @@ -2,14 +2,14 @@ """ from tkinter import Toplevel, Text, TclError,\ - HORIZONTAL, VERTICAL, N, S, E, W + HORIZONTAL, VERTICAL, NS, EW, NSEW, NONE, WORD, SUNKEN from tkinter.ttk import Frame, Scrollbar, Button from tkinter.messagebox import showerror from idlelib.colorizer import color_config -class AutoHiddenScrollbar(Scrollbar): +class AutoHideScrollbar(Scrollbar): """A scrollbar that is automatically hidden when not needed. Only the grid geometry manager is supported. @@ -28,52 +28,70 @@ class AutoHiddenScrollbar(Scrollbar): raise TclError(f'{self.__class__.__name__} does not support "place"') -class TextFrame(Frame): - "Display text with scrollbar." +class ScrollableTextFrame(Frame): + """Display text with scrollbar(s).""" - def __init__(self, parent, rawtext, wrap='word'): + def __init__(self, master, wrap=NONE, **kwargs): """Create a frame for Textview. - parent - parent widget for this frame - rawtext - text to display + master - master widget for this frame + wrap - type of text wrapping to use ('word', 'char' or 'none') + + All parameters except for 'wrap' are passed to Frame.__init__(). + + The Text widget is accessible via the 'text' attribute. + + Note: Changing the wrapping mode of the text widget after + instantiation is not supported. """ - super().__init__(parent) - self['relief'] = 'sunken' - self['height'] = 700 + super().__init__(master, **kwargs) - self.text = text = Text(self, wrap=wrap, highlightthickness=0) - color_config(text) - text.grid(row=0, column=0, sticky=N+S+E+W) + text = self.text = Text(self, wrap=wrap) + text.grid(row=0, column=0, sticky=NSEW) self.grid_rowconfigure(0, weight=1) self.grid_columnconfigure(0, weight=1) - text.insert(0.0, rawtext) - text['state'] = 'disabled' - text.focus_set() # vertical scrollbar - self.yscroll = yscroll = AutoHiddenScrollbar(self, orient=VERTICAL, - takefocus=False, - command=text.yview) - text['yscrollcommand'] = yscroll.set - yscroll.grid(row=0, column=1, sticky=N+S) - - if wrap == 'none': - # horizontal scrollbar - self.xscroll = xscroll = AutoHiddenScrollbar(self, orient=HORIZONTAL, - takefocus=False, - command=text.xview) - text['xscrollcommand'] = xscroll.set - xscroll.grid(row=1, column=0, sticky=E+W) + self.yscroll = AutoHideScrollbar(self, orient=VERTICAL, + takefocus=False, + command=text.yview) + self.yscroll.grid(row=0, column=1, sticky=NS) + text['yscrollcommand'] = self.yscroll.set + + # horizontal scrollbar - only when wrap is set to NONE + if wrap == NONE: + self.xscroll = AutoHideScrollbar(self, orient=HORIZONTAL, + takefocus=False, + command=text.xview) + self.xscroll.grid(row=1, column=0, sticky=EW) + text['xscrollcommand'] = self.xscroll.set + else: + self.xscroll = None class ViewFrame(Frame): "Display TextFrame and Close button." - def __init__(self, parent, text, wrap='word'): + def __init__(self, parent, contents, wrap='word'): + """Create a frame for viewing text with a "Close" button. + + parent - parent widget for this frame + contents - text to display + wrap - type of text wrapping to use ('word', 'char' or 'none') + + The Text widget is accessible via the 'text' attribute. + """ super().__init__(parent) self.parent = parent self.bind('<Return>', self.ok) self.bind('<Escape>', self.ok) - self.textframe = TextFrame(self, text, wrap=wrap) + self.textframe = ScrollableTextFrame(self, relief=SUNKEN, height=700) + + text = self.text = self.textframe.text + text.insert('1.0', contents) + text.configure(wrap=wrap, highlightthickness=0, state='disabled') + color_config(text) + text.focus_set() + self.button_ok = button_ok = Button( self, text='Close', command=self.ok, takefocus=False) self.textframe.pack(side='top', expand=True, fill='both') @@ -87,7 +105,7 @@ class ViewFrame(Frame): class ViewWindow(Toplevel): "A simple text viewer dialog for IDLE." - def __init__(self, parent, title, text, modal=True, wrap='word', + def __init__(self, parent, title, contents, modal=True, wrap=WORD, *, _htest=False, _utest=False): """Show the given text in a scrollable window with a 'close' button. @@ -96,7 +114,7 @@ class ViewWindow(Toplevel): parent - parent of this dialog title - string which is title of popup dialog - text - text to display in dialog + contents - text to display in dialog wrap - type of text wrapping to use ('word', 'char' or 'none') _htest - bool; change box location when running htest. _utest - bool; don't wait_window when running unittest. @@ -109,7 +127,7 @@ class ViewWindow(Toplevel): self.geometry(f'=750x500+{x}+{y}') self.title(title) - self.viewframe = ViewFrame(self, text, wrap=wrap) + self.viewframe = ViewFrame(self, contents, wrap=wrap) self.protocol("WM_DELETE_WINDOW", self.ok) self.button_ok = button_ok = Button(self, text='Close', command=self.ok, takefocus=False) @@ -129,18 +147,18 @@ class ViewWindow(Toplevel): self.destroy() -def view_text(parent, title, text, modal=True, wrap='word', _utest=False): +def view_text(parent, title, contents, modal=True, wrap='word', _utest=False): """Create text viewer for given text. parent - parent of this dialog title - string which is the title of popup dialog - text - text to display in this dialog + contents - text to display in this dialog wrap - type of text wrapping to use ('word', 'char' or 'none') modal - controls if users can interact with other windows while this dialog is displayed _utest - bool; controls wait_window on unittest """ - return ViewWindow(parent, title, text, modal, wrap=wrap, _utest=_utest) + return ViewWindow(parent, title, contents, modal, wrap=wrap, _utest=_utest) def view_file(parent, title, filename, encoding, modal=True, wrap='word', diff --git a/lib-python/3/idlelib/tooltip.py b/lib-python/3/idlelib/tooltip.py index f54ea36f05..69658264db 100644 --- a/lib-python/3/idlelib/tooltip.py +++ b/lib-python/3/idlelib/tooltip.py @@ -75,7 +75,7 @@ class TooltipBase(object): if tw: try: tw.destroy() - except TclError: + except TclError: # pragma: no cover pass @@ -103,8 +103,8 @@ class OnHoverTooltipBase(TooltipBase): def __del__(self): try: self.anchor_widget.unbind("<Enter>", self._id1) - self.anchor_widget.unbind("<Leave>", self._id2) - self.anchor_widget.unbind("<Button>", self._id3) + self.anchor_widget.unbind("<Leave>", self._id2) # pragma: no cover + self.anchor_widget.unbind("<Button>", self._id3) # pragma: no cover except TclError: pass super(OnHoverTooltipBase, self).__del__() @@ -137,7 +137,7 @@ class OnHoverTooltipBase(TooltipBase): """hide the tooltip""" try: self.unschedule() - except TclError: + except TclError: # pragma: no cover pass super(OnHoverTooltipBase, self).hidetip() diff --git a/lib-python/3/idlelib/tree.py b/lib-python/3/idlelib/tree.py index 21426cbb33..6229be4e5a 100644 --- a/lib-python/3/idlelib/tree.py +++ b/lib-python/3/idlelib/tree.py @@ -56,6 +56,30 @@ def listicons(icondir=ICONDIR): column = 0 root.images = images +def wheel_event(event, widget=None): + """Handle scrollwheel event. + + For wheel up, event.delta = 120*n on Windows, -1*n on darwin, + where n can be > 1 if one scrolls fast. Flicking the wheel + generates up to maybe 20 events with n up to 10 or more 1. + Macs use wheel down (delta = 1*n) to scroll up, so positive + delta means to scroll up on both systems. + + X-11 sends Control-Button-4,5 events instead. + + The widget parameter is needed so browser label bindings can pass + the underlying canvas. + + This function depends on widget.yview to not be overridden by + a subclass. + """ + up = {EventType.MouseWheel: event.delta > 0, + EventType.ButtonPress: event.num == 4} + lines = -5 if up[event.type] else 5 + widget = event.widget if widget is None else widget + widget.yview(SCROLL, lines, 'units') + return 'break' + class TreeNode: @@ -260,6 +284,9 @@ class TreeNode: anchor="nw", window=self.label) self.label.bind("<1>", self.select_or_edit) self.label.bind("<Double-1>", self.flip) + self.label.bind("<MouseWheel>", lambda e: wheel_event(e, self.canvas)) + self.label.bind("<Button-4>", lambda e: wheel_event(e, self.canvas)) + self.label.bind("<Button-5>", lambda e: wheel_event(e, self.canvas)) self.text_id = id def select_or_edit(self, event=None): @@ -410,6 +437,7 @@ class FileTreeItem(TreeItem): # A canvas widget with scroll bars and some useful bindings class ScrolledCanvas: + def __init__(self, master, **opts): if 'yscrollincrement' not in opts: opts['yscrollincrement'] = 17 @@ -431,6 +459,9 @@ class ScrolledCanvas: self.canvas.bind("<Key-Next>", self.page_down) self.canvas.bind("<Key-Up>", self.unit_up) self.canvas.bind("<Key-Down>", self.unit_down) + self.canvas.bind("<MouseWheel>", wheel_event) + self.canvas.bind("<Button-4>", wheel_event) + self.canvas.bind("<Button-5>", wheel_event) #if isinstance(master, Toplevel) or isinstance(master, Tk): self.canvas.bind("<Alt-Key-2>", self.zoom_height) self.canvas.focus_set() diff --git a/lib-python/3/importlib/_bootstrap_external.py b/lib-python/3/importlib/_bootstrap_external.py index 53b24ff1b0..66a16a6839 100644 --- a/lib-python/3/importlib/_bootstrap_external.py +++ b/lib-python/3/importlib/_bootstrap_external.py @@ -246,7 +246,7 @@ _code_type = type(_write_atomic.__code__) # Python 3.7a2 3391 (update GET_AITER #31709) # Python 3.7a4 3392 (PEP 552: Deterministic pycs #31650) # Python 3.7b1 3393 (remove STORE_ANNOTATION opcode #32550) -# Python 3.7b5 3394 (restored docstring as the firts stmt in the body; +# Python 3.7b5 3394 (restored docstring as the first stmt in the body; # this might affected the first line number #32911) # # MAGIC must change whenever the bytecode emitted by the compiler may no diff --git a/lib-python/3/inspect.py b/lib-python/3/inspect.py index 4d4f33dcc5..9848ca4c50 100644 --- a/lib-python/3/inspect.py +++ b/lib-python/3/inspect.py @@ -729,7 +729,7 @@ def getmodule(object, _filename=None): return sys.modules.get(modulesbyfile[file]) # Update the filename to module name cache and check yet again # Copy sys.modules in order to cope with changes while iterating - for modname, module in list(sys.modules.items()): + for modname, module in sys.modules.copy().items(): if ismodule(module) and hasattr(module, '__file__'): f = module.__file__ if f == _filesbymodname.get(modname, None): @@ -2358,7 +2358,7 @@ def _signature_from_callable(obj, *, if (obj.__init__ is object.__init__ and obj.__new__ is object.__new__): # Return a signature of 'object' builtin. - return signature(object) + return sigcls.from_callable(object) else: raise ValueError( 'no signature found for builtin type {!r}'.format(obj)) @@ -3108,7 +3108,7 @@ def _main(): type(exc).__name__, exc) print(msg, file=sys.stderr) - exit(2) + sys.exit(2) if has_attrs: parts = attrs.split(".") @@ -3118,7 +3118,7 @@ def _main(): if module.__name__ in sys.builtin_module_names: print("Can't get info for builtin modules.", file=sys.stderr) - exit(1) + sys.exit(1) if args.details: print('Target: {}'.format(target)) diff --git a/lib-python/3/ipaddress.py b/lib-python/3/ipaddress.py index 4eec1f337c..54882934c3 100644 --- a/lib-python/3/ipaddress.py +++ b/lib-python/3/ipaddress.py @@ -532,6 +532,30 @@ class _IPAddressBase: except ValueError: cls._report_invalid_netmask(ip_str) + @classmethod + def _split_addr_prefix(cls, address): + """Helper function to parse address of Network/Interface. + + Arg: + address: Argument of Network/Interface. + + Returns: + (addr, prefix) tuple. + """ + # a packed address or integer + if isinstance(address, (bytes, int)): + return address, cls._max_prefixlen + + if not isinstance(address, tuple): + # Assume input argument to be string or any object representation + # which converts into a formatted IP prefix string. + address = _split_optional_netmask(address) + + # Constructing from a tuple (addr, [mask]) + if len(address) > 1: + return address + return address[0], cls._max_prefixlen + def __reduce__(self): return self.__class__, (str(self),) @@ -1381,32 +1405,13 @@ class IPv4Address(_BaseV4, _BaseAddress): class IPv4Interface(IPv4Address): def __init__(self, address): - if isinstance(address, (bytes, int)): - IPv4Address.__init__(self, address) - self.network = IPv4Network(self._ip) - self._prefixlen = self._max_prefixlen - return - - if isinstance(address, tuple): - IPv4Address.__init__(self, address[0]) - if len(address) > 1: - self._prefixlen = int(address[1]) - else: - self._prefixlen = self._max_prefixlen - - self.network = IPv4Network(address, strict=False) - self.netmask = self.network.netmask - self.hostmask = self.network.hostmask - return - - addr = _split_optional_netmask(address) - IPv4Address.__init__(self, addr[0]) - - self.network = IPv4Network(address, strict=False) - self._prefixlen = self.network._prefixlen + addr, mask = self._split_addr_prefix(address) + IPv4Address.__init__(self, addr) + self.network = IPv4Network((addr, mask), strict=False) self.netmask = self.network.netmask self.hostmask = self.network.hostmask + self._prefixlen = self.network._prefixlen def __str__(self): return '%s/%d' % (self._string_from_ip_int(self._ip), @@ -1437,7 +1442,7 @@ class IPv4Interface(IPv4Address): return False def __hash__(self): - return self._ip ^ self._prefixlen ^ int(self.network.network_address) + return hash((self._ip, self._prefixlen, int(self.network.network_address))) __reduce__ = _IPAddressBase.__reduce__ @@ -1511,24 +1516,9 @@ class IPv4Network(_BaseV4, _BaseNetwork): an IPv4 address. ValueError: If strict is True and a network address is not supplied. - """ _BaseNetwork.__init__(self, address) - - # Constructing from a packed address or integer - if isinstance(address, (int, bytes)): - addr = address - mask = self._max_prefixlen - # Constructing from a tuple (addr, [mask]) - elif isinstance(address, tuple): - addr = address[0] - mask = address[1] if len(address) > 1 else self._max_prefixlen - # Assume input argument to be string or any object representation - # which converts into a formatted IP prefix string. - else: - args = _split_optional_netmask(address) - addr = self._ip_int_from_string(args[0]) - mask = args[1] if len(args) == 2 else self._max_prefixlen + addr, mask = self._split_addr_prefix(address) self.network_address = IPv4Address(addr) self.netmask, self._prefixlen = self._make_netmask(mask) @@ -2061,28 +2051,13 @@ class IPv6Address(_BaseV6, _BaseAddress): class IPv6Interface(IPv6Address): def __init__(self, address): - if isinstance(address, (bytes, int)): - IPv6Address.__init__(self, address) - self.network = IPv6Network(self._ip) - self._prefixlen = self._max_prefixlen - return - if isinstance(address, tuple): - IPv6Address.__init__(self, address[0]) - if len(address) > 1: - self._prefixlen = int(address[1]) - else: - self._prefixlen = self._max_prefixlen - self.network = IPv6Network(address, strict=False) - self.netmask = self.network.netmask - self.hostmask = self.network.hostmask - return + addr, mask = self._split_addr_prefix(address) - addr = _split_optional_netmask(address) - IPv6Address.__init__(self, addr[0]) - self.network = IPv6Network(address, strict=False) + IPv6Address.__init__(self, addr) + self.network = IPv6Network((addr, mask), strict=False) self.netmask = self.network.netmask - self._prefixlen = self.network._prefixlen self.hostmask = self.network.hostmask + self._prefixlen = self.network._prefixlen def __str__(self): return '%s/%d' % (self._string_from_ip_int(self._ip), @@ -2113,7 +2088,7 @@ class IPv6Interface(IPv6Address): return False def __hash__(self): - return self._ip ^ self._prefixlen ^ int(self.network.network_address) + return hash((self._ip, self._prefixlen, int(self.network.network_address))) __reduce__ = _IPAddressBase.__reduce__ @@ -2191,24 +2166,9 @@ class IPv6Network(_BaseV6, _BaseNetwork): an IPv6 address. ValueError: If strict was True and a network address was not supplied. - """ _BaseNetwork.__init__(self, address) - - # Constructing from a packed address or integer - if isinstance(address, (int, bytes)): - addr = address - mask = self._max_prefixlen - # Constructing from a tuple (addr, [mask]) - elif isinstance(address, tuple): - addr = address[0] - mask = address[1] if len(address) > 1 else self._max_prefixlen - # Assume input argument to be string or any object representation - # which converts into a formatted IP prefix string. - else: - args = _split_optional_netmask(address) - addr = self._ip_int_from_string(args[0]) - mask = args[1] if len(args) == 2 else self._max_prefixlen + addr, mask = self._split_addr_prefix(address) self.network_address = IPv6Address(addr) self.netmask, self._prefixlen = self._make_netmask(mask) diff --git a/lib-python/3/json/tool.py b/lib-python/3/json/tool.py index 5932f4ecde..c136fb7d3b 100644 --- a/lib-python/3/json/tool.py +++ b/lib-python/3/json/tool.py @@ -20,9 +20,9 @@ def main(): description = ('A simple command line interface for json module ' 'to validate and pretty-print JSON objects.') parser = argparse.ArgumentParser(prog=prog, description=description) - parser.add_argument('infile', nargs='?', type=argparse.FileType(), + parser.add_argument('infile', nargs='?', type=argparse.FileType(encoding="utf-8"), help='a JSON file to be validated or pretty-printed') - parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), + parser.add_argument('outfile', nargs='?', type=argparse.FileType('w', encoding="utf-8"), help='write the output of infile to outfile') parser.add_argument('--sort-keys', action='store_true', default=False, help='sort the output of dictionaries alphabetically by key') @@ -42,4 +42,7 @@ def main(): if __name__ == '__main__': - main() + try: + main() + except BrokenPipeError as exc: + sys.exit(exc.errno) diff --git a/lib-python/3/lib2to3/Grammar.txt b/lib-python/3/lib2to3/Grammar.txt index a7ddad3cf3..8ce7fd8a89 100644 --- a/lib-python/3/lib2to3/Grammar.txt +++ b/lib-python/3/lib2to3/Grammar.txt @@ -67,8 +67,8 @@ assert_stmt: 'assert' test [',' test] compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt async_stmt: ASYNC (funcdef | with_stmt | for_stmt) -if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] -while_stmt: 'while' test ':' suite ['else' ':' suite] +if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite] +while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite] for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] try_stmt: ('try' ':' suite ((except_clause ':' suite)+ @@ -91,6 +91,7 @@ testlist_safe: old_test [(',' old_test)+ [',']] old_test: or_test | old_lambdef old_lambdef: 'lambda' [varargslist] ':' old_test +namedexpr_test: test [':=' test] test: or_test ['if' or_test 'else' test] | lambdef or_test: and_test ('or' and_test)* and_test: not_test ('and' not_test)* @@ -111,8 +112,8 @@ atom: ('(' [yield_expr|testlist_gexp] ')' | '{' [dictsetmaker] '}' | '`' testlist1 '`' | NAME | NUMBER | STRING+ | '.' '.' '.') -listmaker: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) -testlist_gexp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +listmaker: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) +testlist_gexp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] ) lambdef: 'lambda' [varargslist] ':' test trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME subscriptlist: subscript (',' subscript)* [','] @@ -137,9 +138,10 @@ arglist: argument (',' argument)* [','] # multiple (test comp_for) arguments are blocked; keyword unpackings # that precede iterable unpackings are blocked; etc. argument: ( test [comp_for] | + test ':=' test | test '=' test | - '**' expr | - star_expr ) + '**' test | + '*' test ) comp_iter: comp_for | comp_if comp_for: [ASYNC] 'for' exprlist 'in' testlist_safe [comp_iter] diff --git a/lib-python/3/lib2to3/fixes/fix_apply.py b/lib-python/3/lib2to3/fixes/fix_apply.py index 826ec8c9b6..6408582c42 100644 --- a/lib-python/3/lib2to3/fixes/fix_apply.py +++ b/lib-python/3/lib2to3/fixes/fix_apply.py @@ -37,10 +37,8 @@ class FixApply(fixer_base.BaseFix): # I feel like we should be able to express this logic in the # PATTERN above but I don't know how to do it so... if args: - if args.type == self.syms.star_expr: - return # Make no change. if (args.type == self.syms.argument and - args.children[0].value == '**'): + args.children[0].value in {'**', '*'}): return # Make no change. if kwds and (kwds.type == self.syms.argument and kwds.children[0].value == '**'): diff --git a/lib-python/3/lib2to3/fixes/fix_filter.py b/lib-python/3/lib2to3/fixes/fix_filter.py index a7a5a154f6..38e9078f11 100644 --- a/lib-python/3/lib2to3/fixes/fix_filter.py +++ b/lib-python/3/lib2to3/fixes/fix_filter.py @@ -17,7 +17,7 @@ Python 2.6 figure it out. from .. import fixer_base from ..pytree import Node from ..pygram import python_symbols as syms -from ..fixer_util import Name, ArgList, ListComp, in_special_context +from ..fixer_util import Name, ArgList, ListComp, in_special_context, parenthesize class FixFilter(fixer_base.ConditionalFix): @@ -65,10 +65,14 @@ class FixFilter(fixer_base.ConditionalFix): trailers.append(t.clone()) if "filter_lambda" in results: + xp = results.get("xp").clone() + if xp.type == syms.test: + xp.prefix = "" + xp = parenthesize(xp) + new = ListComp(results.get("fp").clone(), results.get("fp").clone(), - results.get("it").clone(), - results.get("xp").clone()) + results.get("it").clone(), xp) new = Node(syms.power, [new] + trailers, prefix="") elif "none" in results: diff --git a/lib-python/3/lib2to3/fixes/fix_intern.py b/lib-python/3/lib2to3/fixes/fix_intern.py index a852330908..d752843092 100644 --- a/lib-python/3/lib2to3/fixes/fix_intern.py +++ b/lib-python/3/lib2to3/fixes/fix_intern.py @@ -30,10 +30,8 @@ class FixIntern(fixer_base.BaseFix): # PATTERN above but I don't know how to do it so... obj = results['obj'] if obj: - if obj.type == self.syms.star_expr: - return # Make no change. if (obj.type == self.syms.argument and - obj.children[0].value == '**'): + obj.children[0].value in {'**', '*'}): return # Make no change. names = ('sys', 'intern') new = ImportAndCall(node, results, names) diff --git a/lib-python/3/lib2to3/fixes/fix_reload.py b/lib-python/3/lib2to3/fixes/fix_reload.py index 6c7fbbd3be..b30841131c 100644 --- a/lib-python/3/lib2to3/fixes/fix_reload.py +++ b/lib-python/3/lib2to3/fixes/fix_reload.py @@ -27,10 +27,8 @@ class FixReload(fixer_base.BaseFix): # PATTERN above but I don't know how to do it so... obj = results['obj'] if obj: - if obj.type == self.syms.star_expr: - return # Make no change. if (obj.type == self.syms.argument and - obj.children[0].value == '**'): + obj.children[0].value in {'**', '*'}): return # Make no change. names = ('importlib', 'reload') new = ImportAndCall(node, results, names) diff --git a/lib-python/3/lib2to3/pgen2/grammar.py b/lib-python/3/lib2to3/pgen2/grammar.py index 088c58bfa9..997fdf530f 100644 --- a/lib-python/3/lib2to3/pgen2/grammar.py +++ b/lib-python/3/lib2to3/pgen2/grammar.py @@ -202,6 +202,7 @@ opmap_raw = """ // DOUBLESLASH //= DOUBLESLASHEQUAL -> RARROW +:= COLONEQUAL """ opmap = {} diff --git a/lib-python/3/lib2to3/pgen2/token.py b/lib-python/3/lib2to3/pgen2/token.py index 1a679554d2..5f6612f5b3 100755 --- a/lib-python/3/lib2to3/pgen2/token.py +++ b/lib-python/3/lib2to3/pgen2/token.py @@ -65,7 +65,8 @@ RARROW = 55 AWAIT = 56 ASYNC = 57 ERRORTOKEN = 58 -N_TOKENS = 59 +COLONEQUAL = 59 +N_TOKENS = 60 NT_OFFSET = 256 #--end constants-- diff --git a/lib-python/3/lib2to3/pgen2/tokenize.py b/lib-python/3/lib2to3/pgen2/tokenize.py index 279d322971..94dd792805 100644 --- a/lib-python/3/lib2to3/pgen2/tokenize.py +++ b/lib-python/3/lib2to3/pgen2/tokenize.py @@ -93,7 +93,7 @@ Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", r"~") Bracket = '[][(){}]' -Special = group(r'\r?\n', r'[:;.,`@]') +Special = group(r'\r?\n', r':=', r'[:;.,`@]') Funny = group(Operator, Bracket, Special) PlainToken = group(Number, Funny, String, Name) diff --git a/lib-python/3/lib2to3/refactor.py b/lib-python/3/lib2to3/refactor.py index 7841b99a5c..55fd60fa27 100644 --- a/lib-python/3/lib2to3/refactor.py +++ b/lib-python/3/lib2to3/refactor.py @@ -14,6 +14,7 @@ __author__ = "Guido van Rossum <guido@python.org>" # Python imports import io import os +import pkgutil import sys import logging import operator @@ -30,13 +31,12 @@ from . import btm_matcher as bm def get_all_fix_names(fixer_pkg, remove_prefix=True): """Return a sorted list of all available fix names in the given package.""" pkg = __import__(fixer_pkg, [], [], ["*"]) - fixer_dir = os.path.dirname(pkg.__file__) fix_names = [] - for name in sorted(os.listdir(fixer_dir)): - if name.startswith("fix_") and name.endswith(".py"): + for finder, name, ispkg in pkgutil.iter_modules(pkg.__path__): + if name.startswith("fix_"): if remove_prefix: name = name[4:] - fix_names.append(name[:-3]) + fix_names.append(name) return fix_names diff --git a/lib-python/3/lib2to3/tests/test_fixers.py b/lib-python/3/lib2to3/tests/test_fixers.py index 3da5dd845c..a285241981 100644 --- a/lib-python/3/lib2to3/tests/test_fixers.py +++ b/lib-python/3/lib2to3/tests/test_fixers.py @@ -2954,6 +2954,11 @@ class Test_filter(FixerTestCase): a = """x = [x for x in range(10) if x%2 == 0]""" self.check(b, a) + # bpo-38871 + b = """filter(lambda x: True if x > 2 else False, [1, 2, 3])""" + a = """[x for x in [1, 2, 3] if (True if x > 2 else False)]""" + self.check(b, a) + def test_filter_trailers(self): b = """x = filter(None, 'abc')[0]""" a = """x = [_f for _f in 'abc' if _f][0]""" diff --git a/lib-python/3/lib2to3/tests/test_parser.py b/lib-python/3/lib2to3/tests/test_parser.py index 829e5a7292..753c846b7b 100644 --- a/lib-python/3/lib2to3/tests/test_parser.py +++ b/lib-python/3/lib2to3/tests/test_parser.py @@ -253,6 +253,13 @@ class TestUnpackingGeneralizations(GrammarTest): def test_double_star_dict_literal_after_keywords(self): self.validate("""func(spam='fried', **{'eggs':'scrambled'})""") + def test_double_star_expression(self): + self.validate("""func(**{'a':2} or {})""") + self.validate("""func(**() or {})""") + + def test_star_expression(self): + self.validate("""func(*[] or [2])""") + def test_list_display(self): self.validate("""[*{2}, 3, *[4]]""") @@ -622,6 +629,21 @@ class TestLiterals(GrammarTest): self.validate(s) +class TestNamedAssignments(GrammarTest): + + def test_named_assignment_if(self): + driver.parse_string("if f := x(): pass\n") + + def test_named_assignment_while(self): + driver.parse_string("while f := x(): pass\n") + + def test_named_assignment_generator(self): + driver.parse_string("any((lastNum := num) == 1 for num in [1, 2, 3])\n") + + def test_named_assignment_listcomp(self): + driver.parse_string("[(lastNum := num) == 1 for num in [1, 2, 3]]\n") + + def diff_texts(a, b, filename): a = a.splitlines() b = b.splitlines() diff --git a/lib-python/3/linecache.py b/lib-python/3/linecache.py index 3afcce1f0a..c87e1807bf 100644 --- a/lib-python/3/linecache.py +++ b/lib-python/3/linecache.py @@ -73,10 +73,10 @@ def checkcache(filename=None): try: stat = os.stat(fullname) except OSError: - del cache[filename] + cache.pop(filename, None) continue if size != stat.st_size or mtime != stat.st_mtime: - del cache[filename] + cache.pop(filename, None) def updatecache(filename, module_globals=None): @@ -86,7 +86,7 @@ def updatecache(filename, module_globals=None): if filename in cache: if len(cache[filename]) != 1: - del cache[filename] + cache.pop(filename, None) if not filename or (filename.startswith('<') and filename.endswith('>')): return [] diff --git a/lib-python/3/locale.py b/lib-python/3/locale.py index f3d3973d03..dd8a08524a 100644 --- a/lib-python/3/locale.py +++ b/lib-python/3/locale.py @@ -492,6 +492,10 @@ def _parse_localename(localename): return tuple(code.split('.')[:2]) elif code == 'C': return None, None + elif code == 'UTF-8': + # On macOS "LC_CTYPE=UTF-8" is a valid locale setting + # for getting UTF-8 handling for text. + return None, 'UTF-8' raise ValueError('unknown locale: %s' % localename) def _build_localename(localetuple): diff --git a/lib-python/3/logging/__init__.py b/lib-python/3/logging/__init__.py index 6e01714886..b596f80f6f 100644 --- a/lib-python/3/logging/__init__.py +++ b/lib-python/3/logging/__init__.py @@ -1619,12 +1619,15 @@ class Logger(Filterer): return self._cache[level] except KeyError: _acquireLock() - if self.manager.disable >= level: - is_enabled = self._cache[level] = False - else: - is_enabled = self._cache[level] = level >= self.getEffectiveLevel() - _releaseLock() - + try: + if self.manager.disable >= level: + is_enabled = self._cache[level] = False + else: + is_enabled = self._cache[level] = ( + level >= self.getEffectiveLevel() + ) + finally: + _releaseLock() return is_enabled def getChild(self, suffix): diff --git a/lib-python/3/logging/config.py b/lib-python/3/logging/config.py index fa1a398aee..626aeea7c2 100644 --- a/lib-python/3/logging/config.py +++ b/lib-python/3/logging/config.py @@ -1,4 +1,4 @@ -# Copyright 2001-2016 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2019 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -19,7 +19,7 @@ Configuration functions for the logging package for Python. The core package is based on PEP 282 and comments thereto in comp.lang.python, and influenced by Apache's log4j system. -Copyright (C) 2001-2016 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2019 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ @@ -173,9 +173,10 @@ def _handle_existing_loggers(existing, child_loggers, disable_existing): for log in existing: logger = root.manager.loggerDict[log] if log in child_loggers: - logger.level = logging.NOTSET - logger.handlers = [] - logger.propagate = True + if not isinstance(logger, logging.PlaceHolder): + logger.setLevel(logging.NOTSET) + logger.handlers = [] + logger.propagate = True else: logger.disabled = disable_existing @@ -446,7 +447,7 @@ class BaseConfigurator(object): value = ConvertingList(value) value.configurator = self elif not isinstance(value, ConvertingTuple) and\ - isinstance(value, tuple): + isinstance(value, tuple) and not hasattr(value, '_fields'): value = ConvertingTuple(value) value.configurator = self elif isinstance(value, str): # str for py3k diff --git a/lib-python/3/mimetypes.py b/lib-python/3/mimetypes.py index 1206d8e9b6..bcf522835f 100644 --- a/lib-python/3/mimetypes.py +++ b/lib-python/3/mimetypes.py @@ -66,13 +66,13 @@ class MimeTypes: def __init__(self, filenames=(), strict=True): if not inited: init() - self.encodings_map = encodings_map.copy() - self.suffix_map = suffix_map.copy() + self.encodings_map = _encodings_map_default.copy() + self.suffix_map = _suffix_map_default.copy() self.types_map = ({}, {}) # dict for (non-strict, strict) self.types_map_inv = ({}, {}) - for (ext, type) in types_map.items(): + for (ext, type) in _types_map_default.items(): self.add_type(type, ext, True) - for (ext, type) in common_types.items(): + for (ext, type) in _common_types_default.items(): self.add_type(type, ext, False) for name in filenames: self.read(name, strict) @@ -113,6 +113,7 @@ class MimeTypes: Optional `strict' argument when False adds a bunch of commonly found, but non-standard types. """ + url = os.fspath(url) scheme, url = urllib.parse.splittype(url) if scheme == 'data': # syntax of data URLs: @@ -345,11 +346,19 @@ def init(files=None): global suffix_map, types_map, encodings_map, common_types global inited, _db inited = True # so that MimeTypes.__init__() doesn't call us again - db = MimeTypes() - if files is None: + + if files is None or _db is None: + db = MimeTypes() if _winreg: db.read_windows_registry() - files = knownfiles + + if files is None: + files = knownfiles + else: + files = knownfiles + list(files) + else: + db = _db + for file in files: if os.path.isfile(file): db.read(file) @@ -373,12 +382,12 @@ def read_mime_types(file): def _default_mime_types(): - global suffix_map - global encodings_map - global types_map - global common_types + global suffix_map, _suffix_map_default + global encodings_map, _encodings_map_default + global types_map, _types_map_default + global common_types, _common_types_default - suffix_map = { + suffix_map = _suffix_map_default = { '.svgz': '.svg.gz', '.tgz': '.tar.gz', '.taz': '.tar.gz', @@ -387,7 +396,7 @@ def _default_mime_types(): '.txz': '.tar.xz', } - encodings_map = { + encodings_map = _encodings_map_default = { '.gz': 'gzip', '.Z': 'compress', '.bz2': 'bzip2', @@ -398,151 +407,155 @@ def _default_mime_types(): # at http://www.iana.org/assignments/media-types # or extensions, i.e. using the x- prefix - # If you add to these, please keep them sorted! - types_map = { + # If you add to these, please keep them sorted by mime type. + # Make sure the entry with the preferred file extension for a particular mime type + # appears before any others of the same mimetype. + types_map = _types_map_default = { + '.js' : 'application/javascript', + '.mjs' : 'application/javascript', + '.json' : 'application/json', + '.doc' : 'application/msword', + '.dot' : 'application/msword', + '.wiz' : 'application/msword', + '.bin' : 'application/octet-stream', '.a' : 'application/octet-stream', + '.dll' : 'application/octet-stream', + '.exe' : 'application/octet-stream', + '.o' : 'application/octet-stream', + '.obj' : 'application/octet-stream', + '.so' : 'application/octet-stream', + '.oda' : 'application/oda', + '.pdf' : 'application/pdf', + '.p7c' : 'application/pkcs7-mime', + '.ps' : 'application/postscript', '.ai' : 'application/postscript', - '.aif' : 'audio/x-aiff', - '.aifc' : 'audio/x-aiff', - '.aiff' : 'audio/x-aiff', - '.au' : 'audio/basic', - '.avi' : 'video/x-msvideo', - '.bat' : 'text/plain', + '.eps' : 'application/postscript', + '.m3u' : 'application/vnd.apple.mpegurl', + '.m3u8' : 'application/vnd.apple.mpegurl', + '.xls' : 'application/vnd.ms-excel', + '.xlb' : 'application/vnd.ms-excel', + '.ppt' : 'application/vnd.ms-powerpoint', + '.pot' : 'application/vnd.ms-powerpoint', + '.ppa' : 'application/vnd.ms-powerpoint', + '.pps' : 'application/vnd.ms-powerpoint', + '.pwz' : 'application/vnd.ms-powerpoint', + '.wasm' : 'application/wasm', '.bcpio' : 'application/x-bcpio', - '.bin' : 'application/octet-stream', - '.bmp' : 'image/bmp', - '.c' : 'text/plain', - '.cdf' : 'application/x-netcdf', '.cpio' : 'application/x-cpio', '.csh' : 'application/x-csh', - '.css' : 'text/css', - '.csv' : 'text/csv', - '.dll' : 'application/octet-stream', - '.doc' : 'application/msword', - '.dot' : 'application/msword', '.dvi' : 'application/x-dvi', - '.eml' : 'message/rfc822', - '.eps' : 'application/postscript', - '.etx' : 'text/x-setext', - '.exe' : 'application/octet-stream', - '.gif' : 'image/gif', '.gtar' : 'application/x-gtar', - '.h' : 'text/plain', '.hdf' : 'application/x-hdf', - '.htm' : 'text/html', - '.html' : 'text/html', - '.ico' : 'image/vnd.microsoft.icon', - '.ief' : 'image/ief', - '.jpe' : 'image/jpeg', - '.jpeg' : 'image/jpeg', - '.jpg' : 'image/jpeg', - '.js' : 'application/javascript', - '.json' : 'application/json', - '.ksh' : 'text/plain', '.latex' : 'application/x-latex', - '.m1v' : 'video/mpeg', - '.m3u' : 'application/vnd.apple.mpegurl', - '.m3u8' : 'application/vnd.apple.mpegurl', - '.man' : 'application/x-troff-man', - '.me' : 'application/x-troff-me', - '.mht' : 'message/rfc822', - '.mhtml' : 'message/rfc822', '.mif' : 'application/x-mif', - '.mjs' : 'application/javascript', - '.mov' : 'video/quicktime', - '.movie' : 'video/x-sgi-movie', - '.mp2' : 'audio/mpeg', - '.mp3' : 'audio/mpeg', - '.mp4' : 'video/mp4', - '.mpa' : 'video/mpeg', - '.mpe' : 'video/mpeg', - '.mpeg' : 'video/mpeg', - '.mpg' : 'video/mpeg', - '.ms' : 'application/x-troff-ms', + '.cdf' : 'application/x-netcdf', '.nc' : 'application/x-netcdf', - '.nws' : 'message/rfc822', - '.o' : 'application/octet-stream', - '.obj' : 'application/octet-stream', - '.oda' : 'application/oda', '.p12' : 'application/x-pkcs12', - '.p7c' : 'application/pkcs7-mime', - '.pbm' : 'image/x-portable-bitmap', - '.pdf' : 'application/pdf', '.pfx' : 'application/x-pkcs12', - '.pgm' : 'image/x-portable-graymap', - '.pl' : 'text/plain', - '.png' : 'image/png', - '.pnm' : 'image/x-portable-anymap', - '.pot' : 'application/vnd.ms-powerpoint', - '.ppa' : 'application/vnd.ms-powerpoint', - '.ppm' : 'image/x-portable-pixmap', - '.pps' : 'application/vnd.ms-powerpoint', - '.ppt' : 'application/vnd.ms-powerpoint', - '.ps' : 'application/postscript', - '.pwz' : 'application/vnd.ms-powerpoint', - '.py' : 'text/x-python', + '.ram' : 'application/x-pn-realaudio', '.pyc' : 'application/x-python-code', '.pyo' : 'application/x-python-code', - '.qt' : 'video/quicktime', - '.ra' : 'audio/x-pn-realaudio', - '.ram' : 'application/x-pn-realaudio', - '.ras' : 'image/x-cmu-raster', - '.rdf' : 'application/xml', - '.rgb' : 'image/x-rgb', - '.roff' : 'application/x-troff', - '.rtx' : 'text/richtext', - '.sgm' : 'text/x-sgml', - '.sgml' : 'text/x-sgml', '.sh' : 'application/x-sh', '.shar' : 'application/x-shar', - '.snd' : 'audio/basic', - '.so' : 'application/octet-stream', - '.src' : 'application/x-wais-source', + '.swf' : 'application/x-shockwave-flash', '.sv4cpio': 'application/x-sv4cpio', '.sv4crc' : 'application/x-sv4crc', - '.svg' : 'image/svg+xml', - '.swf' : 'application/x-shockwave-flash', - '.t' : 'application/x-troff', '.tar' : 'application/x-tar', '.tcl' : 'application/x-tcl', '.tex' : 'application/x-tex', '.texi' : 'application/x-texinfo', '.texinfo': 'application/x-texinfo', - '.tif' : 'image/tiff', - '.tiff' : 'image/tiff', + '.roff' : 'application/x-troff', + '.t' : 'application/x-troff', '.tr' : 'application/x-troff', - '.tsv' : 'text/tab-separated-values', - '.txt' : 'text/plain', + '.man' : 'application/x-troff-man', + '.me' : 'application/x-troff-me', + '.ms' : 'application/x-troff-ms', '.ustar' : 'application/x-ustar', - '.vcf' : 'text/x-vcard', - '.wav' : 'audio/x-wav', - '.webm' : 'video/webm', - '.wiz' : 'application/msword', + '.src' : 'application/x-wais-source', + '.xsl' : 'application/xml', + '.rdf' : 'application/xml', '.wsdl' : 'application/xml', - '.xbm' : 'image/x-xbitmap', - '.xlb' : 'application/vnd.ms-excel', - '.xls' : 'application/vnd.ms-excel', - '.xml' : 'text/xml', '.xpdl' : 'application/xml', + '.zip' : 'application/zip', + '.au' : 'audio/basic', + '.snd' : 'audio/basic', + '.mp3' : 'audio/mpeg', + '.mp2' : 'audio/mpeg', + '.aif' : 'audio/x-aiff', + '.aifc' : 'audio/x-aiff', + '.aiff' : 'audio/x-aiff', + '.ra' : 'audio/x-pn-realaudio', + '.wav' : 'audio/x-wav', + '.bmp' : 'image/bmp', + '.gif' : 'image/gif', + '.ief' : 'image/ief', + '.jpg' : 'image/jpeg', + '.jpe' : 'image/jpeg', + '.jpeg' : 'image/jpeg', + '.png' : 'image/png', + '.svg' : 'image/svg+xml', + '.tiff' : 'image/tiff', + '.tif' : 'image/tiff', + '.ico' : 'image/vnd.microsoft.icon', + '.ras' : 'image/x-cmu-raster', + '.bmp' : 'image/x-ms-bmp', + '.pnm' : 'image/x-portable-anymap', + '.pbm' : 'image/x-portable-bitmap', + '.pgm' : 'image/x-portable-graymap', + '.ppm' : 'image/x-portable-pixmap', + '.rgb' : 'image/x-rgb', + '.xbm' : 'image/x-xbitmap', '.xpm' : 'image/x-xpixmap', - '.xsl' : 'application/xml', '.xwd' : 'image/x-xwindowdump', - '.zip' : 'application/zip', + '.eml' : 'message/rfc822', + '.mht' : 'message/rfc822', + '.mhtml' : 'message/rfc822', + '.nws' : 'message/rfc822', + '.css' : 'text/css', + '.csv' : 'text/csv', + '.html' : 'text/html', + '.htm' : 'text/html', + '.txt' : 'text/plain', + '.bat' : 'text/plain', + '.c' : 'text/plain', + '.h' : 'text/plain', + '.ksh' : 'text/plain', + '.pl' : 'text/plain', + '.rtx' : 'text/richtext', + '.tsv' : 'text/tab-separated-values', + '.py' : 'text/x-python', + '.etx' : 'text/x-setext', + '.sgm' : 'text/x-sgml', + '.sgml' : 'text/x-sgml', + '.vcf' : 'text/x-vcard', + '.xml' : 'text/xml', + '.mp4' : 'video/mp4', + '.mpeg' : 'video/mpeg', + '.m1v' : 'video/mpeg', + '.mpa' : 'video/mpeg', + '.mpe' : 'video/mpeg', + '.mpg' : 'video/mpeg', + '.mov' : 'video/quicktime', + '.qt' : 'video/quicktime', + '.webm' : 'video/webm', + '.avi' : 'video/x-msvideo', + '.movie' : 'video/x-sgi-movie', } # These are non-standard types, commonly found in the wild. They will # only match if strict=0 flag is given to the API methods. # Please sort these too - common_types = { - '.jpg' : 'image/jpg', - '.mid' : 'audio/midi', + common_types = _common_types_default = { + '.rtf' : 'application/rtf', '.midi': 'audio/midi', + '.mid' : 'audio/midi', + '.jpg' : 'image/jpg', + '.pict': 'image/pict', '.pct' : 'image/pict', '.pic' : 'image/pict', - '.pict': 'image/pict', - '.rtf' : 'application/rtf', - '.xul' : 'text/xul' + '.xul' : 'text/xul', } diff --git a/lib-python/3/multiprocessing/connection.py b/lib-python/3/multiprocessing/connection.py index 1f3ea504ff..6b92cf8a12 100644 --- a/lib-python/3/multiprocessing/connection.py +++ b/lib-python/3/multiprocessing/connection.py @@ -102,7 +102,7 @@ def address_type(address): return 'AF_INET' elif type(address) is str and address.startswith('\\\\'): return 'AF_PIPE' - elif type(address) is str: + elif type(address) is str or util.is_abstract_socket_namespace(address): return 'AF_UNIX' else: raise ValueError('address type of %r unrecognized' % address) @@ -587,7 +587,8 @@ class SocketListener(object): self._family = family self._last_accepted = None - if family == 'AF_UNIX': + if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address): + # Linux abstract socket namespaces do not need to be explicitly unlinked self._unlink = util.Finalize( self, os.unlink, args=(address,), exitpriority=0 ) diff --git a/lib-python/3/multiprocessing/forkserver.py b/lib-python/3/multiprocessing/forkserver.py index 040b46e66a..ee5f76780b 100644 --- a/lib-python/3/multiprocessing/forkserver.py +++ b/lib-python/3/multiprocessing/forkserver.py @@ -39,6 +39,25 @@ class ForkServer(object): self._lock = threading.Lock() self._preload_modules = ['__main__'] + def _stop(self): + # Method used by unit tests to stop the server + with self._lock: + self._stop_unlocked() + + def _stop_unlocked(self): + if self._forkserver_pid is None: + return + + # close the "alive" file descriptor asks the server to stop + os.close(self._forkserver_alive_fd) + self._forkserver_alive_fd = None + + os.waitpid(self._forkserver_pid, 0) + self._forkserver_pid = None + + os.unlink(self._forkserver_address) + self._forkserver_address = None + def set_forkserver_preload(self, modules_names): '''Set list of module names to try to load in forkserver process.''' if not all(type(mod) is str for mod in self._preload_modules): @@ -116,7 +135,8 @@ class ForkServer(object): with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') listener.bind(address) - os.chmod(address, 0o600) + if not util.is_abstract_socket_namespace(address): + os.chmod(address, 0o600) listener.listen() # all client processes own the write end of the "alive" pipe; diff --git a/lib-python/3/multiprocessing/managers.py b/lib-python/3/multiprocessing/managers.py index 8e8d28f4b7..c5043a4d3c 100644 --- a/lib-python/3/multiprocessing/managers.py +++ b/lib-python/3/multiprocessing/managers.py @@ -50,7 +50,7 @@ if view_types[0] is not list: # only needed in Py3.0 class Token(object): ''' - Type to uniquely indentify a shared object + Type to uniquely identify a shared object ''' __slots__ = ('typeid', 'address', 'id') @@ -805,7 +805,7 @@ class BaseProxy(object): def _callmethod(self, methodname, args=(), kwds={}): ''' - Try to call a method of the referrent and return a copy of the result + Try to call a method of the referent and return a copy of the result ''' try: conn = self._tls.connection diff --git a/lib-python/3/multiprocessing/popen_spawn_win32.py b/lib-python/3/multiprocessing/popen_spawn_win32.py index e01953d32b..d944209490 100644 --- a/lib-python/3/multiprocessing/popen_spawn_win32.py +++ b/lib-python/3/multiprocessing/popen_spawn_win32.py @@ -69,7 +69,7 @@ class Popen(object): try: hp, ht, pid, tid = _winapi.CreateProcess( python_exe, cmd, - env, None, False, 0, None, None, None) + None, None, False, 0, env, None, None) _winapi.CloseHandle(ht) except: _winapi.CloseHandle(rhandle) diff --git a/lib-python/3/multiprocessing/util.py b/lib-python/3/multiprocessing/util.py index 0c4eb24732..327fe42d6e 100644 --- a/lib-python/3/multiprocessing/util.py +++ b/lib-python/3/multiprocessing/util.py @@ -102,10 +102,42 @@ def log_to_stderr(level=None): _log_to_stderr = True return _logger + +# Abstract socket support + +def _platform_supports_abstract_sockets(): + if sys.platform == "linux": + return True + if hasattr(sys, 'getandroidapilevel'): + return True + return False + + +def is_abstract_socket_namespace(address): + if not address: + return False + if isinstance(address, bytes): + return address[0] == 0 + elif isinstance(address, str): + return address[0] == "\0" + raise TypeError('address type of {address!r} unrecognized') + + +abstract_sockets_supported = _platform_supports_abstract_sockets() + # # Function returning a temp directory which will be removed on exit # +def _remove_temp_dir(rmtree, tempdir): + rmtree(tempdir) + + current_process = process.current_process() + # current_process() can be None if the finalizer is called + # late during Python finalization + if current_process is not None: + current_process._config['tempdir'] = None + def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') @@ -113,7 +145,10 @@ def get_temp_dir(): import shutil, tempfile tempdir = tempfile.mkdtemp(prefix='pymp-') info('created temp directory %s', tempdir) - Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100) + # keep a strong reference to shutil.rmtree(), since the finalizer + # can be called late during Python shutdown + Finalize(None, _remove_temp_dir, args=(shutil.rmtree, tempdir), + exitpriority=-100) process.current_process()._config['tempdir'] = tempdir return tempdir @@ -421,3 +456,24 @@ def spawnv_passfds(path, args, passfds): finally: os.close(errpipe_read) os.close(errpipe_write) + + +def _cleanup_tests(): + """Cleanup multiprocessing resources when multiprocessing tests + completed.""" + + from test import support + + # cleanup multiprocessing + process._cleanup() + + # Stop the ForkServer process if it's running + from multiprocessing import forkserver + forkserver._forkserver._stop() + + # bpo-37421: Explicitly call _run_finalizers() to remove immediately + # temporary directories created by multiprocessing.util.get_temp_dir(). + _run_finalizers() + support.gc_collect() + + support.reap_children() diff --git a/lib-python/3/nntplib.py b/lib-python/3/nntplib.py index 5961a28ab7..9f5610ea87 100644 --- a/lib-python/3/nntplib.py +++ b/lib-python/3/nntplib.py @@ -1103,7 +1103,7 @@ if __name__ == '__main__': nntplib built-in demo - display the latest articles in a newsgroup""") parser.add_argument('-g', '--group', default='gmane.comp.python.general', help='group to fetch messages from (default: %(default)s)') - parser.add_argument('-s', '--server', default='news.gmane.org', + parser.add_argument('-s', '--server', default='news.gmane.io', help='NNTP server hostname (default: %(default)s)') parser.add_argument('-p', '--port', default=-1, type=int, help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT)) diff --git a/lib-python/3/os.py b/lib-python/3/os.py index 499e628561..9853e37c61 100644 --- a/lib-python/3/os.py +++ b/lib-python/3/os.py @@ -26,6 +26,8 @@ import abc import sys import stat as st +from _collections_abc import _check_methods + _names = sys.builtin_module_names # Note: more names are added to __all__ later. @@ -300,10 +302,11 @@ def walk(top, topdown=True, onerror=None, followlinks=False): (e.g., via del or slice assignment), and walk will only recurse into the subdirectories whose names remain in dirnames; this can be used to prune the search, or to impose a specific order of visiting. Modifying dirnames when - topdown is false is ineffective, since the directories in dirnames have - already been generated by the time dirnames itself is generated. No matter - the value of topdown, the list of subdirectories is retrieved before the - tuples for the directory and its subdirectories are generated. + topdown is false has no effect on the behavior of os.walk(), since the + directories in dirnames have already been generated by the time dirnames + itself is generated. No matter the value of topdown, the list of + subdirectories is retrieved before the tuples for the directory and its + subdirectories are generated. By default errors from the os.scandir() call are ignored. If optional arg 'onerror' is specified, it should be a function; it @@ -574,7 +577,7 @@ def execvpe(file, args, env): """execvpe(file, args, env) Execute the executable file (which is searched for along $PATH) - with argument list args and environment env , replacing the + with argument list args and environment env, replacing the current process. args may be a list or tuple of strings. """ _execvpe(file, args, env) @@ -1075,4 +1078,6 @@ class PathLike(abc.ABC): @classmethod def __subclasshook__(cls, subclass): - return hasattr(subclass, '__fspath__') + if cls is PathLike: + return _check_methods(subclass, '__fspath__') + return NotImplemented diff --git a/lib-python/3/pathlib.py b/lib-python/3/pathlib.py index 24437f8f95..f2967492ae 100644 --- a/lib-python/3/pathlib.py +++ b/lib-python/3/pathlib.py @@ -187,6 +187,9 @@ class _WindowsFlavour(_Flavour): def casefold_parts(self, parts): return [p.lower() for p in parts] + def compile_pattern(self, pattern): + return re.compile(fnmatch.translate(pattern), re.IGNORECASE).fullmatch + def resolve(self, path, strict=False): s = str(path) if not s: @@ -309,6 +312,9 @@ class _PosixFlavour(_Flavour): def casefold_parts(self, parts): return parts + def compile_pattern(self, pattern): + return re.compile(fnmatch.translate(pattern)).fullmatch + def resolve(self, path, strict=False): sep = self.sep accessor = path._accessor @@ -444,7 +450,7 @@ _normal_accessor = _NormalAccessor() # Globbing helpers # -def _make_selector(pattern_parts): +def _make_selector(pattern_parts, flavour): pat = pattern_parts[0] child_parts = pattern_parts[1:] if pat == '**': @@ -455,7 +461,7 @@ def _make_selector(pattern_parts): cls = _WildcardSelector else: cls = _PreciseSelector - return cls(pat, child_parts) + return cls(pat, child_parts, flavour) if hasattr(functools, "lru_cache"): _make_selector = functools.lru_cache()(_make_selector) @@ -465,10 +471,10 @@ class _Selector: """A selector matches a specific glob pattern part against the children of a given path.""" - def __init__(self, child_parts): + def __init__(self, child_parts, flavour): self.child_parts = child_parts if child_parts: - self.successor = _make_selector(child_parts) + self.successor = _make_selector(child_parts, flavour) self.dironly = True else: self.successor = _TerminatingSelector() @@ -494,9 +500,9 @@ class _TerminatingSelector: class _PreciseSelector(_Selector): - def __init__(self, name, child_parts): + def __init__(self, name, child_parts, flavour): self.name = name - _Selector.__init__(self, child_parts) + _Selector.__init__(self, child_parts, flavour) def _select_from(self, parent_path, is_dir, exists, scandir): try: @@ -510,42 +516,45 @@ class _PreciseSelector(_Selector): class _WildcardSelector(_Selector): - def __init__(self, pat, child_parts): - self.pat = re.compile(fnmatch.translate(pat)) - _Selector.__init__(self, child_parts) + def __init__(self, pat, child_parts, flavour): + self.match = flavour.compile_pattern(pat) + _Selector.__init__(self, child_parts, flavour) def _select_from(self, parent_path, is_dir, exists, scandir): try: - cf = parent_path._flavour.casefold - entries = list(scandir(parent_path)) + with scandir(parent_path) as scandir_it: + entries = list(scandir_it) for entry in entries: - entry_is_dir = False - try: - entry_is_dir = entry.is_dir() - except OSError as e: - if not _ignore_error(e): - raise - if not self.dironly or entry_is_dir: - name = entry.name - casefolded = cf(name) - if self.pat.match(casefolded): - path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, is_dir, exists, scandir): - yield p + if self.dironly: + try: + # "entry.is_dir()" can raise PermissionError + # in some cases (see bpo-38894), which is not + # among the errors ignored by _ignore_error() + if not entry.is_dir(): + continue + except OSError as e: + if not _ignore_error(e): + raise + continue + name = entry.name + if self.match(name): + path = parent_path._make_child_relpath(name) + for p in self.successor._select_from(path, is_dir, exists, scandir): + yield p except PermissionError: return - class _RecursiveWildcardSelector(_Selector): - def __init__(self, pat, child_parts): - _Selector.__init__(self, child_parts) + def __init__(self, pat, child_parts, flavour): + _Selector.__init__(self, child_parts, flavour) def _iterate_directories(self, parent_path, is_dir, scandir): yield parent_path try: - entries = list(scandir(parent_path)) + with scandir(parent_path) as scandir_it: + entries = list(scandir_it) for entry in entries: entry_is_dir = False try: @@ -793,7 +802,11 @@ class PurePath(object): @property def suffix(self): - """The final component's last suffix, if any.""" + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ name = self.name i = name.rfind('.') if 0 < i < len(name) - 1: @@ -803,7 +816,11 @@ class PurePath(object): @property def suffixes(self): - """A list of the final component's suffixes, if any.""" + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ name = self.name if name.endswith('.'): return [] @@ -1101,11 +1118,10 @@ class Path(PurePath): """ if not pattern: raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - pattern = self._flavour.casefold(pattern) drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) if drv or root: raise NotImplementedError("Non-relative patterns are unsupported") - selector = _make_selector(tuple(pattern_parts)) + selector = _make_selector(tuple(pattern_parts), self._flavour) for p in selector.select_from(self): yield p @@ -1114,11 +1130,10 @@ class Path(PurePath): directories) matching the given relative pattern, anywhere in this subtree. """ - pattern = self._flavour.casefold(pattern) drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) if drv or root: raise NotImplementedError("Non-relative patterns are unsupported") - selector = _make_selector(("**",) + tuple(pattern_parts)) + selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour) for p in selector.select_from(self): yield p diff --git a/lib-python/3/pdb.py b/lib-python/3/pdb.py index 59b23dfc8b..806bc6741f 100755 --- a/lib-python/3/pdb.py +++ b/lib-python/3/pdb.py @@ -159,16 +159,14 @@ class Pdb(bdb.Bdb, cmd.Cmd): self.allow_kbdint = False self.nosigint = nosigint - # Read $HOME/.pdbrc and ./.pdbrc + # Read ~/.pdbrc and ./.pdbrc self.rcLines = [] if readrc: - if 'HOME' in os.environ: - envHome = os.environ['HOME'] - try: - with open(os.path.join(envHome, ".pdbrc")) as rcFile: - self.rcLines.extend(rcFile) - except OSError: - pass + try: + with open(os.path.expanduser('~/.pdbrc')) as rcFile: + self.rcLines.extend(rcFile) + except OSError: + pass try: with open(".pdbrc") as rcFile: self.rcLines.extend(rcFile) @@ -1659,7 +1657,7 @@ To let the script run up to a given line X in the debugged file, use def main(): import getopt - opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['--help', '--command=']) + opts, args = getopt.getopt(sys.argv[1:], 'mhc:', ['help', 'command=']) if not args: print(_usage) diff --git a/lib-python/3/platform.py b/lib-python/3/platform.py index 6ab06b5832..7af46ffd17 100755 --- a/lib-python/3/platform.py +++ b/lib-python/3/platform.py @@ -570,9 +570,9 @@ def win32_ver(release='', version='', csd='', ptype=''): else: try: cvkey = r'SOFTWARE\Microsoft\Windows NT\CurrentVersion' - with winreg.OpenKeyEx(HKEY_LOCAL_MACHINE, cvkey) as key: - ptype = QueryValueEx(key, 'CurrentType')[0] - except: + with winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE, cvkey) as key: + ptype = winreg.QueryValueEx(key, 'CurrentType')[0] + except OSError: pass return release, version, csd, ptype diff --git a/lib-python/3/plistlib.py b/lib-python/3/plistlib.py index 21ebec3f00..33b79a133b 100644 --- a/lib-python/3/plistlib.py +++ b/lib-python/3/plistlib.py @@ -929,7 +929,7 @@ _FORMATS={ def load(fp, *, fmt=None, use_builtin_types=True, dict_type=dict): - """Read a .plist file. 'fp' should be (readable) file object. + """Read a .plist file. 'fp' should be a readable and binary file object. Return the unpacked root object (which usually is a dictionary). """ if fmt is None: @@ -960,8 +960,8 @@ def loads(value, *, fmt=None, use_builtin_types=True, dict_type=dict): def dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False): - """Write 'value' to a .plist file. 'fp' should be a (writable) - file object. + """Write 'value' to a .plist file. 'fp' should be a writable, + binary file object. """ if fmt not in _FORMATS: raise ValueError("Unsupported format: %r"%(fmt,)) diff --git a/lib-python/3/pydoc.py b/lib-python/3/pydoc.py index 44df8c854a..978e4cd0ba 100644 --- a/lib-python/3/pydoc.py +++ b/lib-python/3/pydoc.py @@ -66,6 +66,7 @@ import pkgutil import platform import re import sys +import sysconfig import time import tokenize import urllib.parse @@ -398,9 +399,7 @@ class Doc: docmodule = docclass = docroutine = docother = docproperty = docdata = fail - def getdocloc(self, object, - basedir=os.path.join(sys.base_exec_prefix, "lib", - "python%d.%d" % sys.version_info[:2])): + def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): """Return the location of module docs or None""" try: diff --git a/lib-python/3/pydoc_data/topics.py b/lib-python/3/pydoc_data/topics.py index 5e3229ab86..d42bb995f9 100644 --- a/lib-python/3/pydoc_data/topics.py +++ b/lib-python/3/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Jun 18 16:49:39 2019 +# Autogenerated by Sphinx on Sat Aug 15 01:12:49 2020 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -99,27 +99,26 @@ topics = {'assert': 'The "assert" statement\n' 'assigned,\n' ' from left to right, to the corresponding targets.\n' '\n' - ' * If the target list contains one target prefixed with an\n' - ' asterisk, called a “starred” target: The object must be ' - 'an\n' - ' iterable with at least as many items as there are targets ' - 'in the\n' - ' target list, minus one. The first items of the iterable ' - 'are\n' - ' assigned, from left to right, to the targets before the ' + ' * If the target list contains one target prefixed with an ' + 'asterisk,\n' + ' called a “starred” target: The object must be an iterable ' + 'with at\n' + ' least as many items as there are targets in the target ' + 'list, minus\n' + ' one. The first items of the iterable are assigned, from ' + 'left to\n' + ' right, to the targets before the starred target. The ' + 'final items\n' + ' of the iterable are assigned to the targets after the ' 'starred\n' - ' target. The final items of the iterable are assigned to ' - 'the\n' - ' targets after the starred target. A list of the remaining ' - 'items\n' - ' in the iterable is then assigned to the starred target ' - '(the list\n' - ' can be empty).\n' + ' target. A list of the remaining items in the iterable is ' + 'then\n' + ' assigned to the starred target (the list can be empty).\n' '\n' ' * Else: The object must be an iterable with the same number ' - 'of\n' - ' items as there are targets in the target list, and the ' - 'items are\n' + 'of items\n' + ' as there are targets in the target list, and the items ' + 'are\n' ' assigned, from left to right, to the corresponding ' 'targets.\n' '\n' @@ -135,10 +134,10 @@ topics = {'assert': 'The "assert" statement\n' 'in the\n' ' current local namespace.\n' '\n' - ' * Otherwise: the name is bound to the object in the global\n' - ' namespace or the outer namespace determined by ' - '"nonlocal",\n' - ' respectively.\n' + ' * Otherwise: the name is bound to the object in the global ' + 'namespace\n' + ' or the outer namespace determined by "nonlocal", ' + 'respectively.\n' '\n' ' The name is rebound if it was already bound. This may cause ' 'the\n' @@ -224,26 +223,27 @@ topics = {'assert': 'The "assert" statement\n' 'called with\n' ' appropriate arguments.\n' '\n' - '* If the target is a slicing: The primary expression in the\n' - ' reference is evaluated. It should yield a mutable sequence ' - 'object\n' - ' (such as a list). The assigned object should be a sequence ' - 'object\n' - ' of the same type. Next, the lower and upper bound ' - 'expressions are\n' - ' evaluated, insofar they are present; defaults are zero and ' - 'the\n' - ' sequence’s length. The bounds should evaluate to integers. ' - 'If\n' - ' either bound is negative, the sequence’s length is added to ' - 'it. The\n' - ' resulting bounds are clipped to lie between zero and the ' + '* If the target is a slicing: The primary expression in the ' + 'reference\n' + ' is evaluated. It should yield a mutable sequence object ' + '(such as a\n' + ' list). The assigned object should be a sequence object of ' + 'the same\n' + ' type. Next, the lower and upper bound expressions are ' + 'evaluated,\n' + ' insofar they are present; defaults are zero and the ' 'sequence’s\n' - ' length, inclusive. Finally, the sequence object is asked to ' - 'replace\n' - ' the slice with the items of the assigned sequence. The ' - 'length of\n' - ' the slice may be different from the length of the assigned ' + ' length. The bounds should evaluate to integers. If either ' + 'bound is\n' + ' negative, the sequence’s length is added to it. The ' + 'resulting\n' + ' bounds are clipped to lie between zero and the sequence’s ' + 'length,\n' + ' inclusive. Finally, the sequence object is asked to replace ' + 'the\n' + ' slice with the items of the assigned sequence. The length ' + 'of the\n' + ' slice may be different from the length of the assigned ' 'sequence,\n' ' thus changing the length of the target sequence, if the ' 'target\n' @@ -530,13 +530,17 @@ topics = {'assert': 'The "assert" statement\n' '\n' '-[ Footnotes ]-\n' '\n' - '[1] The exception is propagated to the invocation stack unless\n' - ' there is a "finally" clause which happens to raise another\n' - ' exception. That new exception causes the old one to be lost.\n' + '[1] The exception is propagated to the invocation stack unless ' + 'there\n' + ' is a "finally" clause which happens to raise another ' + 'exception.\n' + ' That new exception causes the old one to be lost.\n' '\n' - '[2] A string literal appearing as the first statement in the\n' - ' function body is transformed into the function’s "__doc__"\n' - ' attribute and therefore the function’s *docstring*.\n' + '[2] A string literal appearing as the first statement in the ' + 'function\n' + ' body is transformed into the function’s "__doc__" attribute ' + 'and\n' + ' therefore the function’s *docstring*.\n' '\n' '[3] A string literal appearing as the first statement in the class\n' ' body is transformed into the namespace’s "__doc__" item and\n' @@ -674,11 +678,13 @@ topics = {'assert': 'The "assert" statement\n' 'needs, for\n' ' example, "object.__getattribute__(self, name)".\n' '\n' - ' Note: This method may still be bypassed when looking ' - 'up special\n' - ' methods as the result of implicit invocation via ' - 'language syntax\n' - ' or built-in functions. See Special method lookup.\n' + ' Note:\n' + '\n' + ' This method may still be bypassed when looking up ' + 'special methods\n' + ' as the result of implicit invocation via language ' + 'syntax or\n' + ' built-in functions. See Special method lookup.\n' '\n' 'object.__setattr__(self, name, value)\n' '\n' @@ -735,10 +741,11 @@ topics = {'assert': 'The "assert" statement\n' 'returned.\n' '\n' 'The "__dir__" function should accept no arguments, and ' - 'return a list\n' - 'of strings that represents the names accessible on ' - 'module. If present,\n' - 'this function overrides the standard "dir()" search on a ' + 'return a\n' + 'sequence of strings that represents the names accessible ' + 'on module. If\n' + 'present, this function overrides the standard "dir()" ' + 'search on a\n' 'module.\n' '\n' 'For a more fine grained customization of the module ' @@ -761,15 +768,16 @@ topics = {'assert': 'The "assert" statement\n' '\n' ' sys.modules[__name__].__class__ = VerboseModule\n' '\n' - 'Note: Defining module "__getattr__" and setting module ' - '"__class__"\n' - ' only affect lookups made using the attribute access ' - 'syntax –\n' - ' directly accessing the module globals (whether by code ' - 'within the\n' - ' module, or via a reference to the module’s globals ' - 'dictionary) is\n' - ' unaffected.\n' + 'Note:\n' + '\n' + ' Defining module "__getattr__" and setting module ' + '"__class__" only\n' + ' affect lookups made using the attribute access syntax ' + '– directly\n' + ' accessing the module globals (whether by code within ' + 'the module, or\n' + ' via a reference to the module’s globals dictionary) is ' + 'unaffected.\n' '\n' 'Changed in version 3.5: "__class__" module attribute is ' 'now writable.\n' @@ -833,6 +841,24 @@ topics = {'assert': 'The "assert" statement\n' 'created. The\n' ' descriptor has been assigned to *name*.\n' '\n' + ' Note:\n' + '\n' + ' "__set_name__()" is only called implicitly as part ' + 'of the "type"\n' + ' constructor, so it will need to be called ' + 'explicitly with the\n' + ' appropriate parameters when a descriptor is added ' + 'to a class\n' + ' after initial creation:\n' + '\n' + ' class A:\n' + ' pass\n' + ' descr = custom_descriptor()\n' + ' A.attr = descr\n' + " descr.__set_name__(A, 'attr')\n" + '\n' + ' See Creating the class object for more details.\n' + '\n' ' New in version 3.6.\n' '\n' 'The attribute "__objclass__" is interpreted by the ' @@ -982,10 +1008,9 @@ topics = {'assert': 'The "assert" statement\n' '--------------------------\n' '\n' '* When inheriting from a class without *__slots__*, the ' - '*__dict__*\n' - ' and *__weakref__* attribute of the instances will ' - 'always be\n' - ' accessible.\n' + '*__dict__* and\n' + ' *__weakref__* attribute of the instances will always ' + 'be accessible.\n' '\n' '* Without a *__dict__* variable, instances cannot be ' 'assigned new\n' @@ -1000,14 +1025,12 @@ topics = {'assert': 'The "assert" statement\n' ' declaration.\n' '\n' '* Without a *__weakref__* variable for each instance, ' - 'classes\n' - ' defining *__slots__* do not support weak references to ' - 'its\n' - ' instances. If weak reference support is needed, then ' - 'add\n' - ' "\'__weakref__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' + 'classes defining\n' + ' *__slots__* do not support weak references to its ' + 'instances. If weak\n' + ' reference support is needed, then add ' + '"\'__weakref__\'" to the\n' + ' sequence of strings in the *__slots__* declaration.\n' '\n' '* *__slots__* are implemented at the class level by ' 'creating\n' @@ -1020,24 +1043,23 @@ topics = {'assert': 'The "assert" statement\n' ' attribute would overwrite the descriptor assignment.\n' '\n' '* The action of a *__slots__* declaration is not limited ' - 'to the\n' - ' class where it is defined. *__slots__* declared in ' - 'parents are\n' - ' available in child classes. However, child subclasses ' - 'will get a\n' - ' *__dict__* and *__weakref__* unless they also define ' - '*__slots__*\n' - ' (which should only contain names of any *additional* ' - 'slots).\n' + 'to the class\n' + ' where it is defined. *__slots__* declared in parents ' + 'are available\n' + ' in child classes. However, child subclasses will get a ' + '*__dict__*\n' + ' and *__weakref__* unless they also define *__slots__* ' + '(which should\n' + ' only contain names of any *additional* slots).\n' '\n' '* If a class defines a slot also defined in a base ' - 'class, the\n' - ' instance variable defined by the base class slot is ' - 'inaccessible\n' - ' (except by retrieving its descriptor directly from the ' - 'base class).\n' - ' This renders the meaning of the program undefined. In ' - 'the future, a\n' + 'class, the instance\n' + ' variable defined by the base class slot is ' + 'inaccessible (except by\n' + ' retrieving its descriptor directly from the base ' + 'class). This\n' + ' renders the meaning of the program undefined. In the ' + 'future, a\n' ' check may be added to prevent this.\n' '\n' '* Nonempty *__slots__* does not work for classes derived ' @@ -1046,9 +1068,9 @@ topics = {'assert': 'The "assert" statement\n' '"bytes" and "tuple".\n' '\n' '* Any non-string iterable may be assigned to ' - '*__slots__*. Mappings\n' - ' may also be used; however, in the future, special ' - 'meaning may be\n' + '*__slots__*. Mappings may\n' + ' also be used; however, in the future, special meaning ' + 'may be\n' ' assigned to the values corresponding to each key.\n' '\n' '* *__class__* assignment works only if both classes have ' @@ -1061,7 +1083,13 @@ topics = {'assert': 'The "assert" statement\n' 'attributes created by\n' ' slots (the other bases must have empty slot layouts) - ' 'violations\n' - ' raise "TypeError".\n', + ' raise "TypeError".\n' + '\n' + '* If an iterator is used for *__slots__* then a ' + 'descriptor is created\n' + ' for each of the iterator’s values. However, the ' + '*__slots__*\n' + ' attribute will be an empty iterator.\n', 'attribute-references': 'Attribute references\n' '********************\n' '\n' @@ -1816,9 +1844,9 @@ topics = {'assert': 'The "assert" statement\n' ' value is false. A counter-intuitive implication is that ' 'not-a-number\n' ' values are not equal to themselves. For example, if "x =\n' - ' float(\'NaN\')", "3 < x", "x < 3", "x == x", "x != x" are ' - 'all false.\n' - ' This behavior is compliant with IEEE 754.\n' + ' float(\'NaN\')", "3 < x", "x < 3" and "x == x" are all ' + 'false, while "x\n' + ' != x" is true. This behavior is compliant with IEEE 754.\n' '\n' '* Binary sequences (instances of "bytes" or "bytearray") can ' 'be\n' @@ -1834,15 +1862,15 @@ topics = {'assert': 'The "assert" statement\n' '\n' ' Strings and binary sequences cannot be directly compared.\n' '\n' - '* Sequences (instances of "tuple", "list", or "range") can ' - 'be\n' - ' compared only within each of their types, with the ' - 'restriction that\n' - ' ranges do not support order comparison. Equality ' - 'comparison across\n' - ' these types results in inequality, and ordering comparison ' - 'across\n' - ' these types raises "TypeError".\n' + '* Sequences (instances of "tuple", "list", or "range") can be ' + 'compared\n' + ' only within each of their types, with the restriction that ' + 'ranges do\n' + ' not support order comparison. Equality comparison across ' + 'these\n' + ' types results in inequality, and ordering comparison across ' + 'these\n' + ' types raises "TypeError".\n' '\n' ' Sequences compare lexicographically using comparison of\n' ' corresponding elements, whereby reflexivity of the elements ' @@ -1891,8 +1919,8 @@ topics = {'assert': 'The "assert" statement\n' ' false because the type is not the same).\n' '\n' ' * Collections that support order comparison are ordered the ' - 'same\n' - ' as their first unequal elements (for example, "[1,2,x] <= ' + 'same as\n' + ' their first unequal elements (for example, "[1,2,x] <= ' '[1,2,y]"\n' ' has the same value as "x <= y"). If a corresponding ' 'element does\n' @@ -1910,8 +1938,8 @@ topics = {'assert': 'The "assert" statement\n' '"TypeError".\n' '\n' '* Sets (instances of "set" or "frozenset") can be compared ' - 'within\n' - ' and across their types.\n' + 'within and\n' + ' across their types.\n' '\n' ' They define order comparison operators to mean subset and ' 'superset\n' @@ -1930,8 +1958,8 @@ topics = {'assert': 'The "assert" statement\n' ' Comparison of sets enforces reflexivity of its elements.\n' '\n' '* Most other built-in types have no comparison methods ' - 'implemented,\n' - ' so they inherit the default comparison behavior.\n' + 'implemented, so\n' + ' they inherit the default comparison behavior.\n' '\n' 'User-defined classes that customize their comparison behavior ' 'should\n' @@ -1980,10 +2008,10 @@ topics = {'assert': 'The "assert" statement\n' ' "total_ordering()" decorator.\n' '\n' '* The "hash()" result should be consistent with equality. ' - 'Objects\n' - ' that are equal should either have the same hash value, or ' - 'be marked\n' - ' as unhashable.\n' + 'Objects that\n' + ' are equal should either have the same hash value, or be ' + 'marked as\n' + ' unhashable.\n' '\n' 'Python does not enforce these consistency rules. In fact, ' 'the\n' @@ -2257,10 +2285,11 @@ topics = {'assert': 'The "assert" statement\n' ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, ' '2]".\n' '\n' - 'Note: There is a subtlety when the sequence is being modified by ' - 'the\n' - ' loop (this can only occur for mutable sequences, e.g. lists). ' - 'An\n' + 'Note:\n' + '\n' + ' There is a subtlety when the sequence is being modified by the ' + 'loop\n' + ' (this can only occur for mutable sequences, e.g. lists). An\n' ' internal counter is used to keep track of which item is used ' 'next,\n' ' and this is incremented on each iteration. When this counter ' @@ -2481,20 +2510,22 @@ topics = {'assert': 'The "assert" statement\n' 'follows:\n' '\n' '1. The context expression (the expression given in the ' - '"with_item")\n' - ' is evaluated to obtain a context manager.\n' + '"with_item") is\n' + ' evaluated to obtain a context manager.\n' '\n' '2. The context manager’s "__exit__()" is loaded for later use.\n' '\n' '3. The context manager’s "__enter__()" method is invoked.\n' '\n' - '4. If a target was included in the "with" statement, the return\n' - ' value from "__enter__()" is assigned to it.\n' + '4. If a target was included in the "with" statement, the return ' + 'value\n' + ' from "__enter__()" is assigned to it.\n' + '\n' + ' Note:\n' '\n' - ' Note: The "with" statement guarantees that if the ' - '"__enter__()"\n' - ' method returns without an error, then "__exit__()" will ' - 'always be\n' + ' The "with" statement guarantees that if the "__enter__()" ' + 'method\n' + ' returns without an error, then "__exit__()" will always be\n' ' called. Thus, if an error occurs during the assignment to ' 'the\n' ' target list, it will be treated the same as an error ' @@ -2967,14 +2998,17 @@ topics = {'assert': 'The "assert" statement\n' '\n' '-[ Footnotes ]-\n' '\n' - '[1] The exception is propagated to the invocation stack unless\n' - ' there is a "finally" clause which happens to raise another\n' - ' exception. That new exception causes the old one to be ' - 'lost.\n' + '[1] The exception is propagated to the invocation stack unless ' + 'there\n' + ' is a "finally" clause which happens to raise another ' + 'exception.\n' + ' That new exception causes the old one to be lost.\n' '\n' - '[2] A string literal appearing as the first statement in the\n' - ' function body is transformed into the function’s "__doc__"\n' - ' attribute and therefore the function’s *docstring*.\n' + '[2] A string literal appearing as the first statement in the ' + 'function\n' + ' body is transformed into the function’s "__doc__" attribute ' + 'and\n' + ' therefore the function’s *docstring*.\n' '\n' '[3] A string literal appearing as the first statement in the ' 'class\n' @@ -3064,7 +3098,7 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'When a description of an arithmetic operator below uses the ' 'phrase\n' - '“the numeric arguments are converted to a common type,” this ' + '“the numeric arguments are converted to a common type”, this ' 'means\n' 'that the operator implementation for built-in types works as ' 'follows:\n' @@ -3074,8 +3108,8 @@ topics = {'assert': 'The "assert" statement\n' ' complex;\n' '\n' '* otherwise, if either argument is a floating point number, ' - 'the\n' - ' other is converted to floating point;\n' + 'the other\n' + ' is converted to floating point;\n' '\n' '* otherwise, both must be integers and no conversion is ' 'necessary.\n' @@ -3183,7 +3217,9 @@ topics = {'assert': 'The "assert" statement\n' 'for\n' ' objects that still exist when the interpreter exits.\n' '\n' - ' Note: "del x" doesn’t directly call "x.__del__()" — the ' + ' Note:\n' + '\n' + ' "del x" doesn’t directly call "x.__del__()" — the ' 'former\n' ' decrements the reference count for "x" by one, and the ' 'latter is\n' @@ -3207,13 +3243,15 @@ topics = {'assert': 'The "assert" statement\n' '\n' ' See also: Documentation for the "gc" module.\n' '\n' - ' Warning: Due to the precarious circumstances under ' - 'which\n' - ' "__del__()" methods are invoked, exceptions that occur ' - 'during\n' - ' their execution are ignored, and a warning is printed ' - 'to\n' - ' "sys.stderr" instead. In particular:\n' + ' Warning:\n' + '\n' + ' Due to the precarious circumstances under which ' + '"__del__()"\n' + ' methods are invoked, exceptions that occur during ' + 'their execution\n' + ' are ignored, and a warning is printed to "sys.stderr" ' + 'instead.\n' + ' In particular:\n' '\n' ' * "__del__()" can be invoked when arbitrary code is ' 'being\n' @@ -3226,22 +3264,20 @@ topics = {'assert': 'The "assert" statement\n' ' that gets interrupted to execute "__del__()".\n' '\n' ' * "__del__()" can be executed during interpreter ' - 'shutdown. As\n' - ' a consequence, the global variables it needs to ' - 'access\n' - ' (including other modules) may already have been ' - 'deleted or set\n' - ' to "None". Python guarantees that globals whose name ' - 'begins\n' - ' with a single underscore are deleted from their ' - 'module before\n' - ' other globals are deleted; if no other references to ' - 'such\n' - ' globals exist, this may help in assuring that ' - 'imported modules\n' - ' are still available at the time when the "__del__()" ' - 'method is\n' - ' called.\n' + 'shutdown. As a\n' + ' consequence, the global variables it needs to access ' + '(including\n' + ' other modules) may already have been deleted or set ' + 'to "None".\n' + ' Python guarantees that globals whose name begins ' + 'with a single\n' + ' underscore are deleted from their module before ' + 'other globals\n' + ' are deleted; if no other references to such globals ' + 'exist, this\n' + ' may help in assuring that imported modules are still ' + 'available\n' + ' at the time when the "__del__()" method is called.\n' '\n' 'object.__repr__(self)\n' '\n' @@ -3417,19 +3453,21 @@ topics = {'assert': 'The "assert" statement\n' ' def __hash__(self):\n' ' return hash((self.name, self.nick, self.color))\n' '\n' - ' Note: "hash()" truncates the value returned from an ' - 'object’s\n' - ' custom "__hash__()" method to the size of a ' - '"Py_ssize_t". This\n' - ' is typically 8 bytes on 64-bit builds and 4 bytes on ' - '32-bit\n' - ' builds. If an object’s "__hash__()" must ' - 'interoperate on builds\n' - ' of different bit sizes, be sure to check the width on ' - 'all\n' - ' supported builds. An easy way to do this is with ' - '"python -c\n' - ' "import sys; print(sys.hash_info.width)"".\n' + ' Note:\n' + '\n' + ' "hash()" truncates the value returned from an object’s ' + 'custom\n' + ' "__hash__()" method to the size of a "Py_ssize_t". ' + 'This is\n' + ' typically 8 bytes on 64-bit builds and 4 bytes on ' + '32-bit builds.\n' + ' If an object’s "__hash__()" must interoperate on ' + 'builds of\n' + ' different bit sizes, be sure to check the width on all ' + 'supported\n' + ' builds. An easy way to do this is with "python -c ' + '"import sys;\n' + ' print(sys.hash_info.width)"".\n' '\n' ' If a class does not define an "__eq__()" method it ' 'should not\n' @@ -3487,10 +3525,12 @@ topics = {'assert': 'The "assert" statement\n' ' hashable by an "isinstance(obj, ' 'collections.abc.Hashable)" call.\n' '\n' - ' Note: By default, the "__hash__()" values of str, bytes ' - 'and\n' - ' datetime objects are “salted” with an unpredictable ' - 'random value.\n' + ' Note:\n' + '\n' + ' By default, the "__hash__()" values of str, bytes and ' + 'datetime\n' + ' objects are “salted” with an unpredictable random ' + 'value.\n' ' Although they remain constant within an individual ' 'Python\n' ' process, they are not predictable between repeated ' @@ -4080,9 +4120,11 @@ topics = {'assert': 'The "assert" statement\n' 'its\n' ' value.\n' '\n' - ' Note: "print()" can also be used, but is not a debugger ' - 'command —\n' - ' this executes the Python "print()" function.\n' + ' Note:\n' + '\n' + ' "print()" can also be used, but is not a debugger command — ' + 'this\n' + ' executes the Python "print()" function.\n' '\n' 'pp expression\n' '\n' @@ -4199,11 +4241,22 @@ topics = {'assert': 'The "assert" statement\n' ' Quit from the debugger. The program being executed is ' 'aborted.\n' '\n' + 'debug code\n' + '\n' + ' Enter a recursive debugger that steps through the code ' + 'argument\n' + ' (which is an arbitrary expression or statement to be executed ' + 'in\n' + ' the current environment).\n' + '\n' + 'retval\n' + 'Print the return value for the last return of a function.\n' + '\n' '-[ Footnotes ]-\n' '\n' '[1] Whether a frame is considered to originate in a certain ' - 'module\n' - ' is determined by the "__name__" in the frame globals.\n', + 'module is\n' + ' determined by the "__name__" in the frame globals.\n', 'del': 'The "del" statement\n' '*******************\n' '\n' @@ -4376,13 +4429,15 @@ topics = {'assert': 'The "assert" statement\n' 'about the\n' 'exceptional condition.\n' '\n' - 'Note: Exception messages are not part of the Python API. ' - 'Their\n' - ' contents may change from one version of Python to the next ' - 'without\n' - ' warning and should not be relied on by code which will run ' - 'under\n' - ' multiple versions of the interpreter.\n' + 'Note:\n' + '\n' + ' Exception messages are not part of the Python API. Their ' + 'contents\n' + ' may change from one version of Python to the next without ' + 'warning\n' + ' and should not be relied on by code which will run under ' + 'multiple\n' + ' versions of the interpreter.\n' '\n' 'See also the description of the "try" statement in section The ' 'try\n' @@ -4392,10 +4447,9 @@ topics = {'assert': 'The "assert" statement\n' '-[ Footnotes ]-\n' '\n' '[1] This limitation occurs because the code that is executed ' - 'by\n' - ' these operations is not available at the time the module ' - 'is\n' - ' compiled.\n', + 'by these\n' + ' operations is not available at the time the module is ' + 'compiled.\n', 'execmodel': 'Execution model\n' '***************\n' '\n' @@ -4697,13 +4751,15 @@ topics = {'assert': 'The "assert" statement\n' 'about the\n' 'exceptional condition.\n' '\n' - 'Note: Exception messages are not part of the Python API. ' - 'Their\n' - ' contents may change from one version of Python to the next ' - 'without\n' - ' warning and should not be relied on by code which will run ' - 'under\n' - ' multiple versions of the interpreter.\n' + 'Note:\n' + '\n' + ' Exception messages are not part of the Python API. Their ' + 'contents\n' + ' may change from one version of Python to the next without ' + 'warning\n' + ' and should not be relied on by code which will run under ' + 'multiple\n' + ' versions of the interpreter.\n' '\n' 'See also the description of the "try" statement in section The ' 'try\n' @@ -4712,11 +4768,10 @@ topics = {'assert': 'The "assert" statement\n' '\n' '-[ Footnotes ]-\n' '\n' - '[1] This limitation occurs because the code that is executed ' - 'by\n' - ' these operations is not available at the time the module ' - 'is\n' - ' compiled.\n', + '[1] This limitation occurs because the code that is executed by ' + 'these\n' + ' operations is not available at the time the module is ' + 'compiled.\n', 'exprlists': 'Expression lists\n' '****************\n' '\n' @@ -4836,8 +4891,11 @@ topics = {'assert': 'The "assert" statement\n' 'i\n' ':= a to b do"; e.g., "list(range(3))" returns the list "[0, 1, 2]".\n' '\n' - 'Note: There is a subtlety when the sequence is being modified by the\n' - ' loop (this can only occur for mutable sequences, e.g. lists). An\n' + 'Note:\n' + '\n' + ' There is a subtlety when the sequence is being modified by the ' + 'loop\n' + ' (this can only occur for mutable sequences, e.g. lists). An\n' ' internal counter is used to keep track of which item is used next,\n' ' and this is incremented on each iteration. When this counter has\n' ' reached the length of the sequence the loop terminates. This ' @@ -5036,11 +5094,11 @@ topics = {'assert': 'The "assert" statement\n' 'only\n' 'supported by the numeric types.\n' '\n' - 'A general convention is that an empty format string ("""") ' + 'A general convention is that an empty format specification ' 'produces\n' 'the same result as if you had called "str()" on the value. ' 'A non-empty\n' - 'format string typically modifies the result.\n' + 'format specification typically modifies the result.\n' '\n' 'The general form of a *standard format specifier* is:\n' '\n' @@ -5193,9 +5251,12 @@ topics = {'assert': 'The "assert" statement\n' 'Changed in version 3.6: Added the "\'_\'" option (see also ' '**PEP 515**).\n' '\n' - '*width* is a decimal integer defining the minimum field ' - 'width. If not\n' - 'specified, then the field width will be determined by the ' + '*width* is a decimal integer defining the minimum total ' + 'field width,\n' + 'including any prefixes, separators, and other formatting ' + 'characters.\n' + 'If not specified, then the field width will be determined ' + 'by the\n' 'content.\n' '\n' 'When no explicit alignment is given, preceding the *width* ' @@ -5365,17 +5426,19 @@ topics = {'assert': 'The "assert" statement\n' ' | | significand, and the decimal point is also ' 'removed if |\n' ' | | there are no remaining digits following ' - 'it. Positive and |\n' - ' | | negative infinity, positive and negative ' - 'zero, and nans, |\n' - ' | | are formatted as "inf", "-inf", "0", "-0" ' - 'and "nan" |\n' - ' | | respectively, regardless of the ' - 'precision. A precision of |\n' - ' | | "0" is treated as equivalent to a ' - 'precision of "1". The |\n' - ' | | default precision is ' - '"6". |\n' + 'it, unless the |\n' + ' | | "\'#\'" option is used. Positive and ' + 'negative infinity, |\n' + ' | | positive and negative zero, and nans, are ' + 'formatted as |\n' + ' | | "inf", "-inf", "0", "-0" and "nan" ' + 'respectively, |\n' + ' | | regardless of the precision. A precision ' + 'of "0" is |\n' + ' | | treated as equivalent to a precision of ' + '"1". The default |\n' + ' | | precision is ' + '"6". |\n' ' ' '+-----------+------------------------------------------------------------+\n' ' | "\'G\'" | General format. Same as "\'g\'" except ' @@ -5830,25 +5893,26 @@ topics = {'assert': 'The "assert" statement\n' 'defined.\n' ' See section The import statement.\n' '\n' - ' Note: The name "_" is often used in conjunction with\n' + ' Note:\n' + '\n' + ' The name "_" is often used in conjunction with\n' ' internationalization; refer to the documentation for the\n' ' "gettext" module for more information on this ' 'convention.\n' '\n' '"__*__"\n' - ' System-defined names. These names are defined by the ' - 'interpreter\n' - ' and its implementation (including the standard library). ' - 'Current\n' - ' system names are discussed in the Special method names ' - 'section and\n' - ' elsewhere. More will likely be defined in future versions ' - 'of\n' - ' Python. *Any* use of "__*__" names, in any context, that ' - 'does not\n' - ' follow explicitly documented use, is subject to breakage ' - 'without\n' - ' warning.\n' + ' System-defined names, informally known as “dunder” names. ' + 'These\n' + ' names are defined by the interpreter and its ' + 'implementation\n' + ' (including the standard library). Current system names are\n' + ' discussed in the Special method names section and ' + 'elsewhere. More\n' + ' will likely be defined in future versions of Python. *Any* ' + 'use of\n' + ' "__*__" names, in any context, that does not follow ' + 'explicitly\n' + ' documented use, is subject to breakage without warning.\n' '\n' '"__*"\n' ' Class-private names. Names in this category, when used ' @@ -5935,8 +5999,8 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'A non-normative HTML file listing all valid identifier ' 'characters for\n' - 'Unicode 4.1 can be found at https://www.dcl.hpi.uni-\n' - 'potsdam.de/home/loewis/table-3131.html.\n' + 'Unicode 4.1 can be found at\n' + 'https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt\n' '\n' '\n' 'Keywords\n' @@ -5977,26 +6041,28 @@ topics = {'assert': 'The "assert" statement\n' 'defined.\n' ' See section The import statement.\n' '\n' - ' Note: The name "_" is often used in conjunction with\n' + ' Note:\n' + '\n' + ' The name "_" is often used in conjunction with\n' ' internationalization; refer to the documentation for ' 'the\n' ' "gettext" module for more information on this ' 'convention.\n' '\n' '"__*__"\n' - ' System-defined names. These names are defined by the ' - 'interpreter\n' - ' and its implementation (including the standard library). ' - 'Current\n' - ' system names are discussed in the Special method names ' - 'section and\n' - ' elsewhere. More will likely be defined in future versions ' - 'of\n' - ' Python. *Any* use of "__*__" names, in any context, that ' - 'does not\n' - ' follow explicitly documented use, is subject to breakage ' - 'without\n' - ' warning.\n' + ' System-defined names, informally known as “dunder” names. ' + 'These\n' + ' names are defined by the interpreter and its ' + 'implementation\n' + ' (including the standard library). Current system names ' + 'are\n' + ' discussed in the Special method names section and ' + 'elsewhere. More\n' + ' will likely be defined in future versions of Python. ' + '*Any* use of\n' + ' "__*__" names, in any context, that does not follow ' + 'explicitly\n' + ' documented use, is subject to breakage without warning.\n' '\n' '"__*"\n' ' Class-private names. Names in this category, when used ' @@ -6062,8 +6128,9 @@ topics = {'assert': 'The "assert" statement\n' '\n' '1. find a module, loading and initializing it if necessary\n' '\n' - '2. define a name or names in the local namespace for the scope\n' - ' where the "import" statement occurs.\n' + '2. define a name or names in the local namespace for the scope ' + 'where\n' + ' the "import" statement occurs.\n' '\n' 'When the statement contains multiple clauses (separated by commas) ' 'the\n' @@ -6089,8 +6156,9 @@ topics = {'assert': 'The "assert" statement\n' 'made\n' 'available in the local namespace in one of three ways:\n' '\n' - '* If the module name is followed by "as", then the name following\n' - ' "as" is bound directly to the imported module.\n' + '* If the module name is followed by "as", then the name following ' + '"as"\n' + ' is bound directly to the imported module.\n' '\n' '* If no other name is specified, and the module being imported is ' 'a\n' @@ -6735,7 +6803,7 @@ topics = {'assert': 'The "assert" statement\n' 'object.__rfloordiv__(self, other)\n' 'object.__rmod__(self, other)\n' 'object.__rdivmod__(self, other)\n' - 'object.__rpow__(self, other)\n' + 'object.__rpow__(self, other[, modulo])\n' 'object.__rlshift__(self, other)\n' 'object.__rrshift__(self, other)\n' 'object.__rand__(self, other)\n' @@ -6764,15 +6832,17 @@ topics = {'assert': 'The "assert" statement\n' '"__rpow__()" (the\n' ' coercion rules would become too complicated).\n' '\n' - ' Note: If the right operand’s type is a subclass of the ' - 'left\n' - ' operand’s type and that subclass provides the ' - 'reflected method\n' - ' for the operation, this method will be called before ' - 'the left\n' - ' operand’s non-reflected method. This behavior allows ' - 'subclasses\n' - ' to override their ancestors’ operations.\n' + ' Note:\n' + '\n' + ' If the right operand’s type is a subclass of the left ' + 'operand’s\n' + ' type and that subclass provides the reflected method ' + 'for the\n' + ' operation, this method will be called before the left ' + 'operand’s\n' + ' non-reflected method. This behavior allows subclasses ' + 'to\n' + ' override their ancestors’ operations.\n' '\n' 'object.__iadd__(self, other)\n' 'object.__isub__(self, other)\n' @@ -6846,8 +6916,9 @@ topics = {'assert': 'The "assert" statement\n' 'numeric\n' ' object is an integer type. Must return an integer.\n' '\n' - ' Note: In order to have a coherent integer type class, ' - 'when\n' + ' Note:\n' + '\n' + ' In order to have a coherent integer type class, when\n' ' "__index__()" is defined "__int__()" should also be ' 'defined, and\n' ' both should return the same value.\n' @@ -6877,7 +6948,7 @@ topics = {'assert': 'The "assert" statement\n' 'program is represented by objects or by relations between ' 'objects. (In\n' 'a sense, and in conformance to Von Neumann’s model of a “stored\n' - 'program computer,” code is also represented by objects.)\n' + 'program computer”, code is also represented by objects.)\n' '\n' 'Every object has an identity, a type and a value. An object’s\n' '*identity* never changes once it has been created; you may think ' @@ -7078,10 +7149,10 @@ topics = {'assert': 'The "assert" statement\n' '| "x(arguments...)", "x.attribute" | ' 'attribute reference |\n' '+-------------------------------------------------+---------------------------------------+\n' - '| "(expressions...)", "[expressions...]", "{key: | ' - 'Binding or tuple display, list |\n' - '| value...}", "{expressions...}" | ' - 'display, dictionary display, set |\n' + '| "(expressions...)", "[expressions...]", "{key: | ' + 'Binding or parenthesized expression, |\n' + '| value...}", "{expressions...}" | list ' + 'display, dictionary display, set |\n' '| | ' 'display |\n' '+-------------------------------------------------+---------------------------------------+\n' @@ -7089,8 +7160,8 @@ topics = {'assert': 'The "assert" statement\n' '-[ Footnotes ]-\n' '\n' '[1] While "abs(x%y) < abs(y)" is true mathematically, ' - 'for floats\n' - ' it may not be true numerically due to roundoff. For ' + 'for floats it\n' + ' may not be true numerically due to roundoff. For ' 'example, and\n' ' assuming a platform on which a Python float is an ' 'IEEE 754 double-\n' @@ -7155,22 +7226,22 @@ topics = {'assert': 'The "assert" statement\n' '"unicodedata.normalize()".\n' '\n' '[4] Due to automatic garbage-collection, free lists, and ' - 'the\n' - ' dynamic nature of descriptors, you may notice ' - 'seemingly unusual\n' - ' behaviour in certain uses of the "is" operator, like ' - 'those\n' - ' involving comparisons between instance methods, or ' - 'constants.\n' - ' Check their documentation for more info.\n' + 'the dynamic\n' + ' nature of descriptors, you may notice seemingly ' + 'unusual behaviour\n' + ' in certain uses of the "is" operator, like those ' + 'involving\n' + ' comparisons between instance methods, or constants. ' + 'Check their\n' + ' documentation for more info.\n' '\n' '[5] The "%" operator is also used for string formatting; ' 'the same\n' ' precedence applies.\n' '\n' '[6] The power operator "**" binds less tightly than an ' - 'arithmetic\n' - ' or bitwise unary operator on its right, that is, ' + 'arithmetic or\n' + ' bitwise unary operator on its right, that is, ' '"2**-1" is "0.5".\n', 'pass': 'The "pass" statement\n' '********************\n' @@ -7417,9 +7488,9 @@ topics = {'assert': 'The "assert" statement\n' 'to allow\n' 'efficient iteration through the container; for mappings, ' '"__iter__()"\n' - 'should be the same as "keys()"; for sequences, it should ' - 'iterate\n' - 'through the values.\n' + 'should iterate through the object’s keys; for sequences, ' + 'it should\n' + 'iterate through the values.\n' '\n' 'object.__len__(self)\n' '\n' @@ -7449,16 +7520,22 @@ topics = {'assert': 'The "assert" statement\n' ' estimated length for the object (which may be greater ' 'or less than\n' ' the actual length). The length must be an integer ">=" ' - '0. This\n' + '0. The\n' + ' return value may also be "NotImplemented", which is ' + 'treated the\n' + ' same as if the "__length_hint__" method didn’t exist at ' + 'all. This\n' ' method is purely an optimization and is never required ' 'for\n' ' correctness.\n' '\n' ' New in version 3.4.\n' '\n' - 'Note: Slicing is done exclusively with the following three ' - 'methods.\n' - ' A call like\n' + 'Note:\n' + '\n' + ' Slicing is done exclusively with the following three ' + 'methods. A\n' + ' call like\n' '\n' ' a[1:2] = b\n' '\n' @@ -7489,7 +7566,9 @@ topics = {'assert': 'The "assert" statement\n' 'the\n' ' container), "KeyError" should be raised.\n' '\n' - ' Note: "for" loops expect that an "IndexError" will be ' + ' Note:\n' + '\n' + ' "for" loops expect that an "IndexError" will be ' 'raised for\n' ' illegal indexes to allow proper detection of the end ' 'of the\n' @@ -7567,12 +7646,12 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'The membership test operators ("in" and "not in") are ' 'normally\n' - 'implemented as an iteration through a sequence. However, ' + 'implemented as an iteration through a container. However, ' 'container\n' 'objects can supply the following special method with a ' 'more efficient\n' 'implementation, which also does not require the object be ' - 'a sequence.\n' + 'iterable.\n' '\n' 'object.__contains__(self, item)\n' '\n' @@ -7725,26 +7804,26 @@ topics = {'assert': 'The "assert" statement\n' '-[ Footnotes ]-\n' '\n' '[1] Additional information on these special methods may be ' - 'found\n' - ' in the Python Reference Manual (Basic customization).\n' + 'found in\n' + ' the Python Reference Manual (Basic customization).\n' '\n' '[2] As a consequence, the list "[1, 2]" is considered equal ' - 'to\n' - ' "[1.0, 2.0]", and similarly for tuples.\n' + 'to "[1.0,\n' + ' 2.0]", and similarly for tuples.\n' '\n' '[3] They must have since the parser can’t tell the type of ' 'the\n' ' operands.\n' '\n' '[4] Cased characters are those with general category ' - 'property\n' - ' being one of “Lu” (Letter, uppercase), “Ll” (Letter, ' - 'lowercase),\n' - ' or “Lt” (Letter, titlecase).\n' - '\n' - '[5] To format only a tuple you should therefore provide a\n' - ' singleton tuple whose only element is the tuple to be ' - 'formatted.\n', + 'property being\n' + ' one of “Lu” (Letter, uppercase), “Ll” (Letter, ' + 'lowercase), or “Lt”\n' + ' (Letter, titlecase).\n' + '\n' + '[5] To format only a tuple you should therefore provide a ' + 'singleton\n' + ' tuple whose only element is the tuple to be formatted.\n', 'specialnames': 'Special method names\n' '********************\n' '\n' @@ -7887,7 +7966,9 @@ topics = {'assert': 'The "assert" statement\n' 'for\n' ' objects that still exist when the interpreter exits.\n' '\n' - ' Note: "del x" doesn’t directly call "x.__del__()" — the ' + ' Note:\n' + '\n' + ' "del x" doesn’t directly call "x.__del__()" — the ' 'former\n' ' decrements the reference count for "x" by one, and the ' 'latter is\n' @@ -7911,12 +7992,15 @@ topics = {'assert': 'The "assert" statement\n' '\n' ' See also: Documentation for the "gc" module.\n' '\n' - ' Warning: Due to the precarious circumstances under which\n' - ' "__del__()" methods are invoked, exceptions that occur ' - 'during\n' - ' their execution are ignored, and a warning is printed ' - 'to\n' - ' "sys.stderr" instead. In particular:\n' + ' Warning:\n' + '\n' + ' Due to the precarious circumstances under which ' + '"__del__()"\n' + ' methods are invoked, exceptions that occur during their ' + 'execution\n' + ' are ignored, and a warning is printed to "sys.stderr" ' + 'instead.\n' + ' In particular:\n' '\n' ' * "__del__()" can be invoked when arbitrary code is ' 'being\n' @@ -7929,22 +8013,20 @@ topics = {'assert': 'The "assert" statement\n' ' that gets interrupted to execute "__del__()".\n' '\n' ' * "__del__()" can be executed during interpreter ' - 'shutdown. As\n' - ' a consequence, the global variables it needs to ' - 'access\n' - ' (including other modules) may already have been ' - 'deleted or set\n' - ' to "None". Python guarantees that globals whose name ' - 'begins\n' - ' with a single underscore are deleted from their ' - 'module before\n' - ' other globals are deleted; if no other references to ' - 'such\n' - ' globals exist, this may help in assuring that ' - 'imported modules\n' - ' are still available at the time when the "__del__()" ' - 'method is\n' - ' called.\n' + 'shutdown. As a\n' + ' consequence, the global variables it needs to access ' + '(including\n' + ' other modules) may already have been deleted or set ' + 'to "None".\n' + ' Python guarantees that globals whose name begins with ' + 'a single\n' + ' underscore are deleted from their module before other ' + 'globals\n' + ' are deleted; if no other references to such globals ' + 'exist, this\n' + ' may help in assuring that imported modules are still ' + 'available\n' + ' at the time when the "__del__()" method is called.\n' '\n' 'object.__repr__(self)\n' '\n' @@ -8120,19 +8202,21 @@ topics = {'assert': 'The "assert" statement\n' ' def __hash__(self):\n' ' return hash((self.name, self.nick, self.color))\n' '\n' - ' Note: "hash()" truncates the value returned from an ' - 'object’s\n' - ' custom "__hash__()" method to the size of a ' - '"Py_ssize_t". This\n' - ' is typically 8 bytes on 64-bit builds and 4 bytes on ' - '32-bit\n' - ' builds. If an object’s "__hash__()" must interoperate ' - 'on builds\n' - ' of different bit sizes, be sure to check the width on ' - 'all\n' - ' supported builds. An easy way to do this is with ' - '"python -c\n' - ' "import sys; print(sys.hash_info.width)"".\n' + ' Note:\n' + '\n' + ' "hash()" truncates the value returned from an object’s ' + 'custom\n' + ' "__hash__()" method to the size of a "Py_ssize_t". ' + 'This is\n' + ' typically 8 bytes on 64-bit builds and 4 bytes on ' + '32-bit builds.\n' + ' If an object’s "__hash__()" must interoperate on ' + 'builds of\n' + ' different bit sizes, be sure to check the width on all ' + 'supported\n' + ' builds. An easy way to do this is with "python -c ' + '"import sys;\n' + ' print(sys.hash_info.width)"".\n' '\n' ' If a class does not define an "__eq__()" method it should ' 'not\n' @@ -8188,10 +8272,12 @@ topics = {'assert': 'The "assert" statement\n' ' hashable by an "isinstance(obj, ' 'collections.abc.Hashable)" call.\n' '\n' - ' Note: By default, the "__hash__()" values of str, bytes ' - 'and\n' - ' datetime objects are “salted” with an unpredictable ' - 'random value.\n' + ' Note:\n' + '\n' + ' By default, the "__hash__()" values of str, bytes and ' + 'datetime\n' + ' objects are “salted” with an unpredictable random ' + 'value.\n' ' Although they remain constant within an individual ' 'Python\n' ' process, they are not predictable between repeated ' @@ -8291,11 +8377,13 @@ topics = {'assert': 'The "assert" statement\n' 'needs, for\n' ' example, "object.__getattribute__(self, name)".\n' '\n' - ' Note: This method may still be bypassed when looking up ' - 'special\n' - ' methods as the result of implicit invocation via ' - 'language syntax\n' - ' or built-in functions. See Special method lookup.\n' + ' Note:\n' + '\n' + ' This method may still be bypassed when looking up ' + 'special methods\n' + ' as the result of implicit invocation via language ' + 'syntax or\n' + ' built-in functions. See Special method lookup.\n' '\n' 'object.__setattr__(self, name, value)\n' '\n' @@ -8352,10 +8440,11 @@ topics = {'assert': 'The "assert" statement\n' 'returned.\n' '\n' 'The "__dir__" function should accept no arguments, and ' - 'return a list\n' - 'of strings that represents the names accessible on module. ' - 'If present,\n' - 'this function overrides the standard "dir()" search on a ' + 'return a\n' + 'sequence of strings that represents the names accessible on ' + 'module. If\n' + 'present, this function overrides the standard "dir()" search ' + 'on a\n' 'module.\n' '\n' 'For a more fine grained customization of the module behavior ' @@ -8378,15 +8467,16 @@ topics = {'assert': 'The "assert" statement\n' '\n' ' sys.modules[__name__].__class__ = VerboseModule\n' '\n' - 'Note: Defining module "__getattr__" and setting module ' - '"__class__"\n' - ' only affect lookups made using the attribute access syntax ' - '–\n' - ' directly accessing the module globals (whether by code ' - 'within the\n' - ' module, or via a reference to the module’s globals ' - 'dictionary) is\n' - ' unaffected.\n' + 'Note:\n' + '\n' + ' Defining module "__getattr__" and setting module ' + '"__class__" only\n' + ' affect lookups made using the attribute access syntax – ' + 'directly\n' + ' accessing the module globals (whether by code within the ' + 'module, or\n' + ' via a reference to the module’s globals dictionary) is ' + 'unaffected.\n' '\n' 'Changed in version 3.5: "__class__" module attribute is now ' 'writable.\n' @@ -8450,6 +8540,24 @@ topics = {'assert': 'The "assert" statement\n' 'The\n' ' descriptor has been assigned to *name*.\n' '\n' + ' Note:\n' + '\n' + ' "__set_name__()" is only called implicitly as part of ' + 'the "type"\n' + ' constructor, so it will need to be called explicitly ' + 'with the\n' + ' appropriate parameters when a descriptor is added to a ' + 'class\n' + ' after initial creation:\n' + '\n' + ' class A:\n' + ' pass\n' + ' descr = custom_descriptor()\n' + ' A.attr = descr\n' + " descr.__set_name__(A, 'attr')\n" + '\n' + ' See Creating the class object for more details.\n' + '\n' ' New in version 3.6.\n' '\n' 'The attribute "__objclass__" is interpreted by the "inspect" ' @@ -8597,10 +8705,9 @@ topics = {'assert': 'The "assert" statement\n' '~~~~~~~~~~~~~~~~~~~~~~~~~~\n' '\n' '* When inheriting from a class without *__slots__*, the ' - '*__dict__*\n' - ' and *__weakref__* attribute of the instances will always ' - 'be\n' - ' accessible.\n' + '*__dict__* and\n' + ' *__weakref__* attribute of the instances will always be ' + 'accessible.\n' '\n' '* Without a *__dict__* variable, instances cannot be ' 'assigned new\n' @@ -8614,13 +8721,12 @@ topics = {'assert': 'The "assert" statement\n' ' declaration.\n' '\n' '* Without a *__weakref__* variable for each instance, ' - 'classes\n' - ' defining *__slots__* do not support weak references to ' - 'its\n' - ' instances. If weak reference support is needed, then add\n' - ' "\'__weakref__\'" to the sequence of strings in the ' - '*__slots__*\n' - ' declaration.\n' + 'classes defining\n' + ' *__slots__* do not support weak references to its ' + 'instances. If weak\n' + ' reference support is needed, then add "\'__weakref__\'" to ' + 'the\n' + ' sequence of strings in the *__slots__* declaration.\n' '\n' '* *__slots__* are implemented at the class level by ' 'creating\n' @@ -8633,23 +8739,22 @@ topics = {'assert': 'The "assert" statement\n' ' attribute would overwrite the descriptor assignment.\n' '\n' '* The action of a *__slots__* declaration is not limited to ' - 'the\n' - ' class where it is defined. *__slots__* declared in ' - 'parents are\n' - ' available in child classes. However, child subclasses will ' - 'get a\n' - ' *__dict__* and *__weakref__* unless they also define ' - '*__slots__*\n' - ' (which should only contain names of any *additional* ' - 'slots).\n' + 'the class\n' + ' where it is defined. *__slots__* declared in parents are ' + 'available\n' + ' in child classes. However, child subclasses will get a ' + '*__dict__*\n' + ' and *__weakref__* unless they also define *__slots__* ' + '(which should\n' + ' only contain names of any *additional* slots).\n' '\n' '* If a class defines a slot also defined in a base class, ' - 'the\n' - ' instance variable defined by the base class slot is ' - 'inaccessible\n' - ' (except by retrieving its descriptor directly from the ' - 'base class).\n' - ' This renders the meaning of the program undefined. In the ' + 'the instance\n' + ' variable defined by the base class slot is inaccessible ' + '(except by\n' + ' retrieving its descriptor directly from the base class). ' + 'This\n' + ' renders the meaning of the program undefined. In the ' 'future, a\n' ' check may be added to prevent this.\n' '\n' @@ -8659,9 +8764,9 @@ topics = {'assert': 'The "assert" statement\n' 'and "tuple".\n' '\n' '* Any non-string iterable may be assigned to *__slots__*. ' - 'Mappings\n' - ' may also be used; however, in the future, special meaning ' - 'may be\n' + 'Mappings may\n' + ' also be used; however, in the future, special meaning may ' + 'be\n' ' assigned to the values corresponding to each key.\n' '\n' '* *__class__* assignment works only if both classes have the ' @@ -8676,6 +8781,12 @@ topics = {'assert': 'The "assert" statement\n' 'violations\n' ' raise "TypeError".\n' '\n' + '* If an iterator is used for *__slots__* then a descriptor ' + 'is created\n' + ' for each of the iterator’s values. However, the ' + '*__slots__*\n' + ' attribute will be an empty iterator.\n' + '\n' '\n' 'Customizing class creation\n' '==========================\n' @@ -8725,9 +8836,11 @@ topics = {'assert': 'The "assert" statement\n' 'does nothing,\n' ' but raises an error if it is called with any arguments.\n' '\n' - ' Note: The metaclass hint "metaclass" is consumed by the ' - 'rest of\n' - ' the type machinery, and is never passed to ' + ' Note:\n' + '\n' + ' The metaclass hint "metaclass" is consumed by the rest ' + 'of the\n' + ' type machinery, and is never passed to ' '"__init_subclass__"\n' ' implementations. The actual metaclass (rather than the ' 'explicit\n' @@ -8795,9 +8908,10 @@ topics = {'assert': 'The "assert" statement\n' 'tuple may\n' 'be empty, in such case the original base is ignored.\n' '\n' - 'See also: **PEP 560** - Core support for typing module and ' - 'generic\n' - ' types\n' + 'See also:\n' + '\n' + ' **PEP 560** - Core support for typing module and generic ' + 'types\n' '\n' '\n' 'Determining the appropriate metaclass\n' @@ -8844,7 +8958,13 @@ topics = {'assert': 'The "assert" statement\n' 'bases,\n' '**kwds)" (where the additional keyword arguments, if any, ' 'come from\n' - 'the class definition).\n' + 'the class definition). The "__prepare__" method should be ' + 'implemented\n' + 'as a "classmethod()". The namespace returned by ' + '"__prepare__" is\n' + 'passed in to "__new__", but when the final class object is ' + 'created the\n' + 'namespace is copied into a new "dict".\n' '\n' 'If the metaclass has no "__prepare__" attribute, then the ' 'class\n' @@ -9049,9 +9169,10 @@ topics = {'assert': 'The "assert" statement\n' 'type hints,\n' 'other usage is discouraged.\n' '\n' - 'See also: **PEP 560** - Core support for typing module and ' - 'generic\n' - ' types\n' + 'See also:\n' + '\n' + ' **PEP 560** - Core support for typing module and generic ' + 'types\n' '\n' '\n' 'Emulating callable objects\n' @@ -9121,9 +9242,9 @@ topics = {'assert': 'The "assert" statement\n' 'allow\n' 'efficient iteration through the container; for mappings, ' '"__iter__()"\n' - 'should be the same as "keys()"; for sequences, it should ' - 'iterate\n' - 'through the values.\n' + 'should iterate through the object’s keys; for sequences, it ' + 'should\n' + 'iterate through the values.\n' '\n' 'object.__len__(self)\n' '\n' @@ -9152,16 +9273,22 @@ topics = {'assert': 'The "assert" statement\n' ' estimated length for the object (which may be greater or ' 'less than\n' ' the actual length). The length must be an integer ">=" 0. ' - 'This\n' + 'The\n' + ' return value may also be "NotImplemented", which is ' + 'treated the\n' + ' same as if the "__length_hint__" method didn’t exist at ' + 'all. This\n' ' method is purely an optimization and is never required ' 'for\n' ' correctness.\n' '\n' ' New in version 3.4.\n' '\n' - 'Note: Slicing is done exclusively with the following three ' - 'methods.\n' - ' A call like\n' + 'Note:\n' + '\n' + ' Slicing is done exclusively with the following three ' + 'methods. A\n' + ' call like\n' '\n' ' a[1:2] = b\n' '\n' @@ -9192,8 +9319,10 @@ topics = {'assert': 'The "assert" statement\n' 'the\n' ' container), "KeyError" should be raised.\n' '\n' - ' Note: "for" loops expect that an "IndexError" will be ' - 'raised for\n' + ' Note:\n' + '\n' + ' "for" loops expect that an "IndexError" will be raised ' + 'for\n' ' illegal indexes to allow proper detection of the end of ' 'the\n' ' sequence.\n' @@ -9270,12 +9399,12 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'The membership test operators ("in" and "not in") are ' 'normally\n' - 'implemented as an iteration through a sequence. However, ' + 'implemented as an iteration through a container. However, ' 'container\n' 'objects can supply the following special method with a more ' 'efficient\n' - 'implementation, which also does not require the object be a ' - 'sequence.\n' + 'implementation, which also does not require the object be ' + 'iterable.\n' '\n' 'object.__contains__(self, item)\n' '\n' @@ -9354,7 +9483,7 @@ topics = {'assert': 'The "assert" statement\n' 'object.__rfloordiv__(self, other)\n' 'object.__rmod__(self, other)\n' 'object.__rdivmod__(self, other)\n' - 'object.__rpow__(self, other)\n' + 'object.__rpow__(self, other[, modulo])\n' 'object.__rlshift__(self, other)\n' 'object.__rrshift__(self, other)\n' 'object.__rand__(self, other)\n' @@ -9383,15 +9512,17 @@ topics = {'assert': 'The "assert" statement\n' '"__rpow__()" (the\n' ' coercion rules would become too complicated).\n' '\n' - ' Note: If the right operand’s type is a subclass of the ' - 'left\n' - ' operand’s type and that subclass provides the reflected ' - 'method\n' - ' for the operation, this method will be called before ' - 'the left\n' - ' operand’s non-reflected method. This behavior allows ' - 'subclasses\n' - ' to override their ancestors’ operations.\n' + ' Note:\n' + '\n' + ' If the right operand’s type is a subclass of the left ' + 'operand’s\n' + ' type and that subclass provides the reflected method ' + 'for the\n' + ' operation, this method will be called before the left ' + 'operand’s\n' + ' non-reflected method. This behavior allows subclasses ' + 'to\n' + ' override their ancestors’ operations.\n' '\n' 'object.__iadd__(self, other)\n' 'object.__isub__(self, other)\n' @@ -9465,8 +9596,9 @@ topics = {'assert': 'The "assert" statement\n' 'numeric\n' ' object is an integer type. Must return an integer.\n' '\n' - ' Note: In order to have a coherent integer type class, ' - 'when\n' + ' Note:\n' + '\n' + ' In order to have a coherent integer type class, when\n' ' "__index__()" is defined "__int__()" should also be ' 'defined, and\n' ' both should return the same value.\n' @@ -9790,11 +9922,13 @@ topics = {'assert': 'The "assert" statement\n' '"-1" if\n' ' *sub* is not found.\n' '\n' - ' Note: The "find()" method should be used only if you ' - 'need to know\n' - ' the position of *sub*. To check if *sub* is a ' - 'substring or not,\n' - ' use the "in" operator:\n' + ' Note:\n' + '\n' + ' The "find()" method should be used only if you need ' + 'to know the\n' + ' position of *sub*. To check if *sub* is a substring ' + 'or not, use\n' + ' the "in" operator:\n' '\n' " >>> 'Py' in 'Python'\n" ' True\n' @@ -9823,8 +9957,9 @@ topics = {'assert': 'The "assert" statement\n' ' formatting options that can be specified in format ' 'strings.\n' '\n' - ' Note: When formatting a number ("int", "float", ' - '"complex",\n' + ' Note:\n' + '\n' + ' When formatting a number ("int", "float", "complex",\n' ' "decimal.Decimal" and subclasses) with the "n" type ' '(ex:\n' ' "\'{:n}\'.format(1234)"), the function temporarily ' @@ -9871,20 +10006,20 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'str.isalnum()\n' '\n' - ' Return true if all characters in the string are ' + ' Return "True" if all characters in the string are ' 'alphanumeric and\n' - ' there is at least one character, false otherwise. A ' - 'character "c"\n' - ' is alphanumeric if one of the following returns ' + ' there is at least one character, "False" otherwise. A ' + 'character\n' + ' "c" is alphanumeric if one of the following returns ' '"True":\n' ' "c.isalpha()", "c.isdecimal()", "c.isdigit()", or ' '"c.isnumeric()".\n' '\n' 'str.isalpha()\n' '\n' - ' Return true if all characters in the string are ' + ' Return "True" if all characters in the string are ' 'alphabetic and\n' - ' there is at least one character, false otherwise. ' + ' there is at least one character, "False" otherwise. ' 'Alphabetic\n' ' characters are those characters defined in the Unicode ' 'character\n' @@ -9898,45 +10033,46 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'str.isascii()\n' '\n' - ' Return true if the string is empty or all characters in ' - 'the string\n' - ' are ASCII, false otherwise. ASCII characters have code ' - 'points in\n' - ' the range U+0000-U+007F.\n' + ' Return "True" if the string is empty or all characters ' + 'in the\n' + ' string are ASCII, "False" otherwise. ASCII characters ' + 'have code\n' + ' points in the range U+0000-U+007F.\n' '\n' ' New in version 3.7.\n' '\n' 'str.isdecimal()\n' '\n' - ' Return true if all characters in the string are decimal ' - 'characters\n' - ' and there is at least one character, false otherwise. ' - 'Decimal\n' - ' characters are those that can be used to form numbers ' - 'in base 10,\n' - ' e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a ' - 'decimal character\n' - ' is a character in the Unicode General Category “Nd”.\n' + ' Return "True" if all characters in the string are ' + 'decimal\n' + ' characters and there is at least one character, "False" ' + 'otherwise.\n' + ' Decimal characters are those that can be used to form ' + 'numbers in\n' + ' base 10, e.g. U+0660, ARABIC-INDIC DIGIT ZERO. ' + 'Formally a decimal\n' + ' character is a character in the Unicode General ' + 'Category “Nd”.\n' '\n' 'str.isdigit()\n' '\n' - ' Return true if all characters in the string are digits ' - 'and there is\n' - ' at least one character, false otherwise. Digits ' - 'include decimal\n' - ' characters and digits that need special handling, such ' - 'as the\n' - ' compatibility superscript digits. This covers digits ' - 'which cannot\n' - ' be used to form numbers in base 10, like the Kharosthi ' - 'numbers.\n' - ' Formally, a digit is a character that has the property ' - 'value\n' - ' Numeric_Type=Digit or Numeric_Type=Decimal.\n' + ' Return "True" if all characters in the string are ' + 'digits and there\n' + ' is at least one character, "False" otherwise. Digits ' + 'include\n' + ' decimal characters and digits that need special ' + 'handling, such as\n' + ' the compatibility superscript digits. This covers ' + 'digits which\n' + ' cannot be used to form numbers in base 10, like the ' + 'Kharosthi\n' + ' numbers. Formally, a digit is a character that has the ' + 'property\n' + ' value Numeric_Type=Digit or Numeric_Type=Decimal.\n' '\n' 'str.isidentifier()\n' '\n' - ' Return true if the string is a valid identifier ' + ' Return "True" if the string is a valid identifier ' 'according to the\n' ' language definition, section Identifiers and keywords.\n' '\n' @@ -9946,32 +10082,33 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'str.islower()\n' '\n' - ' Return true if all cased characters [4] in the string ' - 'are lowercase\n' - ' and there is at least one cased character, false ' - 'otherwise.\n' + ' Return "True" if all cased characters [4] in the string ' + 'are\n' + ' lowercase and there is at least one cased character, ' + '"False"\n' + ' otherwise.\n' '\n' 'str.isnumeric()\n' '\n' - ' Return true if all characters in the string are numeric ' - 'characters,\n' - ' and there is at least one character, false otherwise. ' - 'Numeric\n' - ' characters include digit characters, and all characters ' - 'that have\n' - ' the Unicode numeric value property, e.g. U+2155, VULGAR ' - 'FRACTION\n' - ' ONE FIFTH. Formally, numeric characters are those with ' - 'the\n' - ' property value Numeric_Type=Digit, Numeric_Type=Decimal ' - 'or\n' + ' Return "True" if all characters in the string are ' + 'numeric\n' + ' characters, and there is at least one character, ' + '"False" otherwise.\n' + ' Numeric characters include digit characters, and all ' + 'characters\n' + ' that have the Unicode numeric value property, e.g. ' + 'U+2155, VULGAR\n' + ' FRACTION ONE FIFTH. Formally, numeric characters are ' + 'those with\n' + ' the property value Numeric_Type=Digit, ' + 'Numeric_Type=Decimal or\n' ' Numeric_Type=Numeric.\n' '\n' 'str.isprintable()\n' '\n' - ' Return true if all characters in the string are ' + ' Return "True" if all characters in the string are ' 'printable or the\n' - ' string is empty, false otherwise. Nonprintable ' + ' string is empty, "False" otherwise. Nonprintable ' 'characters are\n' ' those characters defined in the Unicode character ' 'database as\n' @@ -9987,32 +10124,36 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'str.isspace()\n' '\n' - ' Return true if there are only whitespace characters in ' - 'the string\n' - ' and there is at least one character, false otherwise. ' - 'Whitespace\n' - ' characters are those characters defined in the Unicode ' - 'character\n' - ' database as “Other” or “Separator” and those with ' - 'bidirectional\n' - ' property being one of “WS”, “B”, or “S”.\n' + ' Return "True" if there are only whitespace characters ' + 'in the string\n' + ' and there is at least one character, "False" ' + 'otherwise.\n' + '\n' + ' A character is *whitespace* if in the Unicode character ' + 'database\n' + ' (see "unicodedata"), either its general category is ' + '"Zs"\n' + ' (“Separator, space”), or its bidirectional class is one ' + 'of "WS",\n' + ' "B", or "S".\n' '\n' 'str.istitle()\n' '\n' - ' Return true if the string is a titlecased string and ' + ' Return "True" if the string is a titlecased string and ' 'there is at\n' ' least one character, for example uppercase characters ' 'may only\n' ' follow uncased characters and lowercase characters only ' 'cased ones.\n' - ' Return false otherwise.\n' + ' Return "False" otherwise.\n' '\n' 'str.isupper()\n' '\n' - ' Return true if all cased characters [4] in the string ' - 'are uppercase\n' - ' and there is at least one cased character, false ' - 'otherwise.\n' + ' Return "True" if all cased characters [4] in the string ' + 'are\n' + ' uppercase and there is at least one cased character, ' + '"False"\n' + ' otherwise.\n' '\n' 'str.join(iterable)\n' '\n' @@ -10658,17 +10799,20 @@ topics = {'assert': 'The "assert" statement\n' '\n' '2. Unlike in Standard C, exactly two hex digits are required.\n' '\n' - '3. In a bytes literal, hexadecimal and octal escapes denote the\n' - ' byte with the given value. In a string literal, these escapes\n' - ' denote a Unicode character with the given value.\n' + '3. In a bytes literal, hexadecimal and octal escapes denote the ' + 'byte\n' + ' with the given value. In a string literal, these escapes ' + 'denote a\n' + ' Unicode character with the given value.\n' '\n' '4. Changed in version 3.3: Support for name aliases [1] has been\n' ' added.\n' '\n' '5. Exactly four hex digits are required.\n' '\n' - '6. Any Unicode character can be encoded this way. Exactly eight\n' - ' hex digits are required.\n' + '6. Any Unicode character can be encoded this way. Exactly eight ' + 'hex\n' + ' digits are required.\n' '\n' 'Unlike Standard C, all unrecognized escape sequences are left in ' 'the\n' @@ -11240,6 +11384,16 @@ topics = {'assert': 'The "assert" statement\n' ' then they can be used interchangeably to index the same\n' ' dictionary entry.\n' '\n' + ' Dictionaries preserve insertion order, meaning that keys will ' + 'be\n' + ' produced in the same order they were added sequentially over ' + 'the\n' + ' dictionary. Replacing an existing key does not change the ' + 'order,\n' + ' however removing a key and re-inserting it will add it to ' + 'the\n' + ' end instead of keeping its old place.\n' + '\n' ' Dictionaries are mutable; they can be created by the "{...}"\n' ' notation (see section Dictionary displays).\n' '\n' @@ -11248,6 +11402,13 @@ topics = {'assert': 'The "assert" statement\n' '"collections"\n' ' module.\n' '\n' + ' Changed in version 3.7: Dictionaries did not preserve ' + 'insertion\n' + ' order in versions of Python before 3.6. In CPython 3.6,\n' + ' insertion order was preserved, but it was considered an\n' + ' implementation detail at that time rather than a language\n' + ' guarantee.\n' + '\n' 'Callable types\n' ' These are the types to which the function call operation (see\n' ' section Calls) can be applied:\n' @@ -11774,10 +11935,9 @@ topics = {'assert': 'The "assert" statement\n' ' "co_lnotab" is a string encoding the mapping from bytecode\n' ' offsets to line numbers (for details see the source code of ' 'the\n' - ' interpreter); "co_stacksize" is the required stack size\n' - ' (including local variables); "co_flags" is an integer ' - 'encoding a\n' - ' number of flags for the interpreter.\n' + ' interpreter); "co_stacksize" is the required stack size;\n' + ' "co_flags" is an integer encoding a number of flags for the\n' + ' interpreter.\n' '\n' ' The following flag bits are defined for "co_flags": bit ' '"0x04"\n' @@ -12087,6 +12247,11 @@ topics = {'assert': 'The "assert" statement\n' 'therefore,\n' ' custom mapping types should support too):\n' '\n' + ' list(d)\n' + '\n' + ' Return a list of all the keys used in the dictionary ' + '*d*.\n' + '\n' ' len(d)\n' '\n' ' Return the number of items in the dictionary *d*.\n' @@ -12243,11 +12408,21 @@ topics = {'assert': 'The "assert" statement\n' 'the\n' ' documentation of view objects.\n' '\n' + ' An equality comparison between one "dict.values()" ' + 'view and\n' + ' another will always return "False". This also applies ' + 'when\n' + ' comparing "dict.values()" to itself:\n' + '\n' + " >>> d = {'a': 1}\n" + ' >>> d.values() == d.values()\n' + ' False\n' + '\n' ' Dictionaries compare equal if and only if they have the ' 'same "(key,\n' - ' value)" pairs. Order comparisons (‘<’, ‘<=’, ‘>=’, ‘>’) ' - 'raise\n' - ' "TypeError".\n' + ' value)" pairs (regardless of ordering). Order comparisons ' + '(‘<’,\n' + ' ‘<=’, ‘>=’, ‘>’) raise "TypeError".\n' '\n' ' Dictionaries preserve insertion order. Note that ' 'updating a key\n' @@ -12276,9 +12451,11 @@ topics = {'assert': 'The "assert" statement\n' 'detail of\n' ' CPython from 3.6.\n' '\n' - 'See also: "types.MappingProxyType" can be used to create a ' - 'read-only\n' - ' view of a "dict".\n' + 'See also:\n' + '\n' + ' "types.MappingProxyType" can be used to create a read-only ' + 'view of a\n' + ' "dict".\n' '\n' '\n' 'Dictionary view objects\n' @@ -12651,13 +12828,14 @@ topics = {'assert': 'The "assert" statement\n' '"None", it\n' ' is treated like "1".\n' '\n' - '6. Concatenating immutable sequences always results in a new\n' - ' object. This means that building up a sequence by repeated\n' - ' concatenation will have a quadratic runtime cost in the ' - 'total\n' - ' sequence length. To get a linear runtime cost, you must ' - 'switch to\n' - ' one of the alternatives below:\n' + '6. Concatenating immutable sequences always results in a new ' + 'object.\n' + ' This means that building up a sequence by repeated ' + 'concatenation\n' + ' will have a quadratic runtime cost in the total sequence ' + 'length.\n' + ' To get a linear runtime cost, you must switch to one of the\n' + ' alternatives below:\n' '\n' ' * if concatenating "str" objects, you can build a list and ' 'use\n' @@ -12675,24 +12853,25 @@ topics = {'assert': 'The "assert" statement\n' ' * for other types, investigate the relevant class ' 'documentation\n' '\n' - '7. Some sequence types (such as "range") only support item\n' - ' sequences that follow specific patterns, and hence don’t ' - 'support\n' - ' sequence concatenation or repetition.\n' - '\n' - '8. "index" raises "ValueError" when *x* is not found in *s*. ' - 'Not\n' - ' all implementations support passing the additional arguments ' - '*i*\n' - ' and *j*. These arguments allow efficient searching of ' - 'subsections\n' - ' of the sequence. Passing the extra arguments is roughly ' - 'equivalent\n' - ' to using "s[i:j].index(x)", only without copying any data and ' - 'with\n' - ' the returned index being relative to the start of the ' + '7. Some sequence types (such as "range") only support item ' + 'sequences\n' + ' that follow specific patterns, and hence don’t support ' 'sequence\n' - ' rather than the start of the slice.\n' + ' concatenation or repetition.\n' + '\n' + '8. "index" raises "ValueError" when *x* is not found in *s*. Not ' + 'all\n' + ' implementations support passing the additional arguments *i* ' + 'and\n' + ' *j*. These arguments allow efficient searching of subsections ' + 'of\n' + ' the sequence. Passing the extra arguments is roughly ' + 'equivalent to\n' + ' using "s[i:j].index(x)", only without copying any data and ' + 'with the\n' + ' returned index being relative to the start of the sequence ' + 'rather\n' + ' than the start of the slice.\n' '\n' '\n' 'Immutable Sequence Types\n' @@ -12820,17 +12999,17 @@ topics = {'assert': 'The "assert" statement\n' '1. *t* must have the same length as the slice it is replacing.\n' '\n' '2. The optional argument *i* defaults to "-1", so that by ' - 'default\n' - ' the last item is removed and returned.\n' + 'default the\n' + ' last item is removed and returned.\n' '\n' '3. "remove" raises "ValueError" when *x* is not found in *s*.\n' '\n' - '4. The "reverse()" method modifies the sequence in place for\n' - ' economy of space when reversing a large sequence. To remind ' - 'users\n' - ' that it operates by side effect, it does not return the ' - 'reversed\n' - ' sequence.\n' + '4. The "reverse()" method modifies the sequence in place for ' + 'economy\n' + ' of space when reversing a large sequence. To remind users ' + 'that it\n' + ' operates by side effect, it does not return the reversed ' + 'sequence.\n' '\n' '5. "clear()" and "copy()" are included for consistency with the\n' ' interfaces of mutable containers that don’t support slicing\n' @@ -12863,9 +13042,9 @@ topics = {'assert': 'The "assert" statement\n' ' * Using a pair of square brackets to denote the empty list: ' '"[]"\n' '\n' - ' * Using square brackets, separating items with commas: ' - '"[a]",\n' - ' "[a, b, c]"\n' + ' * Using square brackets, separating items with commas: "[a]", ' + '"[a,\n' + ' b, c]"\n' '\n' ' * Using a list comprehension: "[x for x in iterable]"\n' '\n' @@ -13164,9 +13343,9 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'See also:\n' '\n' - ' * The linspace recipe shows how to implement a lazy version ' - 'of\n' - ' range suitable for floating point applications.\n', + ' * The linspace recipe shows how to implement a lazy version of ' + 'range\n' + ' suitable for floating point applications.\n', 'typesseq-mutable': 'Mutable Sequence Types\n' '**********************\n' '\n' @@ -13277,19 +13456,18 @@ topics = {'assert': 'The "assert" statement\n' 'replacing.\n' '\n' '2. The optional argument *i* defaults to "-1", so that ' - 'by default\n' - ' the last item is removed and returned.\n' + 'by default the\n' + ' last item is removed and returned.\n' '\n' '3. "remove" raises "ValueError" when *x* is not found in ' '*s*.\n' '\n' '4. The "reverse()" method modifies the sequence in place ' - 'for\n' - ' economy of space when reversing a large sequence. To ' - 'remind users\n' - ' that it operates by side effect, it does not return ' - 'the reversed\n' - ' sequence.\n' + 'for economy\n' + ' of space when reversing a large sequence. To remind ' + 'users that it\n' + ' operates by side effect, it does not return the ' + 'reversed sequence.\n' '\n' '5. "clear()" and "copy()" are included for consistency ' 'with the\n' @@ -13368,19 +13546,23 @@ topics = {'assert': 'The "assert" statement\n' 'The execution of the "with" statement with one “item” proceeds as\n' 'follows:\n' '\n' - '1. The context expression (the expression given in the "with_item")\n' - ' is evaluated to obtain a context manager.\n' + '1. The context expression (the expression given in the "with_item") ' + 'is\n' + ' evaluated to obtain a context manager.\n' '\n' '2. The context manager’s "__exit__()" is loaded for later use.\n' '\n' '3. The context manager’s "__enter__()" method is invoked.\n' '\n' - '4. If a target was included in the "with" statement, the return\n' - ' value from "__enter__()" is assigned to it.\n' + '4. If a target was included in the "with" statement, the return ' + 'value\n' + ' from "__enter__()" is assigned to it.\n' + '\n' + ' Note:\n' '\n' - ' Note: The "with" statement guarantees that if the "__enter__()"\n' - ' method returns without an error, then "__exit__()" will always ' - 'be\n' + ' The "with" statement guarantees that if the "__enter__()" ' + 'method\n' + ' returns without an error, then "__exit__()" will always be\n' ' called. Thus, if an error occurs during the assignment to the\n' ' target list, it will be treated the same as an error occurring\n' ' within the suite would be. See step 6 below.\n' diff --git a/lib-python/3/re.py b/lib-python/3/re.py index 94d486579e..4798043ffc 100644 --- a/lib-python/3/re.py +++ b/lib-python/3/re.py @@ -44,7 +44,7 @@ The special characters are: "|" A|B, creates an RE that will match either A or B. (...) Matches the RE inside the parentheses. The contents can be retrieved or matched later in the string. - (?aiLmsux) Set the A, I, L, M, S, U, or X flag for the RE (see below). + (?aiLmsux) The letters set the corresponding flags defined below. (?:...) Non-grouping version of regular parentheses. (?P<name>...) The substring matched by the group is accessible by name. (?P=name) Matches the text matched earlier by the group named name. @@ -97,7 +97,9 @@ This module exports the following functions: purge Clear the regular expression cache. escape Backslash all non-alphanumerics in a string. -Some of the functions in this module takes flags as optional parameters: +Each function other than purge and escape can take an optional 'flags' argument +consisting of one or more of the following module constants, joined by "|". +A, L, and U are mutually exclusive. A ASCII For string patterns, make \w, \W, \b, \B, \d, \D match the corresponding ASCII character categories (rather than the whole Unicode categories, which is the diff --git a/lib-python/3/shlex.py b/lib-python/3/shlex.py index 2c9786c517..48e31f4799 100644 --- a/lib-python/3/shlex.py +++ b/lib-python/3/shlex.py @@ -55,7 +55,7 @@ class shlex: punctuation_chars = '' elif punctuation_chars is True: punctuation_chars = '();<>|&' - self.punctuation_chars = punctuation_chars + self._punctuation_chars = punctuation_chars if punctuation_chars: # _pushback_chars is a push back queue used by lookahead logic self._pushback_chars = deque() @@ -65,6 +65,10 @@ class shlex: t = self.wordchars.maketrans(dict.fromkeys(punctuation_chars)) self.wordchars = self.wordchars.translate(t) + @property + def punctuation_chars(self): + return self._punctuation_chars + def push_token(self, tok): "Push a token onto the stack popped by the get_token method" if self.debug >= 1: @@ -298,6 +302,7 @@ class shlex: return token def split(s, comments=False, posix=True): + """Split the string *s* using shell-like syntax.""" lex = shlex(s, posix=posix) lex.whitespace_split = True if not comments: diff --git a/lib-python/3/smtplib.py b/lib-python/3/smtplib.py index 6091c7fb7a..22d5097c69 100755 --- a/lib-python/3/smtplib.py +++ b/lib-python/3/smtplib.py @@ -54,7 +54,7 @@ import datetime import sys from email.base64mime import body_encode as encode_base64 -__all__ = ["SMTPException", "SMTPServerDisconnected", "SMTPResponseException", +__all__ = ["SMTPException", "SMTPNotSupportedError", "SMTPServerDisconnected", "SMTPResponseException", "SMTPSenderRefused", "SMTPRecipientsRefused", "SMTPDataError", "SMTPConnectError", "SMTPHeloError", "SMTPAuthenticationError", "quoteaddr", "quotedata", "SMTP"] diff --git a/lib-python/3/socket.py b/lib-python/3/socket.py index cfa605a22a..40c7636cef 100644 --- a/lib-python/3/socket.py +++ b/lib-python/3/socket.py @@ -724,7 +724,11 @@ def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT, sock.close() if err is not None: - raise err + try: + raise err + finally: + # Break explicitly a reference cycle + err = None else: raise error("getaddrinfo returns an empty list") diff --git a/lib-python/3/socketserver.py b/lib-python/3/socketserver.py index 905df9319e..1ad028fa4d 100644 --- a/lib-python/3/socketserver.py +++ b/lib-python/3/socketserver.py @@ -24,7 +24,7 @@ For request-based servers (including socket-based): The classes in this module favor the server type that is simplest to write: a synchronous TCP/IP server. This is bad class design, but -save some typing. (There's also the issue that a deep class hierarchy +saves some typing. (There's also the issue that a deep class hierarchy slows down method lookups.) There are five classes in an inheritance diagram, four of which represent diff --git a/lib-python/3/sqlite3/test/factory.py b/lib-python/3/sqlite3/test/factory.py index ced8445536..95dd24bdfa 100644 --- a/lib-python/3/sqlite3/test/factory.py +++ b/lib-python/3/sqlite3/test/factory.py @@ -98,16 +98,14 @@ class RowFactoryTests(unittest.TestCase): def CheckSqliteRowIndex(self): self.con.row_factory = sqlite.Row - row = self.con.execute("select 1 as a, 2 as b").fetchone() + row = self.con.execute("select 1 as a_1, 2 as b").fetchone() self.assertIsInstance(row, sqlite.Row) - col1, col2 = row["a"], row["b"] - self.assertEqual(col1, 1, "by name: wrong result for column 'a'") - self.assertEqual(col2, 2, "by name: wrong result for column 'a'") + self.assertEqual(row["a_1"], 1, "by name: wrong result for column 'a_1'") + self.assertEqual(row["b"], 2, "by name: wrong result for column 'b'") - col1, col2 = row["A"], row["B"] - self.assertEqual(col1, 1, "by name: wrong result for column 'A'") - self.assertEqual(col2, 2, "by name: wrong result for column 'B'") + self.assertEqual(row["A_1"], 1, "by name: wrong result for column 'A_1'") + self.assertEqual(row["B"], 2, "by name: wrong result for column 'B'") self.assertEqual(row[0], 1, "by index: wrong result for column 0") self.assertEqual(row[1], 2, "by index: wrong result for column 1") @@ -117,12 +115,25 @@ class RowFactoryTests(unittest.TestCase): with self.assertRaises(IndexError): row['c'] with self.assertRaises(IndexError): + row['a_\x11'] + with self.assertRaises(IndexError): + row['a\x7f1'] + with self.assertRaises(IndexError): row[2] with self.assertRaises(IndexError): row[-3] with self.assertRaises(IndexError): row[2**1000] + def CheckSqliteRowIndexUnicode(self): + self.con.row_factory = sqlite.Row + row = self.con.execute("select 1 as \xff").fetchone() + self.assertEqual(row["\xff"], 1) + with self.assertRaises(IndexError): + row['\u0178'] + with self.assertRaises(IndexError): + row['\xdf'] + def CheckSqliteRowSlice(self): # A sqlite.Row can be sliced like a list. self.con.row_factory = sqlite.Row @@ -169,19 +180,33 @@ class RowFactoryTests(unittest.TestCase): row_1 = self.con.execute("select 1 as a, 2 as b").fetchone() row_2 = self.con.execute("select 1 as a, 2 as b").fetchone() row_3 = self.con.execute("select 1 as a, 3 as b").fetchone() + row_4 = self.con.execute("select 1 as b, 2 as a").fetchone() + row_5 = self.con.execute("select 2 as b, 1 as a").fetchone() - self.assertEqual(row_1, row_1) - self.assertEqual(row_1, row_2) - self.assertTrue(row_2 != row_3) + self.assertTrue(row_1 == row_1) + self.assertTrue(row_1 == row_2) + self.assertFalse(row_1 == row_3) + self.assertFalse(row_1 == row_4) + self.assertFalse(row_1 == row_5) + self.assertFalse(row_1 == object()) self.assertFalse(row_1 != row_1) self.assertFalse(row_1 != row_2) - self.assertFalse(row_2 == row_3) + self.assertTrue(row_1 != row_3) + self.assertTrue(row_1 != row_4) + self.assertTrue(row_1 != row_5) + self.assertTrue(row_1 != object()) + + with self.assertRaises(TypeError): + row_1 > row_2 + with self.assertRaises(TypeError): + row_1 < row_2 + with self.assertRaises(TypeError): + row_1 >= row_2 + with self.assertRaises(TypeError): + row_1 <= row_2 - self.assertEqual(row_1, row_2) self.assertEqual(hash(row_1), hash(row_2)) - self.assertNotEqual(row_1, row_3) - self.assertNotEqual(hash(row_1), hash(row_3)) def CheckSqliteRowAsSequence(self): """ Checks if the row object can act like a sequence """ diff --git a/lib-python/3/sqlite3/test/regression.py b/lib-python/3/sqlite3/test/regression.py index 865bd88f74..25c58f562d 100644 --- a/lib-python/3/sqlite3/test/regression.py +++ b/lib-python/3/sqlite3/test/regression.py @@ -67,7 +67,7 @@ class RegressionTests(unittest.TestCase): def CheckColumnNameWithSpaces(self): cur = self.con.cursor() cur.execute('select 1 as "foo bar [datetime]"') - self.assertEqual(cur.description[0][0], "foo bar") + self.assertEqual(cur.description[0][0], "foo bar [datetime]") cur.execute('select 1 as "foo baz"') self.assertEqual(cur.description[0][0], "foo baz") diff --git a/lib-python/3/sqlite3/test/types.py b/lib-python/3/sqlite3/test/types.py index 6bc8d71def..77811212d4 100644 --- a/lib-python/3/sqlite3/test/types.py +++ b/lib-python/3/sqlite3/test/types.py @@ -253,13 +253,13 @@ class ColNamesTests(unittest.TestCase): def CheckColName(self): self.cur.execute("insert into test(x) values (?)", ("xxx",)) - self.cur.execute('select x as "x [bar]" from test') + self.cur.execute('select x as "x y [bar]" from test') val = self.cur.fetchone()[0] self.assertEqual(val, "<xxx>") # Check if the stripping of colnames works. Everything after the first - # whitespace should be stripped. - self.assertEqual(self.cur.description[0][0], "x") + # '[' (and the preceeding space) should be stripped. + self.assertEqual(self.cur.description[0][0], "x y") def CheckCaseInConverterName(self): self.cur.execute("select 'other' as \"x [b1b1]\"") diff --git a/lib-python/3/sre_parse.py b/lib-python/3/sre_parse.py index a53735b07d..cb2c4c3281 100644 --- a/lib-python/3/sre_parse.py +++ b/lib-python/3/sre_parse.py @@ -406,13 +406,7 @@ def _escape(source, escape, state): raise source.error("bad escape %s" % escape, len(escape)) def _uniq(items): - if len(set(items)) == len(items): - return items - newitems = [] - for item in items: - if item not in newitems: - newitems.append(item) - return newitems + return list(dict.fromkeys(items)) def _parse_sub(source, state, verbose, nested): # parse an alternation: a|b|c diff --git a/lib-python/3/stat.py b/lib-python/3/stat.py index 46837c06da..165e057576 100644 --- a/lib-python/3/stat.py +++ b/lib-python/3/stat.py @@ -40,6 +40,10 @@ S_IFREG = 0o100000 # regular file S_IFIFO = 0o010000 # fifo (named pipe) S_IFLNK = 0o120000 # symbolic link S_IFSOCK = 0o140000 # socket file +# Fallbacks for uncommon platform-specific constants +S_IFDOOR = 0 +S_IFPORT = 0 +S_IFWHT = 0 # Functions to test for each file type @@ -71,6 +75,18 @@ def S_ISSOCK(mode): """Return True if mode is from a socket.""" return S_IFMT(mode) == S_IFSOCK +def S_ISDOOR(mode): + """Return True if mode is from a door.""" + return False + +def S_ISPORT(mode): + """Return True if mode is from an event port.""" + return False + +def S_ISWHT(mode): + """Return True if mode is from a whiteout.""" + return False + # Names for permission bits S_ISUID = 0o4000 # set UID bit diff --git a/lib-python/3/subprocess.py b/lib-python/3/subprocess.py index 53a5e72099..3f99be551c 100644 --- a/lib-python/3/subprocess.py +++ b/lib-python/3/subprocess.py @@ -217,22 +217,38 @@ if _mswindows: __str__ = __repr__ -# This lists holds Popen instances for which the underlying process had not -# exited at the time its __del__ method got called: those processes are wait()ed -# for synchronously from _cleanup() when a new Popen object is created, to avoid -# zombie processes. -_active = [] - -def _cleanup(): - for inst in _active[:]: - res = inst._internal_poll(_deadstate=sys.maxsize) - if res is not None: - try: - _active.remove(inst) - except ValueError: - # This can happen if two threads create a new Popen instance. - # It's harmless that it was already removed, so ignore. - pass +if _mswindows: + # On Windows we just need to close `Popen._handle` when we no longer need + # it, so that the kernel can free it. `Popen._handle` gets closed + # implicitly when the `Popen` instance is finalized (see `Handle.__del__`, + # which is calling `CloseHandle` as requested in [1]), so there is nothing + # for `_cleanup` to do. + # + # [1] https://docs.microsoft.com/en-us/windows/desktop/ProcThread/ + # creating-processes + _active = None + + def _cleanup(): + pass +else: + # This lists holds Popen instances for which the underlying process had not + # exited at the time its __del__ method got called: those processes are + # wait()ed for synchronously from _cleanup() when a new Popen object is + # created, to avoid zombie processes. + _active = [] + + def _cleanup(): + if _active is None: + return + for inst in _active[:]: + res = inst._internal_poll(_deadstate=sys.maxsize) + if res is not None: + try: + _active.remove(inst) + except ValueError: + # This can happen if two threads create a new Popen instance. + # It's harmless that it was already removed, so ignore. + pass PIPE = -1 STDOUT = -2 @@ -378,7 +394,7 @@ def check_output(*popenargs, timeout=None, **kwargs): b'when in the course of barman events\n' By default, all communication is in bytes, and therefore any "input" - should be bytes, and the return value wil be bytes. If in text mode, + should be bytes, and the return value will be bytes. If in text mode, any "input" should be a string, and the return value will be a string decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. @@ -472,11 +488,20 @@ def run(*popenargs, with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) - except TimeoutExpired: + except TimeoutExpired as exc: process.kill() - stdout, stderr = process.communicate() - raise TimeoutExpired(process.args, timeout, output=stdout, - stderr=stderr) + if _mswindows: + # Windows accumulates the output in a single blocking + # read() call run on child threads, with the timeout + # being done in a join() on those threads. communicate() + # _after_ kill() is required to collect that and add it + # to the exception. + exc.stdout, exc.stderr = process.communicate() + else: + # POSIX _communicate already populated the output so + # far into the TimeoutExpired exception. + process.wait() + raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. @@ -974,12 +999,16 @@ class Popen(object): return endtime - _time() - def _check_timeout(self, endtime, orig_timeout): + def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, + skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" if endtime is None: return - if _time() > endtime: - raise TimeoutExpired(self.args, orig_timeout) + if skip_check_and_raise or _time() > endtime: + raise TimeoutExpired( + self.args, orig_timeout, + output=b''.join(stdout_seq) if stdout_seq else None, + stderr=b''.join(stderr_seq) if stderr_seq else None) def wait(self, timeout=None): @@ -1668,18 +1697,23 @@ class Popen(object): with _PopenSelector() as selector: if self.stdin and input: selector.register(self.stdin, selectors.EVENT_WRITE) - if self.stdout: + if self.stdout and not self.stdout.closed: selector.register(self.stdout, selectors.EVENT_READ) - if self.stderr: + if self.stderr and not self.stderr.closed: selector.register(self.stderr, selectors.EVENT_READ) while selector.get_map(): timeout = self._remaining_time(endtime) if timeout is not None and timeout < 0: - raise TimeoutExpired(self.args, orig_timeout) + self._check_timeout(endtime, orig_timeout, + stdout, stderr, + skip_check_and_raise=True) + raise RuntimeError( # Impossible :) + '_check_timeout(..., skip_check_and_raise=True) ' + 'failed to raise TimeoutExpired.') ready = selector.select(timeout) - self._check_timeout(endtime, orig_timeout) + self._check_timeout(endtime, orig_timeout, stdout, stderr) # XXX Rewrite these to use non-blocking I/O on the file # objects; they are no longer using C stdio! diff --git a/lib-python/3/symtable.py b/lib-python/3/symtable.py index c7627a6ef6..42ab725645 100644 --- a/lib-python/3/symtable.py +++ b/lib-python/3/symtable.py @@ -188,7 +188,7 @@ class Symbol(object): return bool(self.__scope == GLOBAL_EXPLICIT) def is_local(self): - return bool(self.__flags & DEF_BOUND) + return bool(self.__scope in (LOCAL, CELL)) def is_annotated(self): return bool(self.__flags & DEF_ANNOT) diff --git a/lib-python/3/tarfile.py b/lib-python/3/tarfile.py index edd31e96fb..3be5188c8b 100755 --- a/lib-python/3/tarfile.py +++ b/lib-python/3/tarfile.py @@ -1233,6 +1233,8 @@ class TarInfo(object): length, keyword = match.groups() length = int(length) + if length == 0: + raise InvalidHeaderError("invalid header") value = buf[match.end(2) + 1:match.start(1) + length - 1] # Normally, we could just use "utf-8" as the encoding and "strict" @@ -1629,13 +1631,12 @@ class TarFile(object): raise ValueError("mode must be 'r', 'w' or 'x'") try: - import gzip - gzip.GzipFile - except (ImportError, AttributeError): + from gzip import GzipFile + except ImportError: raise CompressionError("gzip module is not available") try: - fileobj = gzip.GzipFile(name, mode + "b", compresslevel, fileobj) + fileobj = GzipFile(name, mode + "b", compresslevel, fileobj) except OSError: if fileobj is not None and mode == 'r': raise ReadError("not a gzip file") @@ -1663,12 +1664,11 @@ class TarFile(object): raise ValueError("mode must be 'r', 'w' or 'x'") try: - import bz2 + from bz2 import BZ2File except ImportError: raise CompressionError("bz2 module is not available") - fileobj = bz2.BZ2File(fileobj or name, mode, - compresslevel=compresslevel) + fileobj = BZ2File(fileobj or name, mode, compresslevel=compresslevel) try: t = cls.taropen(name, mode, fileobj, **kwargs) @@ -1692,15 +1692,15 @@ class TarFile(object): raise ValueError("mode must be 'r', 'w' or 'x'") try: - import lzma + from lzma import LZMAFile, LZMAError except ImportError: raise CompressionError("lzma module is not available") - fileobj = lzma.LZMAFile(fileobj or name, mode, preset=preset) + fileobj = LZMAFile(fileobj or name, mode, preset=preset) try: t = cls.taropen(name, mode, fileobj, **kwargs) - except (lzma.LZMAError, EOFError): + except (LZMAError, EOFError): fileobj.close() if mode == 'r': raise ReadError("not an lzma file") diff --git a/lib-python/3/tempfile.py b/lib-python/3/tempfile.py index 2143224169..f92db53132 100644 --- a/lib-python/3/tempfile.py +++ b/lib-python/3/tempfile.py @@ -637,10 +637,8 @@ class SpooledTemporaryFile: if 'b' in mode: self._file = _io.BytesIO() else: - # Setting newline="\n" avoids newline translation; - # this is important because otherwise on Windows we'd - # get double newline translation upon rollover(). - self._file = _io.StringIO(newline="\n") + self._file = _io.TextIOWrapper(_io.BytesIO(), + encoding=encoding, newline=newline) self._max_size = max_size self._rolled = False self._TemporaryFileArgs = {'mode': mode, 'buffering': buffering, @@ -660,8 +658,12 @@ class SpooledTemporaryFile: newfile = self._file = TemporaryFile(**self._TemporaryFileArgs) del self._TemporaryFileArgs - newfile.write(file.getvalue()) - newfile.seek(file.tell(), 0) + pos = file.tell() + if hasattr(newfile, 'buffer'): + newfile.buffer.write(file.detach().getvalue()) + else: + newfile.write(file.getvalue()) + newfile.seek(pos, 0) self._rolled = True @@ -742,7 +744,7 @@ class SpooledTemporaryFile: return self._file.readlines(*args) def seek(self, *args): - self._file.seek(*args) + return self._file.seek(*args) @property def softspace(self): diff --git a/lib-python/3/test/_test_multiprocessing.py b/lib-python/3/test/_test_multiprocessing.py index fb65e5e1ab..23b9835a53 100644 --- a/lib-python/3/test/_test_multiprocessing.py +++ b/lib-python/3/test/_test_multiprocessing.py @@ -1057,8 +1057,7 @@ class _TestQueue(BaseTestCase): q = self.Queue() q.put(NotSerializable()) q.put(True) - # bpo-30595: use a timeout of 1 second for slow buildbots - self.assertTrue(q.get(timeout=1.0)) + self.assertTrue(q.get(timeout=TIMEOUT)) close_queue(q) with test.support.captured_stderr(): @@ -2690,16 +2689,17 @@ class _TestMyManager(BaseTestCase): self.common(manager) manager.shutdown() - # If the manager process exited cleanly then the exitcode - # will be zero. Otherwise (after a short timeout) - # terminate() is used, resulting in an exitcode of -SIGTERM. - self.assertEqual(manager._process.exitcode, 0) + # bpo-30356: BaseManager._finalize_manager() sends SIGTERM + # to the manager process if it takes longer than 1 second to stop, + # which happens on slow buildbots. + self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM)) def test_mymanager_context(self): with MyManager() as manager: self.common(manager) # bpo-30356: BaseManager._finalize_manager() sends SIGTERM - # to the manager process if it takes longer than 1 second to stop. + # to the manager process if it takes longer than 1 second to stop, + # which happens on slow buildbots. self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM)) def test_mymanager_context_prestarted(self): @@ -3140,6 +3140,19 @@ class _TestListener(BaseTestCase): if self.TYPE == 'processes': self.assertRaises(OSError, l.accept) + @unittest.skipUnless(util.abstract_sockets_supported, + "test needs abstract socket support") + def test_abstract_socket(self): + with self.connection.Listener("\0something") as listener: + with self.connection.Client(listener.address) as client: + with listener.accept() as d: + client.send(1729) + self.assertEqual(d.recv(), 1729) + + if self.TYPE == 'processes': + self.assertRaises(OSError, listener.accept) + + class _TestListenerClient(BaseTestCase): ALLOWED_TYPES = ('processes', 'threads') @@ -3411,6 +3424,16 @@ class _TestHeap(BaseTestCase): ALLOWED_TYPES = ('processes',) + def setUp(self): + super().setUp() + # Make pristine heap for these tests + self.old_heap = multiprocessing.heap.BufferWrapper._heap + multiprocessing.heap.BufferWrapper._heap = multiprocessing.heap.Heap() + + def tearDown(self): + multiprocessing.heap.BufferWrapper._heap = self.old_heap + super().tearDown() + def test_heap(self): iterations = 5000 maxblocks = 50 @@ -5036,8 +5059,8 @@ def install_tests_in_module_dict(remote_globs, start_method): # Sleep 500 ms to give time to child processes to complete. if need_sleep: time.sleep(0.5) - multiprocessing.process._cleanup() - test.support.gc_collect() + + multiprocessing.util._cleanup_tests() remote_globs['setUpModule'] = setUpModule remote_globs['tearDownModule'] = tearDownModule diff --git a/lib-python/3/test/ann_module.py b/lib-python/3/test/ann_module.py index 9e6b87dac4..0567d6de1b 100644 --- a/lib-python/3/test/ann_module.py +++ b/lib-python/3/test/ann_module.py @@ -6,6 +6,7 @@ Empty lines above are for good reason (testing for correct line numbers) """ from typing import Optional +from functools import wraps __annotations__[1] = 2 @@ -51,3 +52,9 @@ def foo(x: int = 10): def bar(y: List[str]): x: str = 'yes' bar() + +def dec(func): + @wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper diff --git a/lib-python/3/test/clinic.test b/lib-python/3/test/clinic.test index a113b942b0..75c9947256 100644 --- a/lib-python/3/test/clinic.test +++ b/lib-python/3/test/clinic.test @@ -1203,3 +1203,107 @@ static PyObject * test_Py_buffer_converter_impl(PyObject *module, Py_buffer *a, Py_buffer *b, Py_buffer *c, Py_buffer *d, Py_buffer *e) /*[clinic end generated code: output=92937215f10bc937 input=6a9da0f56f9525fd]*/ + +/*[clinic input] +output push +output preset buffer +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=5bff3376ee0df0b5]*/ + +#ifdef CONDITION_A +/*[clinic input] +test_preprocessor_guarded_condition_a +[clinic start generated code]*/ + +static PyObject * +test_preprocessor_guarded_condition_a_impl(PyObject *module) +/*[clinic end generated code: output=ad012af18085add6 input=8edb8706a98cda7e]*/ +#elif CONDITION_B +/*[clinic input] +test_preprocessor_guarded_elif_condition_b +[clinic start generated code]*/ + +static PyObject * +test_preprocessor_guarded_elif_condition_b_impl(PyObject *module) +/*[clinic end generated code: output=615f2dee82b138d1 input=53777cebbf7fee32]*/ +#else +/*[clinic input] +test_preprocessor_guarded_else +[clinic start generated code]*/ + +static PyObject * +test_preprocessor_guarded_else_impl(PyObject *module) +/*[clinic end generated code: output=13af7670aac51b12 input=6657ab31d74c29fc]*/ +#endif + +/*[clinic input] +dump buffer +output pop +[clinic start generated code]*/ + +#if defined(CONDITION_A) + +PyDoc_STRVAR(test_preprocessor_guarded_condition_a__doc__, +"test_preprocessor_guarded_condition_a($module, /)\n" +"--\n" +"\n"); + +#define TEST_PREPROCESSOR_GUARDED_CONDITION_A_METHODDEF \ + {"test_preprocessor_guarded_condition_a", (PyCFunction)test_preprocessor_guarded_condition_a, METH_NOARGS, test_preprocessor_guarded_condition_a__doc__}, + +static PyObject * +test_preprocessor_guarded_condition_a(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_preprocessor_guarded_condition_a_impl(module); +} + +#endif /* defined(CONDITION_A) */ + +#if !defined(CONDITION_A) && (CONDITION_B) + +PyDoc_STRVAR(test_preprocessor_guarded_elif_condition_b__doc__, +"test_preprocessor_guarded_elif_condition_b($module, /)\n" +"--\n" +"\n"); + +#define TEST_PREPROCESSOR_GUARDED_ELIF_CONDITION_B_METHODDEF \ + {"test_preprocessor_guarded_elif_condition_b", (PyCFunction)test_preprocessor_guarded_elif_condition_b, METH_NOARGS, test_preprocessor_guarded_elif_condition_b__doc__}, + +static PyObject * +test_preprocessor_guarded_elif_condition_b(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_preprocessor_guarded_elif_condition_b_impl(module); +} + +#endif /* !defined(CONDITION_A) && (CONDITION_B) */ + +#if !defined(CONDITION_A) && !(CONDITION_B) + +PyDoc_STRVAR(test_preprocessor_guarded_else__doc__, +"test_preprocessor_guarded_else($module, /)\n" +"--\n" +"\n"); + +#define TEST_PREPROCESSOR_GUARDED_ELSE_METHODDEF \ + {"test_preprocessor_guarded_else", (PyCFunction)test_preprocessor_guarded_else, METH_NOARGS, test_preprocessor_guarded_else__doc__}, + +static PyObject * +test_preprocessor_guarded_else(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return test_preprocessor_guarded_else_impl(module); +} + +#endif /* !defined(CONDITION_A) && !(CONDITION_B) */ + +#ifndef TEST_PREPROCESSOR_GUARDED_CONDITION_A_METHODDEF + #define TEST_PREPROCESSOR_GUARDED_CONDITION_A_METHODDEF +#endif /* !defined(TEST_PREPROCESSOR_GUARDED_CONDITION_A_METHODDEF) */ + +#ifndef TEST_PREPROCESSOR_GUARDED_ELIF_CONDITION_B_METHODDEF + #define TEST_PREPROCESSOR_GUARDED_ELIF_CONDITION_B_METHODDEF +#endif /* !defined(TEST_PREPROCESSOR_GUARDED_ELIF_CONDITION_B_METHODDEF) */ + +#ifndef TEST_PREPROCESSOR_GUARDED_ELSE_METHODDEF + #define TEST_PREPROCESSOR_GUARDED_ELSE_METHODDEF +#endif /* !defined(TEST_PREPROCESSOR_GUARDED_ELSE_METHODDEF) */ +/*[clinic end generated code: output=3804bb18d454038c input=3fc80c9989d2f2e1]*/ diff --git a/lib-python/3/test/dataclass_textanno.py b/lib-python/3/test/dataclass_textanno.py new file mode 100644 index 0000000000..3eb6c943d4 --- /dev/null +++ b/lib-python/3/test/dataclass_textanno.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import dataclasses + + +class Foo: + pass + + +@dataclasses.dataclass +class Bar: + foo: Foo diff --git a/lib-python/3/test/datetimetester.py b/lib-python/3/test/datetimetester.py index 2f8975d8c0..ba3722295d 100644 --- a/lib-python/3/test/datetimetester.py +++ b/lib-python/3/test/datetimetester.py @@ -2,11 +2,8 @@ See http://www.zope.org/Members/fdrake/DateTimeWiki/TestCases """ -from test.support import is_resource_enabled - import itertools import bisect - import copy import decimal import sys @@ -22,6 +19,7 @@ from array import array from operator import lt, le, gt, ge, eq, ne, truediv, floordiv, mod from test import support +from test.support import is_resource_enabled, ALWAYS_EQ, LARGEST, SMALLEST import datetime as datetime_module from datetime import MINYEAR, MAXYEAR @@ -53,6 +51,7 @@ OTHERSTUFF = (10, 34.5, "abc", {}, [], ()) INF = float("inf") NAN = float("nan") + ############################################################################# # module tests @@ -340,6 +339,18 @@ class TestTimeZone(unittest.TestCase): self.assertTrue(timezone(ZERO) != None) self.assertFalse(timezone(ZERO) == None) + tz = timezone(ZERO) + self.assertTrue(tz == ALWAYS_EQ) + self.assertFalse(tz != ALWAYS_EQ) + self.assertTrue(tz < LARGEST) + self.assertFalse(tz > LARGEST) + self.assertTrue(tz <= LARGEST) + self.assertFalse(tz >= LARGEST) + self.assertFalse(tz < SMALLEST) + self.assertTrue(tz > SMALLEST) + self.assertFalse(tz <= SMALLEST) + self.assertTrue(tz >= SMALLEST) + def test_aware_datetime(self): # test that timezone instances can be used by datetime t = datetime(1, 1, 1) @@ -377,6 +388,36 @@ class TestTimeZone(unittest.TestCase): tz_copy = copy.deepcopy(tz) self.assertIs(tz_copy, tz) + def test_offset_boundaries(self): + # Test timedeltas close to the boundaries + time_deltas = [ + timedelta(hours=23, minutes=59), + timedelta(hours=23, minutes=59, seconds=59), + timedelta(hours=23, minutes=59, seconds=59, microseconds=999999), + ] + time_deltas.extend([-delta for delta in time_deltas]) + + for delta in time_deltas: + with self.subTest(test_type='good', delta=delta): + timezone(delta) + + # Test timedeltas on and outside the boundaries + bad_time_deltas = [ + timedelta(hours=24), + timedelta(hours=24, microseconds=1), + ] + bad_time_deltas.extend([-delta for delta in bad_time_deltas]) + + for delta in bad_time_deltas: + with self.subTest(test_type='bad', delta=delta): + with self.assertRaises(ValueError): + timezone(delta) + + def test_comparison_with_tzinfo(self): + # Constructing tzinfo objects directly should not be done by users + # and serves only to check the bug described in bpo-37915 + self.assertNotEqual(timezone.utc, tzinfo()) + self.assertNotEqual(timezone(timedelta(hours=1)), tzinfo()) ############################################################################# # Base class for testing a particular aspect of timedelta, time, date and @@ -399,6 +440,24 @@ class HarmlessMixedComparison: self.assertIn(me, [1, 20, [], me]) self.assertIn([], [me, 1, 20, []]) + # Comparison to objects of unsupported types should return + # NotImplemented which falls back to the right hand side's __eq__ + # method. In this case, ALWAYS_EQ.__eq__ always returns True. + # ALWAYS_EQ.__ne__ always returns False. + self.assertTrue(me == ALWAYS_EQ) + self.assertFalse(me != ALWAYS_EQ) + + # If the other class explicitly defines ordering + # relative to our class, it is allowed to do so + self.assertTrue(me < LARGEST) + self.assertFalse(me > LARGEST) + self.assertTrue(me <= LARGEST) + self.assertFalse(me >= LARGEST) + self.assertFalse(me < SMALLEST) + self.assertTrue(me > SMALLEST) + self.assertFalse(me <= SMALLEST) + self.assertTrue(me >= SMALLEST) + def test_harmful_mixed_comparison(self): me = self.theclass(1, 1, 1) @@ -1352,15 +1411,20 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): t.strftime("%f") def test_strftime_trailing_percent(self): - # bpo-35066: make sure trailing '%' doesn't cause - # datetime's strftime to complain + # bpo-35066: Make sure trailing '%' doesn't cause datetime's strftime to + # complain. Different libcs have different handling of trailing + # percents, so we simply check datetime's strftime acts the same as + # time.strftime. t = self.theclass(2005, 3, 2) try: _time.strftime('%') except ValueError: self.skipTest('time module does not support trailing %') - self.assertEqual(t.strftime('%'), '%') - self.assertEqual(t.strftime("m:%m d:%d y:%y %"), "m:03 d:02 y:05 %") + self.assertEqual(t.strftime('%'), _time.strftime('%', t.timetuple())) + self.assertEqual( + t.strftime("m:%m d:%d y:%y %"), + _time.strftime("m:03 d:02 y:05 %", t.timetuple()), + ) def test_format(self): dt = self.theclass(2007, 9, 10) @@ -1524,29 +1588,6 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): self.assertRaises(TypeError, lambda: our < their) self.assertRaises(TypeError, lambda: their < our) - # However, if the other class explicitly defines ordering - # relative to our class, it is allowed to do so - - class LargerThanAnything: - def __lt__(self, other): - return False - def __le__(self, other): - return isinstance(other, LargerThanAnything) - def __eq__(self, other): - return isinstance(other, LargerThanAnything) - def __gt__(self, other): - return not isinstance(other, LargerThanAnything) - def __ge__(self, other): - return True - - their = LargerThanAnything() - self.assertEqual(our == their, False) - self.assertEqual(their == our, False) - self.assertEqual(our != their, True) - self.assertEqual(their != our, True) - self.assertEqual(our < their, True) - self.assertEqual(their < our, False) - def test_bool(self): # All dates are considered true. self.assertTrue(self.theclass.min) @@ -3149,16 +3190,25 @@ class TestTime(HarmlessMixedComparison, unittest.TestCase): def test_compat_unpickle(self): tests = [ - b"cdatetime\ntime\n(S'\\x14;\\x10\\x00\\x10\\x00'\ntR.", - b'cdatetime\ntime\n(U\x06\x14;\x10\x00\x10\x00tR.', - b'\x80\x02cdatetime\ntime\nU\x06\x14;\x10\x00\x10\x00\x85R.', + (b"cdatetime\ntime\n(S'\\x14;\\x10\\x00\\x10\\x00'\ntR.", + (20, 59, 16, 64**2)), + (b'cdatetime\ntime\n(U\x06\x14;\x10\x00\x10\x00tR.', + (20, 59, 16, 64**2)), + (b'\x80\x02cdatetime\ntime\nU\x06\x14;\x10\x00\x10\x00\x85R.', + (20, 59, 16, 64**2)), + (b"cdatetime\ntime\n(S'\\x14;\\x19\\x00\\x10\\x00'\ntR.", + (20, 59, 25, 64**2)), + (b'cdatetime\ntime\n(U\x06\x14;\x19\x00\x10\x00tR.', + (20, 59, 25, 64**2)), + (b'\x80\x02cdatetime\ntime\nU\x06\x14;\x19\x00\x10\x00\x85R.', + (20, 59, 25, 64**2)), ] - args = 20, 59, 16, 64**2 - expected = self.theclass(*args) - for data in tests: - for loads in pickle_loads: - derived = loads(data, encoding='latin1') - self.assertEqual(derived, expected) + for i, (data, args) in enumerate(tests): + with self.subTest(i=i): + expected = self.theclass(*args) + for loads in pickle_loads: + derived = loads(data, encoding='latin1') + self.assertEqual(derived, expected) def test_bool(self): # time is always True. @@ -3622,8 +3672,8 @@ class TestTimeTZ(TestTime, TZInfoBase, unittest.TestCase): self.assertRaises(ValueError, base.replace, microsecond=1000000) def test_mixed_compare(self): - t1 = time(1, 2, 3) - t2 = time(1, 2, 3) + t1 = self.theclass(1, 2, 3) + t2 = self.theclass(1, 2, 3) self.assertEqual(t1, t2) t2 = t2.replace(tzinfo=None) self.assertEqual(t1, t2) diff --git a/lib-python/3/test/keycert.passwd.pem b/lib-python/3/test/keycert.passwd.pem index cbb3c3bccd..c330c36d8f 100644 --- a/lib-python/3/test/keycert.passwd.pem +++ b/lib-python/3/test/keycert.passwd.pem @@ -1,45 +1,45 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,D134E931C96D9DEC - -nuGFEej7vIjkYWSMz5OJeVTNntDRQi6ZM4DBm3g8T7i/0odr3WFqGMMKZcIhLYQf -rgRq7RSKtrJ1y5taVucMV+EuCjyfzDo0TsYt+ZrXv/D08eZhjRmkhoHnGVF0TqQm -nQEXM/ERT4J2RM78dnG+homMkI76qOqxgGbRqQqJo6AiVRcAZ45y8s96bru2TAB8 -+pWjO/v0Je7AFVdwSU52N8OOY6uoSAygW+0UY1WVxbVGJF2XfRsNpPX+YQHYl6e+ -3xM5XBVCgr6kmdAyub5qUJ38X3TpdVGoR0i+CVS9GTr2pSRib1zURAeeHnlqiUZM -4m0Gn9s72nJevU1wxED8pwOhR8fnHEmMKGD2HPhKoOCbzDhwwBZO27TNa1uWeM3f -M5oixKDi2PqMn3y2cDx1NjJtP661688EcJ5a2Ih9BgO9xpnhSyzBWEKcAn0tJB0H -/56M0FW6cdOOIzMveGGL7sHW5E+iOdI1n5e7C6KJUzew78Y9qJnhS53EdI6qTz9R -wsIsj1i070Fk6RbPo6zpLlF6w7Zj8GlZaZA7OZZv9wo5VEV/0ST8gmiiBOBc4C6Y -u9hyLIIu4dFEBKyQHRvBnQSLNpKx6or1OGFDVBay2In9Yh2BHh1+vOj/OIz/wq48 -EHOIV27fRJxLu4jeK5LIGDhuPnMJ8AJYQ0bQOUP6fd7p+TxWkAQZPB/Dx/cs3hxr -nFEdzx+eO+IAsObx/b1EGZyEJyETBslu4GwYX7/KK3HsJhDJ1bdZ//28jOCaoir6 -ZOMT72GRwmVoQTJ0XpccfjHfKJDRLT7C1xvzo4Eibth0hpTZkA75IUYUp6qK/PuJ -kH/qdiC7QIkRKtsrawW4vEDna3YtxIYhQqz9+KwO6u/0gzooZtv1RU4U3ifMDB5u -5P5GAzACRqlY8QYBkM869lvWqzQPHvybC4ak9Yx6/heMO9ddjdIW9BaK8BLxvN/6 -UCD936Y4fWltt09jHZIoxWFykouBwmd7bXooNYXmDRNmjTdVhKJuOEOQw8hDzx7e -pWFJ9Z/V4Qm1tvXbCD7QFqMCDoY3qFvVG8DBqXpmxe1yPfz21FWrT7IuqDXAD3ns -vxfN/2a+Cy04U9FBNVCvWqWIs5AgNpdCMJC2FlXKTy+H3/7rIjNyFyvbX0vxIXtK -liOVNXiyVM++KZXqktqMUDlsJENmIHV9B046luqbgW018fHkyEYlL3iRZGbYegwr -XO9VVIKVPw1BEvJ8VNdGFGuZGepd8qX2ezfYADrNR+4t85HDm8inbjTobSjWuljs -ftUNkOeCHqAvWCFQTLCfdykvV08EJfVY79y7yFPtfRV2gxYokXFifjo3su9sVQr1 -UiIS5ZAsIC1hBXWeXoBN7QVTkFi7Yto6E1q2k10LiT3obpUUUQ/oclhrJOCJVjrS -oRcj2QBy8OT4T9slJr5maTWdgd7Lt6+I6cGQXPaDvjGOJl0eBYM14vhx4rRQWytJ -k07hhHFO4+9CGCuHS8AAy2gR6acYFWt2ZiiNZ0z/iPIHNK4YEyy9aLf6uZH/KQjE -jmHToo7XD6QvCAEC5qTHby3o3LfHIhyZi/4L+AhS4FKUHF6M0peeyYt4z3HaK2d2 -N6mHLPdjwNjra7GOmcns4gzcrdfoF+R293KpPal4PjknvR3dZL4kKP/ougTAM5zv -qDIvRbkHzjP8ChTpoLcJsNVXykNcNkjcSi0GHtIpYjh6QX6P2uvR/S4+Bbb9p9rn -hIy/ovu9tWN2hiPxGPe6torF6BulAxsTYlDercC204AyzsrdA0pr6HBgJH9C6ML1 -TchwodbFJqn9rSv91i1liusAGoOvE81AGBdrXY7LxfSNhYY1IK6yR/POJPTd53sA -uX2/j6Rtoksd/2BHPM6AUnI/2B9slhuzWX2aCtWLeuwvXDS6rYuTigaQmLkzTRfM -dlMI3s9KLXxgi5YVumUZleJWXwBNP7KiKajd+VTSD+7WAhyhM5FIG5wVOaxmy4G2 -TyqZ/Ax9d2VEjTQHWvQlLPQ4Mp0EIz0aEl94K/S8CK8bJRH6+PRkar+dJi1xqlL+ -BYb42At9mEJ8odLlFikvNi1+t7jqXk5jRi5C0xFKx3nTtzoH2zNUeuA3R6vSocVK -45jnze9IkKmxMlJ4loR5sgszdpDCD3kXqjtCcbMTmcrGyzJek3HSOTpiEORoTFOe -Rhg6jH5lm+QcC263oipojS0qEQcnsWJP2CylNYMYHR9O/9NQxT3o2lsRHqZTMELV -uQa/SFH+paQNbZOj8MRwPSqqiIxJFuLswKte1R+W7LKn1yBSM7Pp39lNbzGvJD2E -YRfnCwFpJ54voVAuQ4jXJvigCW2qeCjXlxeD6K2j4eGJEEOmIjIW1wjubyBY6OI3 ------END RSA PRIVATE KEY----- +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIhD+rJdxqb6ECAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDTdyjCP3riOSUfxix4aXEvBIIH +ECGkbsFabrcFMZcplw5jHMaOlG7rYjUzwDJ80JM8uzbv2Jb8SvNlns2+xmnEvH/M +mNvRmnXmplbVjH3XBMK8o2Psnr2V/a0j7/pgqpRxHykG+koOY4gzdt3MAg8JPbS2 +hymSl+Y5EpciO3xLfz4aFL1ZNqspQbO/TD13Ij7DUIy7xIRBMp4taoZCrP0cEBAZ ++wgu9m23I4dh3E8RUBzWyFFNic2MVVHrui6JbHc4dIHfyKLtXJDhUcS0vIC9PvcV +jhorh3UZC4lM+/jjXV5AhzQ0VrJ2tXAUX2dA144XHzkSH2QmwfnajPsci7BL2CGC +rjyTy4NfB/lDwU+55dqJZQSKXMxAapJMrtgw7LD5CKQcN6zmfhXGssJ7HQUXKkaX +I1YOFzuUD7oo56BVCnVswv0jX9RxrE5QYNreMlOP9cS+kIYH65N+PAhlURuQC14K +PgDkHn5knSa2UQA5tc5f7zdHOZhGRUfcjLP+KAWA3nh+/2OKw/X3zuPx75YT/FKe +tACPw5hjEpl62m9Xa0eWepZXwqkIOkzHMmCyNCsbC0mmRoEjmvfnslfsmnh4Dg/c +4YsTYMOLLIeCa+WIc38aA5W2lNO9lW0LwLhX1rP+GRVPv+TVHXlfoyaI+jp0iXrJ +t3xxT0gaiIR/VznyS7Py68QV/zB7VdqbsNzS7LdquHK1k8+7OYiWjY3gqyU40Iu2 +d1eSnIoDvQJwyYp7XYXbOlXNLY+s1Qb7yxcW3vXm0Bg3gKT8r1XHWJ9rj+CxAn5r +ysfkPs1JsesxzzQjwTiDNvHnBnZnwxuxfBr26ektEHmuAXSl8V6dzLN/aaPjpTj4 +CkE7KyqX3U9bLkp+ztl4xWKEmW44nskzm0+iqrtrxMyTfvvID4QrABjZL4zmWIqc +e3ZfA3AYk9VDIegk/YKGC5VZ8YS7ZXQ0ASK652XqJ7QlMKTxxV7zda6Fp4uW6/qN +ezt5wgbGGhZQXj2wDQmWNQYyG/juIgYTpCUA54U5XBIjuR6pg+Ytm0UrvNjsUoAC +wGelyqaLDq8U8jdIFYVTJy9aJjQOYXjsUJ0dZN2aGHSlju0ZGIZc49cTIVQ9BTC5 +Yc0Vlwzpl+LuA25DzKZNSb/ci0lO/cQGJ2uXQQgaNgdsHlu8nukENGJhnIzx4fzK +wEh3yHxhTRCzPPwDfXmx0IHXrPqJhSpAgaXBVIm8OjvmMxO+W75W4uLfNY/B7e2H +3cjklGuvkofOf7sEOrGUYf4cb6Obg8FpvHgpKo5Twwmoh/qvEKckBFqNhZXDDl88 +GbGlSEgyaAV1Ig8s1NJKBolWFa0juyPAwJ8vT1T4iwW7kQ7KXKt2UNn96K/HxkLu +pikvukz8oRHMlfVHa0R48UB1fFHwZLzPmwkpu6ancIxk3uO3yfhf6iDk3bmnyMlz +g3k/b6MrLYaOVByRxay85jH3Vvgqfgn6wa6BJ7xQ81eZ8B45gFuTH0J5JtLL7SH8 +darRPLCYfA+Ums9/H6pU5EXfd3yfjMIbvhCXHkJrrljkZ+th3p8dyto6wmYqIY6I +qR9sU+o6DhRaiP8tCICuhHxQpXylUM6WeJkJwduTJ8KWIvzsj4mReIKOl/oC2jSd +gIdKhb9Q3zj9ce4N5m6v66tyvjxGZ+xf3BvUPDD+LwZeXgf7OBsNVbXzQbzto594 +nbCzPocFi3gERE50ru4K70eQCy08TPG5NpOz+DDdO5vpAuMLYEuI7O3L+3GjW40Q +G5bu7H5/i7o/RWR67qhG/7p9kPw3nkUtYgnvnWaPMIuTfb4c2d069kjlfgWjIbbI +tpSKmm5DHlqTE4/ECAbIEDtSaw9dXHCdL3nh5+n428xDdGbjN4lT86tfu17EYKzl +ydH1RJ1LX3o3TEj9UkmDPt7LnftvwybMFEcP7hM2xD4lC++wKQs7Alg6dTkBnJV4 +5xU78WRntJkJTU7kFkpPKA0QfyCuSF1fAMoukDBkqUdOj6jE0BlJQlHk5iwgnJlt +uEdkTjHZEjIUxWC6llPcAzaPNlmnD45AgfEW+Jn21IvutmJiQAz5lm9Z9PXaR0C8 +hXB6owRY67C0YKQwXhoNf6xQun2xGBGYy5rPEEezX1S1tUH5GR/KW1Lh+FzFqHXI +ZEb5avfDqHKehGAjPON+Br7akuQ125M9LLjKuSyPaQzeeCAy356Xd7XzVwbPddbm +9S9WSPqzaPgh10chIHoNoC8HMd33dB5j9/Q6jrbU/oPlptu/GlorWblvJdcTuBGI +IVn45RFnkG8hCz0GJSNzW7+70YdESQbfJW79vssWMaiSjFE0pMyFXrFR5lBywBTx +PiGEUWtvrKG94X1TMlGUzDzDJOQNZ9dT94bonNe9pVmP5BP4/DzwwiWh6qrzWk6p +j8OE4cfCSh2WvHnhJbH7/N0v+JKjtxeIeJ16jx/K2oK5 +-----END ENCRYPTED PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIIEWTCCAsGgAwIBAgIJAJinz4jHSjLtMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u @@ -66,3 +66,4 @@ jMqTFlmO7kpf/jpCSmamp3/JSEE1BJKHwQ6Ql4nzRA2N1mnvWH7Zxcv043gkHeAu 9Wc2uXpw9xF8itV4Uvcdr3dwqByvIqn7iI/gB+4l41e0u8OmH2MKOx4Nxlly5TNW HcVKQHyOeyvnINuBAQ== -----END CERTIFICATE----- + diff --git a/lib-python/3/test/libregrtest/__init__.py b/lib-python/3/test/libregrtest/__init__.py index 3427b51b60..5e8dba5dbd 100644 --- a/lib-python/3/test/libregrtest/__init__.py +++ b/lib-python/3/test/libregrtest/__init__.py @@ -1,5 +1,2 @@ -# We import importlib *ASAP* in order to test #15386 -import importlib - from test.libregrtest.cmdline import _parse_args, RESOURCE_NAMES, ALL_RESOURCES from test.libregrtest.main import main diff --git a/lib-python/3/test/libregrtest/cmdline.py b/lib-python/3/test/libregrtest/cmdline.py index dc0d880719..6e24a8681b 100644 --- a/lib-python/3/test/libregrtest/cmdline.py +++ b/lib-python/3/test/libregrtest/cmdline.py @@ -207,10 +207,17 @@ def _create_parser(): group.add_argument('-m', '--match', metavar='PAT', dest='match_tests', action='append', help='match test cases and methods with glob pattern PAT') + group.add_argument('-i', '--ignore', metavar='PAT', + dest='ignore_tests', action='append', + help='ignore test cases and methods with glob pattern PAT') group.add_argument('--matchfile', metavar='FILENAME', dest='match_filename', help='similar to --match but get patterns from a ' 'text file, one pattern per line') + group.add_argument('--ignorefile', metavar='FILENAME', + dest='ignore_filename', + help='similar to --matchfile but it receives patterns ' + 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', help='fail as soon as a test fails (only with -v or -W)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', @@ -272,8 +279,10 @@ def _create_parser(): group.add_argument('--junit-xml', dest='xmlpath', metavar='FILENAME', help='writes JUnit-style XML results to the specified ' 'file') - group.add_argument('--tempdir', dest='tempdir', metavar='PATH', + group.add_argument('--tempdir', metavar='PATH', help='override the working directory for the test run') + group.add_argument('--cleanup', action='store_true', + help='remove old test_python_* directories') return parser @@ -313,7 +322,8 @@ def _parse_args(args, **kwargs): findleaks=1, use_resources=None, trace=False, coverdir='coverage', runleaks=False, huntrleaks=False, verbose2=False, print_slow=False, random_seed=None, use_mp=None, verbose3=False, forever=False, - header=False, failfast=False, match_tests=None, pgo=False) + header=False, failfast=False, match_tests=None, ignore_tests=None, + pgo=False) for k, v in kwargs.items(): if not hasattr(ns, k): raise TypeError('%r is an invalid keyword argument ' @@ -389,6 +399,12 @@ def _parse_args(args, **kwargs): with open(ns.match_filename) as fp: for line in fp: ns.match_tests.append(line.strip()) + if ns.ignore_filename: + if ns.ignore_tests is None: + ns.ignore_tests = [] + with open(ns.ignore_filename) as fp: + for line in fp: + ns.ignore_tests.append(line.strip()) if ns.forever: # --forever implies --failfast ns.failfast = True diff --git a/lib-python/3/test/libregrtest/main.py b/lib-python/3/test/libregrtest/main.py index a9b2b352d1..c521d530ce 100644 --- a/lib-python/3/test/libregrtest/main.py +++ b/lib-python/3/test/libregrtest/main.py @@ -14,13 +14,19 @@ from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( findtests, runtest, get_abs_module, STDTESTS, NOTTESTS, PASSED, FAILED, ENV_CHANGED, SKIPPED, RESOURCE_DENIED, - INTERRUPTED, CHILD_ERROR, TEST_DID_NOT_RUN, + INTERRUPTED, CHILD_ERROR, TEST_DID_NOT_RUN, TIMEOUT, PROGRESS_MIN_TIME, format_test_result, is_failed) from test.libregrtest.setup import setup_tests from test.libregrtest.utils import removepy, count, format_duration, printlist from test import support +# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()). +# Used to protect against threading._shutdown() hang. +# Must be smaller than buildbot "1200 seconds without output" limit. +EXIT_TIMEOUT = 120.0 + + class Regrtest: """Execute a test suite. @@ -114,6 +120,8 @@ class Regrtest: self.run_no_tests.append(test_name) elif ok == INTERRUPTED: self.interrupted = True + elif ok == TIMEOUT: + self.bad.append(test_name) else: raise ValueError("invalid test result: %r" % ok) @@ -130,16 +138,8 @@ class Regrtest: print(xml_data, file=sys.__stderr__) raise - def display_progress(self, test_index, text): - if self.ns.quiet: - return - - # "[ 51/405/1] test_tcl passed" - line = f"{test_index:{self.test_count_width}}{self.test_count}" - fails = len(self.bad) + len(self.environment_changed) - if fails and not self.ns.pgo: - line = f"{line}/{fails}" - line = f"[{line}] {text}" + def log(self, line=''): + empty = not line # add the system load prefix: "load avg: 1.80 " load_avg = self.getloadavg() @@ -150,16 +150,26 @@ class Regrtest: test_time = time.monotonic() - self.start_time test_time = datetime.timedelta(seconds=int(test_time)) line = f"{test_time} {line}" + + if empty: + line = line[:-1] + print(line, flush=True) + def display_progress(self, test_index, text): + if self.ns.quiet: + return + + # "[ 51/405/1] test_tcl passed" + line = f"{test_index:{self.test_count_width}}{self.test_count}" + fails = len(self.bad) + len(self.environment_changed) + if fails and not self.ns.pgo: + line = f"{line}/{fails}" + self.log(f"[{line}] {text}") + def parse_args(self, kwargs): ns = _parse_args(sys.argv[1:], **kwargs) - if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'): - print("Warning: The timeout option requires " - "faulthandler.dump_traceback_later", file=sys.stderr) - ns.timeout = None - if ns.xmlpath: support.junit_xml_list = self.testsuite_xml = [] @@ -173,7 +183,19 @@ class Regrtest: # Strip .py extensions. removepy(ns.args) - return ns + if ns.huntrleaks: + warmup, repetitions, _ = ns.huntrleaks + if warmup < 1 or repetitions < 1: + msg = ("Invalid values for the --huntrleaks/-R parameters. The " + "number of warmups and repetitions must be at least 1 " + "each (1:1).") + print(msg, file=sys.stderr, flush=True) + sys.exit(2) + + if ns.tempdir: + ns.tempdir = os.path.expanduser(ns.tempdir) + + self.ns = ns def find_tests(self, tests): self.tests = tests @@ -260,7 +282,7 @@ class Regrtest: def list_cases(self): support.verbose = False - support.set_match_tests(self.ns.match_tests) + support.set_match_tests(self.ns.match_tests, self.ns.ignore_tests) for test_name in self.selected: abstest = get_abs_module(self.ns, test_name) @@ -282,11 +304,11 @@ class Regrtest: self.first_result = self.get_tests_result() - print() - print("Re-running failed tests in verbose mode") + self.log() + self.log("Re-running failed tests in verbose mode") self.rerun = self.bad[:] for test_name in self.rerun: - print(f"Re-running {test_name} in verbose mode", flush=True) + self.log(f"Re-running {test_name} in verbose mode") self.ns.verbose = True result = runtest(self.ns, test_name) @@ -367,7 +389,10 @@ class Regrtest: save_modules = sys.modules.keys() - print("Run tests sequentially") + msg = "Run tests sequentially" + if self.ns.timeout: + msg += " (timeout: %s)" % format_duration(self.ns.timeout) + self.log(msg) previous_test = None for test_index, test_name in enumerate(self.tests, 1): @@ -488,10 +513,6 @@ class Regrtest: self.run_tests_sequential() def finalize(self): - if self.win_load_tracker is not None: - self.win_load_tracker.close() - self.win_load_tracker = None - if self.next_single_filename: if self.next_single_test: with open(self.next_single_filename, 'w') as fp: @@ -537,7 +558,7 @@ class Regrtest: for s in ET.tostringlist(root): f.write(s) - def create_temp_dir(self): + def set_temp_dir(self): if self.ns.tempdir: self.tmp_dir = self.ns.tempdir @@ -558,6 +579,8 @@ class Regrtest: self.tmp_dir = tempfile.gettempdir() self.tmp_dir = os.path.abspath(self.tmp_dir) + + def create_temp_dir(self): os.makedirs(self.tmp_dir, exist_ok=True) # Define a writable temp dir that will be used as cwd while running @@ -565,27 +588,54 @@ class Regrtest: # testing (see the -j option). pid = os.getpid() if self.worker_test_name is not None: - test_cwd = 'worker_{}'.format(pid) + test_cwd = 'test_python_worker_{}'.format(pid) else: test_cwd = 'test_python_{}'.format(pid) test_cwd = os.path.join(self.tmp_dir, test_cwd) return test_cwd + def cleanup(self): + import glob + + path = os.path.join(self.tmp_dir, 'test_python_*') + print("Cleanup %s directory" % self.tmp_dir) + for name in glob.glob(path): + if os.path.isdir(name): + print("Remove directory: %s" % name) + support.rmtree(name) + else: + print("Remove file: %s" % name) + support.unlink(name) + def main(self, tests=None, **kwargs): - self.ns = self.parse_args(kwargs) + self.parse_args(kwargs) + + self.set_temp_dir() + + if self.ns.cleanup: + self.cleanup() + sys.exit(0) test_cwd = self.create_temp_dir() - # Run the tests in a context manager that temporarily changes the CWD - # to a temporary and writable directory. If it's not possible to - # create or change the CWD, the original CWD will be used. - # The original CWD is available from support.SAVEDCWD. - with support.temp_cwd(test_cwd, quiet=True): - # When using multiprocessing, worker processes will use test_cwd - # as their parent temporary directory. So when the main process - # exit, it removes also subdirectories of worker processes. - self.ns.tempdir = test_cwd - self._main(tests, kwargs) + try: + # Run the tests in a context manager that temporarily changes the CWD + # to a temporary and writable directory. If it's not possible to + # create or change the CWD, the original CWD will be used. + # The original CWD is available from support.SAVEDCWD. + with support.temp_cwd(test_cwd, quiet=True): + # When using multiprocessing, worker processes will use test_cwd + # as their parent temporary directory. So when the main process + # exit, it removes also subdirectories of worker processes. + self.ns.tempdir = test_cwd + + self._main(tests, kwargs) + except SystemExit as exc: + # bpo-38203: Python can hang at exit in Py_Finalize(), especially + # on threading._shutdown() call: put a timeout + faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True) + + sys.exit(exc.code) def getloadavg(self): if self.win_load_tracker is not None: @@ -597,15 +647,6 @@ class Regrtest: return None def _main(self, tests, kwargs): - if self.ns.huntrleaks: - warmup, repetitions, _ = self.ns.huntrleaks - if warmup < 1 or repetitions < 1: - msg = ("Invalid values for the --huntrleaks/-R parameters. The " - "number of warmups and repetitions must be at least 1 " - "each (1:1).") - print(msg, file=sys.stderr, flush=True) - sys.exit(2) - if self.worker_test_name is not None: from test.libregrtest.runtest_mp import run_tests_worker run_tests_worker(self.ns, self.worker_test_name) @@ -639,11 +680,16 @@ class Regrtest: # typeperf.exe for x64, x86 or ARM print(f'Failed to create WindowsLoadTracker: {error}') - self.run_tests() - self.display_result() + try: + self.run_tests() + self.display_result() - if self.ns.verbose2 and self.bad: - self.rerun_failed_tests() + if self.ns.verbose2 and self.bad: + self.rerun_failed_tests() + finally: + if self.win_load_tracker is not None: + self.win_load_tracker.close() + self.win_load_tracker = None self.finalize() diff --git a/lib-python/3/test/libregrtest/runtest.py b/lib-python/3/test/libregrtest/runtest.py index a43b7666cd..558f2099c6 100644 --- a/lib-python/3/test/libregrtest/runtest.py +++ b/lib-python/3/test/libregrtest/runtest.py @@ -13,7 +13,7 @@ import unittest from test import support from test.libregrtest.refleak import dash_R, clear_caches from test.libregrtest.save_env import saved_test_environment -from test.libregrtest.utils import print_warning +from test.libregrtest.utils import format_duration, print_warning # Test result constants. @@ -25,6 +25,7 @@ RESOURCE_DENIED = -3 INTERRUPTED = -4 CHILD_ERROR = -5 # error in a child process TEST_DID_NOT_RUN = -6 +TIMEOUT = -7 _FORMAT_TEST_RESULT = { PASSED: '%s passed', @@ -35,6 +36,7 @@ _FORMAT_TEST_RESULT = { INTERRUPTED: '%s interrupted', CHILD_ERROR: '%s crashed', TEST_DID_NOT_RUN: '%s run no tests', + TIMEOUT: '%s timed out', } # Minimum duration of a test to display its duration or to mention that @@ -75,7 +77,10 @@ def is_failed(result, ns): def format_test_result(result): fmt = _FORMAT_TEST_RESULT.get(result.result, "%s") - return fmt % result.test_name + text = fmt % result.test_name + if result.result == TIMEOUT: + text = '%s (%s)' % (text, format_duration(result.test_time)) + return text def findtestdir(path=None): @@ -118,7 +123,7 @@ def _runtest(ns, test_name): start_time = time.perf_counter() try: - support.set_match_tests(ns.match_tests) + support.set_match_tests(ns.match_tests, ns.ignore_tests) support.junit_xml_list = xml_list = [] if ns.xmlpath else None if ns.failfast: support.failfast = True @@ -179,6 +184,7 @@ def runtest(ns, test_name): FAILED test failed PASSED test passed EMPTY_TEST_SUITE test ran no subtests. + TIMEOUT test timed out. If ns.xmlpath is not None, xml_data is a list containing each generated testsuite element. @@ -307,9 +313,7 @@ def cleanup_test_droppings(test_name, verbose): # since if a test leaves a file open, it cannot be deleted by name (while # there's nothing we can do about that here either, we can display the # name of the offending test, which is a real help). - for name in (support.TESTFN, - "db_home", - ): + for name in (support.TESTFN,): if not os.path.exists(name): continue diff --git a/lib-python/3/test/libregrtest/runtest_mp.py b/lib-python/3/test/libregrtest/runtest_mp.py index aa2409b4ef..7a18e45434 100644 --- a/lib-python/3/test/libregrtest/runtest_mp.py +++ b/lib-python/3/test/libregrtest/runtest_mp.py @@ -3,6 +3,7 @@ import faulthandler import json import os import queue +import signal import subprocess import sys import threading @@ -13,17 +14,26 @@ from test import support from test.libregrtest.runtest import ( runtest, INTERRUPTED, CHILD_ERROR, PROGRESS_MIN_TIME, - format_test_result, TestResult, is_failed) + format_test_result, TestResult, is_failed, TIMEOUT) from test.libregrtest.setup import setup_tests -from test.libregrtest.utils import format_duration +from test.libregrtest.utils import format_duration, print_warning # Display the running tests if nothing happened last N seconds PROGRESS_UPDATE = 30.0 # seconds +assert PROGRESS_UPDATE >= PROGRESS_MIN_TIME + +# Kill the main process after 5 minutes. It is supposed to write an update +# every PROGRESS_UPDATE seconds. Tolerate 5 minutes for Python slowest +# buildbot workers. +MAIN_PROCESS_TIMEOUT = 5 * 60.0 +assert MAIN_PROCESS_TIMEOUT >= PROGRESS_UPDATE # Time to wait until a worker completes: should be immediate JOIN_TIMEOUT = 30.0 # seconds +USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) + def must_stop(result, ns): if result.result == INTERRUPTED: @@ -52,12 +62,16 @@ def run_test_in_subprocess(testname, ns): # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. + kw = {} + if USE_PROCESS_GROUP: + kw['start_new_session'] = True return subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, close_fds=(os.name != 'nt'), - cwd=support.SAVEDCWD) + cwd=support.SAVEDCWD, + **kw) def run_tests_worker(ns, test_name): @@ -102,77 +116,140 @@ class ExitThread(Exception): pass -class MultiprocessThread(threading.Thread): - def __init__(self, pending, output, ns): +class TestWorkerProcess(threading.Thread): + def __init__(self, worker_id, runner): super().__init__() - self.pending = pending - self.output = output - self.ns = ns + self.worker_id = worker_id + self.pending = runner.pending + self.output = runner.output + self.ns = runner.ns + self.timeout = runner.worker_timeout + self.regrtest = runner.regrtest self.current_test_name = None self.start_time = None self._popen = None self._killed = False + self._stopped = False def __repr__(self): - info = ['MultiprocessThread'] - test = self.current_test_name + info = [f'TestWorkerProcess #{self.worker_id}'] if self.is_alive(): - info.append('alive') + info.append("running") + else: + info.append('stopped') + test = self.current_test_name if test: info.append(f'test={test}') popen = self._popen - if popen: - info.append(f'pid={popen.pid}') + if popen is not None: + dt = time.monotonic() - self.start_time + info.extend((f'pid={self._popen.pid}', + f'time={format_duration(dt)}')) return '<%s>' % ' '.join(info) - def kill(self): - self._killed = True - + def _kill(self): popen = self._popen if popen is None: return - popen.kill() - # stdout and stderr must be closed to ensure that communicate() - # does not hang - popen.stdout.close() - popen.stderr.close() - def _runtest(self, test_name): + if self._killed: + return + self._killed = True + + if USE_PROCESS_GROUP: + what = f"{self} process group" + else: + what = f"{self}" + + print(f"Kill {what}", file=sys.stderr, flush=True) try: - self.start_time = time.monotonic() - self.current_test_name = test_name + if USE_PROCESS_GROUP: + os.killpg(popen.pid, signal.SIGKILL) + else: + popen.kill() + except ProcessLookupError: + # popen.kill(): the process completed, the TestWorkerProcess thread + # read its exit status, but Popen.send_signal() read the returncode + # just before Popen.wait() set returncode. + pass + except OSError as exc: + print_warning(f"Failed to kill {what}: {exc!r}") - self._popen = run_test_in_subprocess(test_name, self.ns) - popen = self._popen - with popen: - try: - if self._killed: - # If kill() has been called before self._popen is set, - # self._popen is still running. Call again kill() - # to ensure that the process is killed. - self.kill() - raise ExitThread - - try: - stdout, stderr = popen.communicate() - except OSError: - if self._killed: - # kill() has been called: communicate() fails - # on reading closed stdout/stderr - raise ExitThread - raise - except: - self.kill() - popen.wait() - raise - - retcode = popen.wait() - finally: + def stop(self): + # Method called from a different thread to stop this thread + self._stopped = True + self._kill() + + def mp_result_error(self, test_name, error_type, stdout='', stderr='', + err_msg=None): + test_time = time.monotonic() - self.start_time + result = TestResult(test_name, error_type, test_time, None) + return MultiprocessResult(result, stdout, stderr, err_msg) + + def _run_process(self, test_name): + self.start_time = time.monotonic() + + self.current_test_name = test_name + try: + popen = run_test_in_subprocess(test_name, self.ns) + + self._killed = False + self._popen = popen + except: self.current_test_name = None + raise + + try: + if self._stopped: + # If kill() has been called before self._popen is set, + # self._popen is still running. Call again kill() + # to ensure that the process is killed. + self._kill() + raise ExitThread + + try: + stdout, stderr = popen.communicate(timeout=self.timeout) + retcode = popen.returncode + assert retcode is not None + except subprocess.TimeoutExpired: + if self._stopped: + # kill() has been called: communicate() fails + # on reading closed stdout/stderr + raise ExitThread + + # On timeout, kill the process + self._kill() + + # None means TIMEOUT for the caller + retcode = None + # bpo-38207: Don't attempt to call communicate() again: on it + # can hang until all child processes using stdout and stderr + # pipes completes. + stdout = stderr = '' + except OSError: + if self._stopped: + # kill() has been called: communicate() fails + # on reading closed stdout/stderr + raise ExitThread + raise + else: + stdout = stdout.strip() + stderr = stderr.rstrip() + + return (retcode, stdout, stderr) + except: + self._kill() + raise + finally: + self._wait_completed() self._popen = None + self.current_test_name = None + + def _runtest(self, test_name): + retcode, stdout, stderr = self._run_process(test_name) - stdout = stdout.strip() - stderr = stderr.rstrip() + if retcode is None: + return self.mp_result_error(test_name, TIMEOUT, stdout, stderr) err_msg = None if retcode != 0: @@ -191,13 +268,13 @@ class MultiprocessThread(threading.Thread): err_msg = "Failed to parse worker JSON: %s" % exc if err_msg is not None: - test_time = time.monotonic() - self.start_time - result = TestResult(test_name, CHILD_ERROR, test_time, None) + return self.mp_result_error(test_name, CHILD_ERROR, + stdout, stderr, err_msg) return MultiprocessResult(result, stdout, stderr, err_msg) def run(self): - while not self._killed: + while not self._stopped: try: try: test_name = next(self.pending) @@ -215,6 +292,44 @@ class MultiprocessThread(threading.Thread): self.output.put((True, traceback.format_exc())) break + def _wait_completed(self): + popen = self._popen + + # stdout and stderr must be closed to ensure that communicate() + # does not hang + popen.stdout.close() + popen.stderr.close() + + try: + popen.wait(JOIN_TIMEOUT) + except (subprocess.TimeoutExpired, OSError) as exc: + print_warning(f"Failed to wait for {self} completion " + f"(timeout={format_duration(JOIN_TIMEOUT)}): " + f"{exc!r}") + + def wait_stopped(self, start_time): + # bpo-38207: MultiprocessTestRunner.stop_workers() called self.stop() + # which killed the process. Sometimes, killing the process from the + # main thread does not interrupt popen.communicate() in + # TestWorkerProcess thread. This loop with a timeout is a workaround + # for that. + # + # Moreover, if this method fails to join the thread, it is likely + # that Python will hang at exit while calling threading._shutdown() + # which tries again to join the blocked thread. Regrtest.main() + # uses EXIT_TIMEOUT to workaround this second bug. + while True: + # Write a message every second + self.join(1.0) + if not self.is_alive(): + break + dt = time.monotonic() - start_time + self.regrtest.log(f"Waiting for {self} thread " + f"for {format_duration(dt)}") + if dt > JOIN_TIMEOUT: + print_warning(f"Failed to join {self} in {format_duration(dt)}") + break + def get_running(workers): running = [] @@ -229,41 +344,41 @@ def get_running(workers): return running -class MultiprocessRunner: +class MultiprocessTestRunner: def __init__(self, regrtest): self.regrtest = regrtest + self.log = self.regrtest.log self.ns = regrtest.ns self.output = queue.Queue() self.pending = MultiprocessIterator(self.regrtest.tests) if self.ns.timeout is not None: - self.test_timeout = self.ns.timeout * 1.5 + # Rely on faulthandler to kill a worker process. This timouet is + # when faulthandler fails to kill a worker process. Give a maximum + # of 5 minutes to faulthandler to kill the worker. + self.worker_timeout = min(self.ns.timeout * 1.5, + self.ns.timeout + 5 * 60) else: - self.test_timeout = None + self.worker_timeout = None self.workers = None def start_workers(self): - self.workers = [MultiprocessThread(self.pending, self.output, self.ns) - for _ in range(self.ns.use_mp)] - print("Run tests in parallel using %s child processes" - % len(self.workers)) + self.workers = [TestWorkerProcess(index, self) + for index in range(1, self.ns.use_mp + 1)] + msg = f"Run tests in parallel using {len(self.workers)} child processes" + if self.ns.timeout: + msg += (" (timeout: %s, worker timeout: %s)" + % (format_duration(self.ns.timeout), + format_duration(self.worker_timeout))) + self.log(msg) for worker in self.workers: worker.start() - def wait_workers(self): + def stop_workers(self): start_time = time.monotonic() for worker in self.workers: - worker.kill() + worker.stop() for worker in self.workers: - while True: - worker.join(1.0) - if not worker.is_alive(): - break - dt = time.monotonic() - start_time - print("Wait for regrtest worker %r for %.1f sec" % (worker, dt)) - if dt > JOIN_TIMEOUT: - print("Warning -- failed to join a regrtest worker %s" - % worker) - break + worker.wait_stopped(start_time) def _get_result(self): if not any(worker.is_alive() for worker in self.workers): @@ -273,12 +388,14 @@ class MultiprocessRunner: except queue.Empty: return None + use_faulthandler = (self.ns.timeout is not None) + timeout = PROGRESS_UPDATE while True: - if self.test_timeout is not None: - faulthandler.dump_traceback_later(self.test_timeout, exit=True) + if use_faulthandler: + faulthandler.dump_traceback_later(MAIN_PROCESS_TIMEOUT, + exit=True) # wait for a thread - timeout = max(PROGRESS_UPDATE, PROGRESS_MIN_TIME) try: return self.output.get(timeout=timeout) except queue.Empty: @@ -287,7 +404,7 @@ class MultiprocessRunner: # display progress running = get_running(self.workers) if running and not self.ns.pgo: - print('running: %s' % ', '.join(running), flush=True) + self.log('running: %s' % ', '.join(running)) def display_result(self, mp_result): result = mp_result.result @@ -307,8 +424,7 @@ class MultiprocessRunner: if item[0]: # Thread got an exception format_exc = item[1] - print(f"regrtest worker thread failed: {format_exc}", - file=sys.stderr, flush=True) + print_warning(f"regrtest worker thread failed: {format_exc}") return True self.test_index += 1 @@ -343,13 +459,14 @@ class MultiprocessRunner: print() self.regrtest.interrupted = True finally: - if self.test_timeout is not None: + if self.ns.timeout is not None: faulthandler.cancel_dump_traceback_later() - # a test failed (and --failfast is set) or all tests completed - self.pending.stop() - self.wait_workers() + # Always ensure that all worker processes are no longer + # worker when we exit this function + self.pending.stop() + self.stop_workers() def run_tests_multiprocess(regrtest): - MultiprocessRunner(regrtest).run_tests() + MultiprocessTestRunner(regrtest).run_tests() diff --git a/lib-python/3/test/libregrtest/setup.py b/lib-python/3/test/libregrtest/setup.py index 4362e92fbd..ea7f2c2f18 100644 --- a/lib-python/3/test/libregrtest/setup.py +++ b/lib-python/3/test/libregrtest/setup.py @@ -67,29 +67,34 @@ def setup_tests(ns): if ns.threshold is not None: gc.set_threshold(ns.threshold) + suppress_msvcrt_asserts(ns.verbose and ns.verbose >= 2) + + support.use_resources = ns.use_resources + + +def suppress_msvcrt_asserts(verbose): try: import msvcrt except ImportError: - pass - else: - msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| - msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| - msvcrt.SEM_NOGPFAULTERRORBOX| - msvcrt.SEM_NOOPENFILEERRORBOX) - try: - msvcrt.CrtSetReportMode - except AttributeError: - # release build - pass + return + + msvcrt.SetErrorMode(msvcrt.SEM_FAILCRITICALERRORS| + msvcrt.SEM_NOALIGNMENTFAULTEXCEPT| + msvcrt.SEM_NOGPFAULTERRORBOX| + msvcrt.SEM_NOOPENFILEERRORBOX) + try: + msvcrt.CrtSetReportMode + except AttributeError: + # release build + return + + for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: + if verbose: + msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) + msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) else: - for m in [msvcrt.CRT_WARN, msvcrt.CRT_ERROR, msvcrt.CRT_ASSERT]: - if ns.verbose and ns.verbose >= 2: - msvcrt.CrtSetReportMode(m, msvcrt.CRTDBG_MODE_FILE) - msvcrt.CrtSetReportFile(m, msvcrt.CRTDBG_FILE_STDERR) - else: - msvcrt.CrtSetReportMode(m, 0) + msvcrt.CrtSetReportMode(m, 0) - support.use_resources = ns.use_resources def replace_stdout(): diff --git a/lib-python/3/test/libregrtest/utils.py b/lib-python/3/test/libregrtest/utils.py index fb9971a64f..98a60f7a74 100644 --- a/lib-python/3/test/libregrtest/utils.py +++ b/lib-python/3/test/libregrtest/utils.py @@ -16,11 +16,14 @@ def format_duration(seconds): if minutes: parts.append('%s min' % minutes) if seconds: - parts.append('%s sec' % seconds) - if ms: - parts.append('%s ms' % ms) + if parts: + # 2 min 1 sec + parts.append('%s sec' % seconds) + else: + # 1.0 sec + parts.append('%.1f sec' % (seconds + ms / 1000)) if not parts: - return '0 ms' + return '%s ms' % ms parts = parts[:2] return ' '.join(parts) diff --git a/lib-python/3/test/libregrtest/win_utils.py b/lib-python/3/test/libregrtest/win_utils.py index adfe278ba3..028c01106d 100644 --- a/lib-python/3/test/libregrtest/win_utils.py +++ b/lib-python/3/test/libregrtest/win_utils.py @@ -1,30 +1,43 @@ import _winapi +import math import msvcrt import os import subprocess import uuid +import winreg from test import support +from test.libregrtest.utils import print_warning # Max size of asynchronous reads BUFSIZE = 8192 -# Exponential damping factor (see below) -LOAD_FACTOR_1 = 0.9200444146293232478931553241 # Seconds per measurement -SAMPLING_INTERVAL = 5 -COUNTER_NAME = r'\System\Processor Queue Length' +SAMPLING_INTERVAL = 1 +# Exponential damping factor to compute exponentially weighted moving average +# on 1 minute (60 seconds) +LOAD_FACTOR_1 = 1 / math.exp(SAMPLING_INTERVAL / 60) +# Initialize the load using the arithmetic mean of the first NVALUE values +# of the Processor Queue Length +NVALUE = 5 +# Windows registry subkey of HKEY_LOCAL_MACHINE where the counter names +# of typeperf are registered +COUNTER_REGISTRY_KEY = (r"SOFTWARE\Microsoft\Windows NT\CurrentVersion" + r"\Perflib\CurrentLanguage") class WindowsLoadTracker(): """ This class asynchronously interacts with the `typeperf` command to read - the system load on Windows. Mulitprocessing and threads can't be used + the system load on Windows. Multiprocessing and threads can't be used here because they interfere with the test suite's cases for those modules. """ def __init__(self): - self.load = 0.0 + self._values = [] + self._load = None + self._buffer = '' + self._popen = None self.start() def start(self): @@ -54,52 +67,124 @@ class WindowsLoadTracker(): overlap.GetOverlappedResult(True) # Spawn off the load monitor - command = ['typeperf', COUNTER_NAME, '-si', str(SAMPLING_INTERVAL)] - self.p = subprocess.Popen(command, stdout=command_stdout, cwd=support.SAVEDCWD) + counter_name = self._get_counter_name() + command = ['typeperf', counter_name, '-si', str(SAMPLING_INTERVAL)] + self._popen = subprocess.Popen(' '.join(command), stdout=command_stdout, cwd=support.SAVEDCWD) # Close our copy of the write end of the pipe os.close(command_stdout) - def close(self): - if self.p is None: + def _get_counter_name(self): + # accessing the registry to get the counter localization name + with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, COUNTER_REGISTRY_KEY) as perfkey: + counters = winreg.QueryValueEx(perfkey, 'Counter')[0] + + # Convert [key1, value1, key2, value2, ...] list + # to {key1: value1, key2: value2, ...} dict + counters = iter(counters) + counters_dict = dict(zip(counters, counters)) + + # System counter has key '2' and Processor Queue Length has key '44' + system = counters_dict['2'] + process_queue_length = counters_dict['44'] + return f'"\\{system}\\{process_queue_length}"' + + def close(self, kill=True): + if self._popen is None: return - self.p.kill() - self.p.wait() - self.p = None + + self._load = None + + if kill: + self._popen.kill() + self._popen.wait() + self._popen = None def __del__(self): self.close() - def read_output(self): - import _winapi - + def _parse_line(self, line): + # typeperf outputs in a CSV format like this: + # "07/19/2018 01:32:26.605","3.000000" + # (date, process queue length) + tokens = line.split(',') + if len(tokens) != 2: + raise ValueError + + value = tokens[1] + if not value.startswith('"') or not value.endswith('"'): + raise ValueError + value = value[1:-1] + return float(value) + + def _read_lines(self): overlapped, _ = _winapi.ReadFile(self.pipe, BUFSIZE, True) bytes_read, res = overlapped.GetOverlappedResult(False) if res != 0: - return - - return overlapped.getbuffer().decode() + return () + + output = overlapped.getbuffer() + output = output.decode('oem', 'replace') + output = self._buffer + output + lines = output.splitlines(True) + + # bpo-36670: typeperf only writes a newline *before* writing a value, + # not after. Sometimes, the written line in incomplete (ex: only + # timestamp, without the process queue length). Only pass the last line + # to the parser if it's a valid value, otherwise store it in + # self._buffer. + try: + self._parse_line(lines[-1]) + except ValueError: + self._buffer = lines.pop(-1) + else: + self._buffer = '' + + return lines def getloadavg(self): - typeperf_output = self.read_output() - # Nothing to update, just return the current load - if not typeperf_output: - return self.load - - # Process the backlog of load values - for line in typeperf_output.splitlines(): - # typeperf outputs in a CSV format like this: - # "07/19/2018 01:32:26.605","3.000000" - toks = line.split(',') - # Ignore blank lines and the initial header - if line.strip() == '' or (COUNTER_NAME in line) or len(toks) != 2: + if self._popen is None: + return None + + returncode = self._popen.poll() + if returncode is not None: + self.close(kill=False) + return None + + try: + lines = self._read_lines() + except BrokenPipeError: + self.close() + return None + + for line in lines: + line = line.rstrip() + + # Ignore the initial header: + # "(PDH-CSV 4.0)","\\\\WIN\\System\\Processor Queue Length" + if 'PDH-CSV' in line: + continue + + # Ignore blank lines + if not line: + continue + + try: + processor_queue_length = self._parse_line(line) + except ValueError: + print_warning("Failed to parse typeperf output: %a" % line) continue - load = float(toks[1].replace('"', '')) # We use an exponentially weighted moving average, imitating the # load calculation on Unix systems. # https://en.wikipedia.org/wiki/Load_(computing)#Unix-style_load_calculation - new_load = self.load * LOAD_FACTOR_1 + load * (1.0 - LOAD_FACTOR_1) - self.load = new_load - - return self.load + # https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average + if self._load is not None: + self._load = (self._load * LOAD_FACTOR_1 + + processor_queue_length * (1.0 - LOAD_FACTOR_1)) + elif len(self._values) < NVALUE: + self._values.append(processor_queue_length) + else: + self._load = sum(self._values) / len(self._values) + + return self._load diff --git a/lib-python/3/test/make_ssl_certs.py b/lib-python/3/test/make_ssl_certs.py index 362276583f..41b5f46c88 100644 --- a/lib-python/3/test/make_ssl_certs.py +++ b/lib-python/3/test/make_ssl_certs.py @@ -206,8 +206,8 @@ if __name__ == '__main__': with open('ssl_key.pem', 'w') as f: f.write(key) print("password protecting ssl_key.pem in ssl_key.passwd.pem") - check_call(['openssl','rsa','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-des3','-passout','pass:somepass']) - check_call(['openssl','rsa','-in','ssl_key.pem','-out','keycert.passwd.pem','-des3','-passout','pass:somepass']) + check_call(['openssl','pkey','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-aes256','-passout','pass:somepass']) + check_call(['openssl','pkey','-in','ssl_key.pem','-out','keycert.passwd.pem','-aes256','-passout','pass:somepass']) with open('keycert.pem', 'w') as f: f.write(key) diff --git a/lib-python/3/test/pickletester.py b/lib-python/3/test/pickletester.py index 1d88fcb859..c576d73349 100644 --- a/lib-python/3/test/pickletester.py +++ b/lib-python/3/test/pickletester.py @@ -998,6 +998,24 @@ class AbstractUnpickleTests(unittest.TestCase): self.assertIs(type(unpickled), collections.UserDict) self.assertEqual(unpickled, collections.UserDict({1: 2})) + def test_bad_reduce(self): + self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0) + self.check_unpickling_error(TypeError, b'N)R.') + self.check_unpickling_error(TypeError, b'cbuiltins\nint\nNR.') + + def test_bad_newobj(self): + error = (pickle.UnpicklingError, TypeError) + self.assertEqual(self.loads(b'cbuiltins\nint\n)\x81.'), 0) + self.check_unpickling_error(error, b'cbuiltins\nlen\n)\x81.') + self.check_unpickling_error(error, b'cbuiltins\nint\nN\x81.') + + def test_bad_newobj_ex(self): + error = (pickle.UnpicklingError, TypeError) + self.assertEqual(self.loads(b'cbuiltins\nint\n)}\x92.'), 0) + self.check_unpickling_error(error, b'cbuiltins\nlen\n)}\x92.') + self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.') + self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.') + def test_bad_stack(self): badpickles = [ b'.', # STOP diff --git a/lib-python/3/test/pythoninfo.py b/lib-python/3/test/pythoninfo.py index 580956633f..d16ad67cdc 100644 --- a/lib-python/3/test/pythoninfo.py +++ b/lib-python/3/test/pythoninfo.py @@ -340,6 +340,9 @@ def collect_gdb(info_add): stderr=subprocess.PIPE, universal_newlines=True) version = proc.communicate()[0] + if proc.returncode: + # ignore gdb failure: test_gdb will log the error + return except OSError: return @@ -439,10 +442,15 @@ def collect_sysconfig(info_add): def collect_ssl(info_add): + import os try: import ssl except ImportError: return + try: + import _ssl + except ImportError: + _ssl = None def format_attr(attr, value): if attr.startswith('OP_'): @@ -459,6 +467,32 @@ def collect_ssl(info_add): ) copy_attributes(info_add, ssl, 'ssl.%s', attributes, formatter=format_attr) + for name, ctx in ( + ('SSLContext', ssl.SSLContext()), + ('default_https_context', ssl._create_default_https_context()), + ('stdlib_context', ssl._create_stdlib_context()), + ): + attributes = ( + 'minimum_version', + 'maximum_version', + 'protocol', + 'options', + 'verify_mode', + ) + copy_attributes(info_add, ctx, f'ssl.{name}.%s', attributes) + + env_names = ["OPENSSL_CONF", "SSLKEYLOGFILE"] + if _ssl is not None and hasattr(_ssl, 'get_default_verify_paths'): + parts = _ssl.get_default_verify_paths() + env_names.extend((parts[0], parts[2])) + + for name in env_names: + try: + value = os.environ[name] + except KeyError: + continue + info_add('ssl.environ[%s]' % name, value) + def collect_socket(info_add): import socket diff --git a/lib-python/3/test/recursion.tar b/lib-python/3/test/recursion.tar Binary files differnew file mode 100644 index 0000000000..b823725196 --- /dev/null +++ b/lib-python/3/test/recursion.tar diff --git a/lib-python/3/test/sortperf.py b/lib-python/3/test/sortperf.py index 171e5cef5e..14a9d827ed 100644 --- a/lib-python/3/test/sortperf.py +++ b/lib-python/3/test/sortperf.py @@ -134,7 +134,7 @@ def tabulate(r): L = list(range(half - 1, -1, -1)) L.extend(range(half)) # Force to float, so that the timings are comparable. This is - # significantly faster if we leave tham as ints. + # significantly faster if we leave them as ints. L = list(map(float, L)) doit(L) # !sort print() diff --git a/lib-python/3/test/ssl_key.passwd.pem b/lib-python/3/test/ssl_key.passwd.pem index e4f1370ab2..46de61ab85 100644 --- a/lib-python/3/test/ssl_key.passwd.pem +++ b/lib-python/3/test/ssl_key.passwd.pem @@ -1,42 +1,42 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,8064BE1494B24B13 - -KJrffOMbo8M0I3PzcYxRZGMpKD1yB3Ii4+bT5XoanxjIJ+4fdx6LfZ0Rsx+riyzs -tymsQu/iYY9j+4rCvN9+eetsL1X6iZpiimKsLexcid9M3fb0vxED5Sgw0dvunCUA -xhqjLIKR92MKbODHf6KrDKCpsiPbjq4gZ7P+uCGXAMHL3MXIJSC0hW9rK7Ce6oyO -CjpIcgB8x+GUWZZZhAFdlzIHMZrteNP2P5HK6QcaT71P034Dz1hhqoj4Q0t+Fta2 -4tfsM/bnTR/l6hwlhPa1e3Uj322tDTDWBScgWANn5+sEWldLmozMaWhZsn22pfk2 -KjRMGXG024JVheV882nbdOBvG7oq+lxkZ/ZP+vvqJqnvYtf7WtM8UivzYpe5Hz5b -kVvWzPjBLUSZ9whM9rDLqSSqMPyPvDTuEmLkuq+xm7pYJmsLqIMP2klZLqRxLX6K -uqwplb8UG440qauxgnQ905PId1l2fJEnRtV+7vXprA0L0QotgXLVHBhLmTFM+3PH -9H3onf31dionUAPrn3nfVE36HhvVgRyvDBnBzJSIMighgq21Qx/d1dk0DRYi1hUI -nCHl0YJPXheVcXR7JiSF2XQCAaFuS1Mr7NCXfWZOZQC/0dkvmHnl9DUAhuqq9BNZ -1cKhZXcKHadg2/r0Zup/oDzmHPUEfTAXT0xbqoWlhkdwbF2veWQ96A/ncx3ISTb4 -PkXBlX9rdia8nmtyQDQRn4NuvchbaGkj4WKFC8pF8Hn7naHqwjpHaDUimBc0CoQW -edNJqruKWwtSVLuwKHCC2gZFX9AXSKJXJz/QRSUlhFGOhuF/J6yKaXj6n5lxWNiQ -54J+OP/hz2aS95CD2+Zf1SKpxdWiLZSIQqESpmmUrXROixNJZ/Z7gI74Dd9dSJOH -W+3AU03vrrFZVrJVZhjcINHoH1Skh6JKscH18L6x4U868nSr4SrRLX8BhHllOQyD -bmU+PZAjF8ZBIaCtTGulDXD29F73MeAZeTSsgQjFu0iKLj1wPiphbx8i/SUtR4YP -X6PVA04g66r1NBw+3RQASVorZ3g1MSFvITHXcbKkBDeJH2z1+c6t/VVyTONnQhM5 -lLgRSk6HCbetvT9PKxWrWutA12pdBYEHdZhMHVf2+xclky7l09w8hg2/qqcdGRGe -oAOZ72t0l5ObNyaruDKUS6f4AjOyWq/Xj5xuFtf1n3tQHyslSyCTPcAbQhDfTHUx -vixb/V9qvYPt7OCn8py7v1M69NH42QVFAvwveDIFjZdqfIKBoJK2V4qPoevJI6uj -Q5ByMt8OXOjSXNpHXpYQWUiWeCwOEBXJX8rzCHdMtg37jJ0zCmeErR1NTdg+EujM -TWYgd06jlT67tURST0aB2kg4ijKgUJefD313LW1zC6gVsTbjSZxYyRbPfSP6flQB -yCi1C19E2OsgleqbkBVC5GlYUzaJT7SGjCRmGx1eqtbrALu+LVH24Wceexlpjydl -+s2nf/DZlKun/tlPh6YioifPCJjByZMQOCEfIox6BkemZETz8uYA4TTWimG13Z03 -gyDGC2jdpEW414J2qcQDvrdUgJ+HlhrAAHaWpMQDbXYxBGoZ+3+ORvQV4kAsCwL8 -k3EIrVpePdik+1xgOWsyLj6QxFXlTMvL6Wc5pnArFPORsgHEolJvxSPTf9aAHNPn -V2WBvxiLBtYpGrujAUM40Syx/aN2RPtcXYPAusHUBw+S8/p+/8Kg8GZmnIXG3F89 -45Eepl2quZYIrou7a1fwIpIIZ0hFiBQ1mlHVMFtxwVHS1bQb3SU2GeO+JcGjdVXc -04qeGuQ5M164eQ5C0T7ZQ1ULiUlFWKD30m+cjqmZzt3d7Q0mKpMKuESIuZJo/wpD -Nas432aLKUhcNx/pOYLkKJRpGZKOupQoD5iUj/j44o8JoFkDK33v2S57XB5QGz28 -9Zuhx49b3W8mbM6EBanlQKLWJGCxXqc/jhYhFWn+b0MhidynFgA0oeWvf6ZDyt6H -Yi5Etxsar09xp0Do3NxtQXLuSUu0ji2pQzSIKuoqQWKqldm6VrpwojiqJhy4WQBQ -aVVyFeWBC7G3Zj76dO+yp2sfJ0itJUQ8AIB9Cg0f34rEZu+r9luPmqBoUeL95Tk7 -YvCOU3Jl8Iqysv8aNpVXT8sa8rrSbruWCByEePZ37RIdHLMVBwVY0eVaFQjrjU7E -mXmM9eaoYLfXOllsQ+M2+qPFUITr/GU3Qig13DhK/+yC1R6V2a0l0WRhMltIPYKW -Ztvvr4hK5LcYCeS113BLiMbDIMMZZYGDZGMdC8DnnVbT2loF0Rfmp80Af31KmMQ4 -6XvMatW9UDjBoY5a/YMpdm7SRwm+MgV2KNPpc2kST87/yi9oprGAb8qiarHiHTM0 ------END RSA PRIVATE KEY----- +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQI072N7W+PDDMCAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBA/AuaRNi4vE4KGqI4In+70BIIH +ENGS5Vex5NID873frmd1UZEHZ+O/Bd0wDb+NUpIqesHkRYf7kKi6Gnr+nKQ/oVVn +Lm3JjE7c8ECP0OkOOXmiXuWL1SkzBBWqCI4stSGUPvBiHsGwNnvJAaGjUffgMlcC +aJOA2+dnejLkzblq4CB2LQdm06N3Xoe9tyqtQaUHxfzJAf5Ydd8uj7vpKN2MMhY7 +icIPJwSyh0N7S6XWVtHEokr9Kp4y2hS5a+BgCWV1/1z0aF7agnSVndmT1VR+nWmc +lM14k+lethmHMB+fsNSjnqeJ7XOPlOTHqhiZ9bBSTgF/xr5Bck/NiKRzHjdovBox +TKg+xchaBhpRh7wBPBIlNJeHmIjv+8obOKjKU98Ig/7R9+IryZaNcKAH0PuOT+Sw +QHXiCGQbOiYHB9UyhDTWiB7YVjd8KHefOFxfHzOQb/iBhbv1x3bTl3DgepvRN6VO +dIsPLoIZe42sdf9GeMsk8mGJyZUQ6AzsfhWk3grb/XscizPSvrNsJ2VL1R7YTyT3 +3WA4ZXR1EqvXnWL7N/raemQjy62iOG6t7fcF5IdP9CMbWP+Plpsz4cQW7FtesCTq +a5ZXraochQz361ODFNIeBEGU+0qqXUtZDlmos/EySkZykSeU/L0bImS62VGE3afo +YXBmznTTT9kkFkqv7H0MerfJsrE/wF8puP3GM01DW2JRgXRpSWlvbPV/2LnMtRuD +II7iH4rWDtTjCN6BWKAgDOnPkc9sZ4XulqT32lcUeV6LTdMBfq8kMEc8eDij1vUT +maVCRpuwaq8EIT3lVgNLufHiG96ojlyYtj3orzw22IjkgC/9ee8UDik9CqbMVmFf +fVHhsw8LNSg8Q4bmwm5Eg2w2it2gtI68+mwr75oCxuJ/8OMjW21Prj8XDh5reie2 +c0lDKQOFZ9UnLU1bXR/6qUM+JFKR4DMq+fOCuoQSVoyVUEOsJpvBOYnYZN9cxsZm +vh9dKafMEcKZ8flsbr+gOmOw7+Py2ifSlf25E/Frb1W4gtbTb0LQVHb6+drutrZj +8HEu4CnHYFCD4ZnOJb26XlZCb8GFBddW86yJYyUqMMV6Q1aJfAOAglsTo1LjIMOZ +byo0BTAmwUevU/iuOXQ4qRBXXcoidDcTCrxfUSPG9wdt9l+m5SdQpWqfQ+fx5O7m +SLlrHyZCiPSFMtC9DxqjIklHjf5W3wslGLgaD30YXa4VDYkRihf3CNsxGQ+tVvef +l0ZjoAitF7Gaua06IESmKnpHe23dkr1cjYq+u2IV+xGH8LeExdwsQ9kpuTeXPnQs +JOA99SsFx1ct32RrwjxnDDsiNkaViTKo9GDkV3jQTfoFgAVqfSgg9wGXpqUqhNG7 +TiSIHCowllLny2zn4XrXCy2niD3VDt0skb3l/PaegHE2z7S5YY85nQtYwpLiwB9M +SQ08DYKxPBZYKtS2iZ/fsA1gjSRQDPg/SIxMhUC3M3qH8iWny1Lzl25F2Uq7VVEX +LdTUtaby49jRTT3CQGr5n6z7bMbUegiY7h8WmOekuThGDH+4xZp6+rDP4GFk4FeK +JcF70vMQYIjQZhadic6olv+9VtUP42ltGG/yP9a3eWRkzfAf2eCh6B1rYdgEWwE8 +rlcZzwM+y6eUmeNF2FVWB8iWtTMQHy+dYNPM+Jtus1KQKxiiq/yCRs7nWvzWRFWA +HRyqV0J6/lqgm4FvfktFt1T0W+mDoLJOR2/zIwMy2lgL5zeHuR3SaMJnCikJbqKS +HB3UvrhAWUcZqdH29+FhVWeM7ybyF1Wccmf+IIC/ePLa6gjtqPV8lG/5kbpcpnB6 +UQY8WWaKMxyr3jJ9bAX5QKshchp04cDecOLZrpFGNNQngR8RxSEkiIgAqNxWunIu +KrdBDrupv/XAgEOclmgToY3iywLJSV5gHAyHWDUhRH4cFCLiGPl4XIcnXOuTze3H +3j+EYSiS3v3DhHjp33YU2pXlJDjiYsKzAXejEh66++Y8qaQdCAad3ruWRCzW3kgk +Md0A1VGzntTnQsewvExQEMZH2LtYIsPv3KCYGeSAuLabX4tbGk79PswjnjLLEOr0 +Ghf6RF6qf5/iFyJoG4vrbKT8kx6ywh0InILCdjUunuDskIBxX6tEcr9XwajoIvb2 +kcmGdjam5kKLS7QOWQTl8/r/cuFes0dj34cX5Qpq+Gd7tRq/D+b0207926Cxvftv +qQ1cVn8HiLxKkZzd3tpf2xnoV1zkTL0oHrNg+qzxoxXUTUcwtIf1d/HRbYEAhi/d +bBBoFeftEHWNq+sJgS9bH+XNzo/yK4u04B5miOq8v4CSkJdzu+ZdF22d4cjiGmtQ +8BTmcn0Unzm+u5H0+QSZe54QBHJGNXXOIKMTkgnOdW27g4DbI1y7fCqJiSMbRW6L +oHmMfbdB3GWqGbsUkhY8i6h9op0MU6WOX7ea2Rxyt4t6 +-----END ENCRYPTED PRIVATE KEY----- diff --git a/lib-python/3/test/support/__init__.py b/lib-python/3/test/support/__init__.py index 87bfa9f546..b78451b9e6 100644 --- a/lib-python/3/test/support/__init__.py +++ b/lib-python/3/test/support/__init__.py @@ -109,6 +109,7 @@ __all__ = [ "run_with_locale", "swap_item", "swap_attr", "Matcher", "set_memlimit", "SuppressCrashReport", "sortdict", "run_with_tz", "PGO", "missing_compiler_executable", "fd_count", + "ALWAYS_EQ", "LARGEST", "SMALLEST" ] class Error(Exception): @@ -559,25 +560,25 @@ def _requires_unix_version(sysname, min_version): For example, @_requires_unix_version('FreeBSD', (7, 2)) raises SkipTest if the FreeBSD version is less than 7.2. """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kw): - if platform.system() == sysname: - version_txt = platform.release().split('-', 1)[0] - try: - version = tuple(map(int, version_txt.split('.'))) - except ValueError: - pass - else: - if version < min_version: - min_version_txt = '.'.join(map(str, min_version)) - raise unittest.SkipTest( - "%s version %s or higher required, not %s" - % (sysname, min_version_txt, version_txt)) - return func(*args, **kw) - wrapper.min_version = min_version - return wrapper - return decorator + import platform + min_version_txt = '.'.join(map(str, min_version)) + version_txt = platform.release().split('-', 1)[0] + if platform.system() == sysname: + try: + version = tuple(map(int, version_txt.split('.'))) + except ValueError: + skip = False + else: + skip = version < min_version + else: + skip = False + + return unittest.skipIf( + skip, + f"{sysname} version {min_version_txt} or higher required, not " + f"{version_txt}" + ) + def requires_freebsd_version(*min_version): """Decorator raising SkipTest if the OS is FreeBSD and the FreeBSD version is @@ -1422,6 +1423,9 @@ def get_socket_conn_refused_errs(): # bpo-31910: socket.create_connection() fails randomly # with EADDRNOTAVAIL on Travis CI errors.append(errno.EADDRNOTAVAIL) + if hasattr(errno, 'EHOSTUNREACH'): + # bpo-37583: The destination host cannot be reached + errors.append(errno.EHOSTUNREACH) return errors @@ -1932,7 +1936,9 @@ def _run_suite(suite): # By default, don't filter tests _match_test_func = None -_match_test_patterns = None + +_accept_test_patterns = None +_ignore_test_patterns = None def match_test(test): @@ -1948,18 +1954,45 @@ def _is_full_match_test(pattern): # as a full test identifier. # Example: 'test.test_os.FileTests.test_access'. # - # Reject patterns which contain fnmatch patterns: '*', '?', '[...]' - # or '[!...]'. For example, reject 'test_access*'. + # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, ignore 'test_access*'. return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) -def set_match_tests(patterns): - global _match_test_func, _match_test_patterns +def set_match_tests(accept_patterns=None, ignore_patterns=None): + global _match_test_func, _accept_test_patterns, _ignore_test_patterns - if patterns == _match_test_patterns: - # No change: no need to recompile patterns. - return + if accept_patterns is None: + accept_patterns = () + if ignore_patterns is None: + ignore_patterns = () + + accept_func = ignore_func = None + + if accept_patterns != _accept_test_patterns: + accept_patterns, accept_func = _compile_match_function(accept_patterns) + if ignore_patterns != _ignore_test_patterns: + ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) + + # Create a copy since patterns can be mutable and so modified later + _accept_test_patterns = tuple(accept_patterns) + _ignore_test_patterns = tuple(ignore_patterns) + + if accept_func is not None or ignore_func is not None: + def match_function(test_id): + accept = True + ignore = False + if accept_func: + accept = accept_func(test_id) + if ignore_func: + ignore = ignore_func(test_id) + return accept and not ignore + + _match_test_func = match_function + + +def _compile_match_function(patterns): if not patterns: func = None # set_match_tests(None) behaves as set_match_tests(()) @@ -1987,10 +2020,7 @@ def set_match_tests(patterns): func = match_test_regex - # Create a copy since patterns can be mutable and so modified later - _match_test_patterns = tuple(patterns) - _match_test_func = func - + return patterns, func def run_unittest(*classes): @@ -2806,7 +2836,7 @@ def fd_count(): if sys.platform.startswith(('linux', 'freebsd')): try: names = os.listdir("/proc/self/fd") - # Substract one because listdir() opens internally a file + # Subtract one because listdir() internally opens a file # descriptor to list the content of the /proc/self/fd/ directory. return len(names) - 1 except FileNotFoundError: @@ -2919,3 +2949,39 @@ class FakePath: raise self.path else: return self.path + + +class _ALWAYS_EQ: + """ + Object that is equal to anything. + """ + def __eq__(self, other): + return True + def __ne__(self, other): + return False + +ALWAYS_EQ = _ALWAYS_EQ() + +@functools.total_ordering +class _LARGEST: + """ + Object that is greater than anything (except itself). + """ + def __eq__(self, other): + return isinstance(other, _LARGEST) + def __lt__(self, other): + return False + +LARGEST = _LARGEST() + +@functools.total_ordering +class _SMALLEST: + """ + Object that is less than anything (except itself). + """ + def __eq__(self, other): + return isinstance(other, _SMALLEST) + def __gt__(self, other): + return False + +SMALLEST = _SMALLEST() diff --git a/lib-python/3/test/test__osx_support.py b/lib-python/3/test/test__osx_support.py index 388a2b1a84..1a5d649b40 100644 --- a/lib-python/3/test/test__osx_support.py +++ b/lib-python/3/test/test__osx_support.py @@ -174,6 +174,29 @@ class Test_OSXSupport(unittest.TestCase): _osx_support._remove_universal_flags( config_vars)) + def test__remove_universal_flags_alternate(self): + # bpo-38360: also test the alternate single-argument form of -isysroot + config_vars = { + 'CFLAGS': '-fno-strict-aliasing -g -O3 -arch ppc -arch i386 ', + 'LDFLAGS': '-arch ppc -arch i386 -g', + 'CPPFLAGS': '-I. -isysroot/Developer/SDKs/MacOSX10.4u.sdk', + 'BLDSHARED': 'gcc-4.0 -bundle -arch ppc -arch i386 -g', + 'LDSHARED': 'gcc-4.0 -bundle -arch ppc -arch i386 ' + '-isysroot/Developer/SDKs/MacOSX10.4u.sdk -g', + } + expected_vars = { + 'CFLAGS': '-fno-strict-aliasing -g -O3 ', + 'LDFLAGS': ' -g', + 'CPPFLAGS': '-I. ', + 'BLDSHARED': 'gcc-4.0 -bundle -g', + 'LDSHARED': 'gcc-4.0 -bundle -g', + } + self.add_expected_saved_initial_values(config_vars, expected_vars) + + self.assertEqual(expected_vars, + _osx_support._remove_universal_flags( + config_vars)) + def test__remove_unsupported_archs(self): config_vars = { 'CC': 'clang', @@ -261,6 +284,34 @@ class Test_OSXSupport(unittest.TestCase): _osx_support._check_for_unavailable_sdk( config_vars)) + def test__check_for_unavailable_sdk_alternate(self): + # bpo-38360: also test the alternate single-argument form of -isysroot + config_vars = { + 'CC': 'clang', + 'CFLAGS': '-fno-strict-aliasing -g -O3 -arch ppc -arch i386 ' + '-isysroot/Developer/SDKs/MacOSX10.1.sdk', + 'LDFLAGS': '-arch ppc -arch i386 -g', + 'CPPFLAGS': '-I. -isysroot/Developer/SDKs/MacOSX10.1.sdk', + 'BLDSHARED': 'gcc-4.0 -bundle -arch ppc -arch i386 -g', + 'LDSHARED': 'gcc-4.0 -bundle -arch ppc -arch i386 ' + '-isysroot/Developer/SDKs/MacOSX10.1.sdk -g', + } + expected_vars = { + 'CC': 'clang', + 'CFLAGS': '-fno-strict-aliasing -g -O3 -arch ppc -arch i386 ' + ' ', + 'LDFLAGS': '-arch ppc -arch i386 -g', + 'CPPFLAGS': '-I. ', + 'BLDSHARED': 'gcc-4.0 -bundle -arch ppc -arch i386 -g', + 'LDSHARED': 'gcc-4.0 -bundle -arch ppc -arch i386 ' + ' -g', + } + self.add_expected_saved_initial_values(config_vars, expected_vars) + + self.assertEqual(expected_vars, + _osx_support._check_for_unavailable_sdk( + config_vars)) + def test_get_platform_osx(self): # Note, get_platform_osx is currently tested more extensively # indirectly by test_sysconfig and test_distutils diff --git a/lib-python/3/test/test_argparse.py b/lib-python/3/test/test_argparse.py index 51f0effaf2..3cdaff61c7 100644 --- a/lib-python/3/test/test_argparse.py +++ b/lib-python/3/test/test_argparse.py @@ -2772,6 +2772,46 @@ class TestMutuallyExclusiveOptionalsAndPositionalsMixed(MEMixin, TestCase): -c c help ''' +class TestMutuallyExclusiveNested(MEMixin, TestCase): + + def get_parser(self, required): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('-a') + group.add_argument('-b') + group2 = group.add_mutually_exclusive_group(required=required) + group2.add_argument('-c') + group2.add_argument('-d') + group3 = group2.add_mutually_exclusive_group(required=required) + group3.add_argument('-e') + group3.add_argument('-f') + return parser + + usage_when_not_required = '''\ + usage: PROG [-h] [-a A | -b B | [-c C | -d D | [-e E | -f F]]] + ''' + usage_when_required = '''\ + usage: PROG [-h] (-a A | -b B | (-c C | -d D | (-e E | -f F))) + ''' + + help = '''\ + + optional arguments: + -h, --help show this help message and exit + -a A + -b B + -c C + -d D + -e E + -f F + ''' + + # We are only interested in testing the behavior of format_usage(). + test_failures_when_not_required = None + test_failures_when_required = None + test_successes_when_not_required = None + test_successes_when_required = None + # ================================================= # Mutually exclusive group in parent parser tests # ================================================= diff --git a/lib-python/3/test/test_ast.py b/lib-python/3/test/test_ast.py index 72f8467847..830fb58a02 100644 --- a/lib-python/3/test/test_ast.py +++ b/lib-python/3/test/test_ast.py @@ -4,6 +4,7 @@ import os import sys import unittest import weakref +from textwrap import dedent from test import support @@ -63,6 +64,10 @@ exec_tests = [ "while v:pass", # If "if v:pass", + # If-Elif + "if a:\n pass\nelif b:\n pass", + # If-Elif-Else + "if a:\n pass\nelif b:\n pass\nelse:\n pass", # With "with x as y: pass", "with x as y, z as q: pass", @@ -124,6 +129,14 @@ exec_tests = [ "{*{1, 2}, 3}", # Asynchronous comprehensions "async def f():\n [i async for b in c]", + # Decorated FunctionDef + "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", + # Decorated AsyncFunctionDef + "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", + # Decorated ClassDef + "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", + # Decorator with generator argument + "@deco(a for a in b)\ndef f(): pass", ] # These are compiled through "single" @@ -461,6 +474,35 @@ class ASTHelpers_Test(unittest.TestCase): "lineno=1, col_offset=0), lineno=1, col_offset=0)])" ) + def test_dump_incomplete(self): + node = ast.Raise(lineno=3, col_offset=4) + self.assertEqual(ast.dump(node), + "Raise()" + ) + self.assertEqual(ast.dump(node, include_attributes=True), + "Raise(lineno=3, col_offset=4)" + ) + node = ast.Raise(exc=ast.Name(id='e', ctx=ast.Load()), lineno=3, col_offset=4) + self.assertEqual(ast.dump(node), + "Raise(exc=Name(id='e', ctx=Load()))" + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "Raise(Name('e', Load()))" + ) + self.assertEqual(ast.dump(node, include_attributes=True), + "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)" + ) + self.assertEqual(ast.dump(node, annotate_fields=False, include_attributes=True), + "Raise(Name('e', Load()), lineno=3, col_offset=4)" + ) + node = ast.Raise(cause=ast.Name(id='e', ctx=ast.Load())) + self.assertEqual(ast.dump(node), + "Raise(cause=Name(id='e', ctx=Load()))" + ) + self.assertEqual(ast.dump(node, annotate_fields=False), + "Raise(cause=Name('e', Load()))" + ) + def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') src.body.right = ast.copy_location(ast.Num(2), src.body.right) @@ -560,6 +602,18 @@ class ASTHelpers_Test(unittest.TestCase): node = ast.parse('async def foo():\n x = "not docstring"') self.assertIsNone(ast.get_docstring(node.body[0])) + def test_elif_stmt_start_position(self): + node = ast.parse('if a:\n pass\nelif b:\n pass\n') + elif_stmt = node.body[0].orelse[0] + self.assertEqual(elif_stmt.lineno, 3) + self.assertEqual(elif_stmt.col_offset, 0) + + def test_elif_stmt_start_position_with_else(self): + node = ast.parse('if a:\n pass\nelif b:\n pass\nelse:\n pass\n') + elif_stmt = node.body[0].orelse[0] + self.assertEqual(elif_stmt.lineno, 3) + self.assertEqual(elif_stmt.col_offset, 0) + def test_literal_eval(self): self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3]) self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42}) @@ -1123,6 +1177,44 @@ class ConstantTests(unittest.TestCase): self.assertEqual(ast.literal_eval(binop), 10+20j) +class NodeVisitorTests(unittest.TestCase): + def test_old_constant_nodes(self): + class Visitor(ast.NodeVisitor): + def visit_Num(self, node): + log.append((node.lineno, 'Num', node.n)) + def visit_Str(self, node): + log.append((node.lineno, 'Str', node.s)) + def visit_Bytes(self, node): + log.append((node.lineno, 'Bytes', node.s)) + def visit_NameConstant(self, node): + log.append((node.lineno, 'NameConstant', node.value)) + def visit_Ellipsis(self, node): + log.append((node.lineno, 'Ellipsis', ...)) + mod = ast.parse(dedent('''\ + i = 42 + f = 4.25 + c = 4.25j + s = 'string' + b = b'bytes' + t = True + n = None + e = ... + ''')) + visitor = Visitor() + log = [] + visitor.visit(mod) + self.assertEqual(log, [ + (1, 'Num', 42), + (2, 'Num', 4.25), + (3, 'Num', 4.25j), + (4, 'Str', 'string'), + (5, 'Bytes', b'bytes'), + (6, 'NameConstant', True), + (7, 'NameConstant', None), + (8, 'Ellipsis', ...), + ]) + + def main(): if __name__ != '__main__': return @@ -1159,6 +1251,8 @@ exec_results = [ ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]), ('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]), ('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]), +('Module', [('If', (1, 0), ('Name', (1, 3), 'a', ('Load',)), [('Pass', (2, 2))], [('If', (3, 0), ('Name', (3, 5), 'b', ('Load',)), [('Pass', (4, 2))], [])])]), +('Module', [('If', (1, 0), ('Name', (1, 3), 'a', ('Load',)), [('Pass', (2, 2))], [('If', (3, 0), ('Name', (3, 5), 'b', ('Load',)), [('Pass', (4, 2))], [('Pass', (6, 2))])])]), ('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]), ('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]), ('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], []), None)]), @@ -1187,6 +1281,10 @@ exec_results = [ ('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Num', (1, 10), 2)], [('Dict', (1, 3), [('Num', (1, 4), 1)], [('Num', (1, 6), 2)]), ('Num', (1, 12), 3)]))]), ('Module', [('Expr', (1, 0), ('Set', (1, 0), [('Starred', (1, 1), ('Set', (1, 2), [('Num', (1, 3), 1), ('Num', (1, 6), 2)]), ('Load',)), ('Num', (1, 10), 3)]))]), ('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('ListComp', (2, 2), ('Name', (2, 2), 'i', ('Load',)), [('comprehension', ('Name', (2, 14), 'b', ('Store',)), ('Name', (2, 19), 'c', ('Load',)), [], 1)]))], [], None)]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (4, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 1), ('Name', (2, 1), 'deco2', ('Load',)), [], []), ('Call', (3, 1), ('Name', (3, 1), 'deco3', ('Load',)), [('Num', (3, 7), 1)], [])], None)]), +('Module', [('AsyncFunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (4, 15))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 1), ('Name', (2, 1), 'deco2', ('Load',)), [], []), ('Call', (3, 1), ('Name', (3, 1), 'deco3', ('Load',)), [('Num', (3, 7), 1)], [])], None)]), +('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (4, 9))], [('Name', (1, 1), 'deco1', ('Load',)), ('Call', (2, 1), ('Name', (2, 1), 'deco2', ('Load',)), [], []), ('Call', (3, 1), ('Name', (3, 1), 'deco3', ('Load',)), [('Num', (3, 7), 1)], [])])]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (2, 9))], [('Call', (1, 1), ('Name', (1, 1), 'deco', ('Load',)), [('GeneratorExp', (1, 6), ('Name', (1, 6), 'a', ('Load',)), [('comprehension', ('Name', (1, 12), 'a', ('Store',)), ('Name', (1, 17), 'b', ('Load',)), [], 0)])], [])], None)]), ] single_results = [ ('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]), diff --git a/lib-python/3/test/test_asyncgen.py b/lib-python/3/test/test_asyncgen.py index 5a36423dc9..5da27417f2 100644 --- a/lib-python/3/test/test_asyncgen.py +++ b/lib-python/3/test/test_asyncgen.py @@ -111,6 +111,31 @@ class AsyncGenTest(unittest.TestCase): def async_iterate(g): res = [] while True: + an = g.__anext__() + try: + while True: + try: + an.__next__() + except StopIteration as ex: + if ex.args: + res.append(ex.args[0]) + break + else: + res.append('EMPTY StopIteration') + break + except StopAsyncIteration: + raise + except Exception as ex: + res.append(str(type(ex))) + break + except StopAsyncIteration: + res.append('STOP') + break + return res + + def async_iterate(g): + res = [] + while True: try: g.__anext__().__next__() except StopAsyncIteration: @@ -297,6 +322,37 @@ class AsyncGenTest(unittest.TestCase): "non-None value .* async generator"): gen().__anext__().send(100) + def test_async_gen_exception_11(self): + def sync_gen(): + yield 10 + yield 20 + + def sync_gen_wrapper(): + yield 1 + sg = sync_gen() + sg.send(None) + try: + sg.throw(GeneratorExit()) + except GeneratorExit: + yield 2 + yield 3 + + async def async_gen(): + yield 10 + yield 20 + + async def async_gen_wrapper(): + yield 1 + asg = async_gen() + await asg.asend(None) + try: + await asg.athrow(GeneratorExit()) + except GeneratorExit: + yield 2 + yield 3 + + self.compare_generators(sync_gen_wrapper(), async_gen_wrapper()) + def test_async_gen_api_01(self): async def gen(): yield 123 @@ -696,6 +752,33 @@ class AsyncGenAsyncioTest(unittest.TestCase): self.loop.run_until_complete(run()) self.assertEqual(DONE, 10) + def test_async_gen_asyncio_aclose_12(self): + DONE = 0 + + async def target(): + await asyncio.sleep(0.01) + 1 / 0 + + async def foo(): + nonlocal DONE + task = asyncio.create_task(target()) + try: + yield 1 + finally: + try: + await task + except ZeroDivisionError: + DONE = 1 + + async def run(): + gen = foo() + it = gen.__aiter__() + await it.__anext__() + await gen.aclose() + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + def test_async_gen_asyncio_asend_01(self): DONE = 0 @@ -1068,6 +1151,90 @@ class AsyncGenAsyncioTest(unittest.TestCase): res = self.loop.run_until_complete(run()) self.assertEqual(res, [i * 2 for i in range(1, 10)]) + def test_asyncgen_nonstarted_hooks_are_cancellable(self): + # See https://bugs.python.org/issue38013 + messages = [] + + def exception_handler(loop, context): + messages.append(context) + + async def async_iterate(): + yield 1 + yield 2 + + async def main(): + loop = asyncio.get_running_loop() + loop.set_exception_handler(exception_handler) + + async for i in async_iterate(): + break + + asyncio.run(main()) + + self.assertEqual([], messages) + + def test_async_gen_await_same_anext_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + nxt = it.__anext__() + await nxt + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)" + ): + await nxt + + await it.aclose() # prevent unfinished iterator warning + + self.loop.run_until_complete(run()) + + def test_async_gen_await_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + nxt = it.aclose() + await nxt + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + await nxt + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_twice_with_different_coros(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + await it.aclose() + await it.aclose() + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_after_exhaustion(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + async for _ in it: + pass + await it.aclose() + + self.loop.run_until_complete(run()) if __name__ == "__main__": unittest.main() diff --git a/lib-python/3/test/test_asyncio/test_base_events.py b/lib-python/3/test/test_asyncio/test_base_events.py index 178ffa69d4..5025d260d4 100644 --- a/lib-python/3/test/test_asyncio/test_base_events.py +++ b/lib-python/3/test/test_asyncio/test_base_events.py @@ -1709,10 +1709,6 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): self.assertRaises(ValueError, self.loop.run_until_complete, fut) fut = self.loop.create_datagram_endpoint( - MyDatagramProto, reuse_address=True, sock=FakeSock()) - self.assertRaises(ValueError, self.loop.run_until_complete, fut) - - fut = self.loop.create_datagram_endpoint( MyDatagramProto, reuse_port=True, sock=FakeSock()) self.assertRaises(ValueError, self.loop.run_until_complete, fut) @@ -1722,7 +1718,6 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): def test_create_datagram_endpoint_sockopts(self): # Socket options should not be applied unless asked for. - # SO_REUSEADDR defaults to on for UNIX. # SO_REUSEPORT is not available on all platforms. coro = self.loop.create_datagram_endpoint( @@ -1731,18 +1726,8 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): transport, protocol = self.loop.run_until_complete(coro) sock = transport.get_extra_info('socket') - reuse_address_default_on = ( - os.name == 'posix' and sys.platform != 'cygwin') reuseport_supported = hasattr(socket, 'SO_REUSEPORT') - if reuse_address_default_on: - self.assertTrue( - sock.getsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR)) - else: - self.assertFalse( - sock.getsockopt( - socket.SOL_SOCKET, socket.SO_REUSEADDR)) if reuseport_supported: self.assertFalse( sock.getsockopt( @@ -1758,13 +1743,12 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): coro = self.loop.create_datagram_endpoint( lambda: MyDatagramProto(create_future=True, loop=self.loop), local_addr=('127.0.0.1', 0), - reuse_address=True, reuse_port=reuseport_supported, allow_broadcast=True) transport, protocol = self.loop.run_until_complete(coro) sock = transport.get_extra_info('socket') - self.assertTrue( + self.assertFalse( sock.getsockopt( socket.SOL_SOCKET, socket.SO_REUSEADDR)) if reuseport_supported: @@ -1779,6 +1763,32 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): self.loop.run_until_complete(protocol.done) self.assertEqual('CLOSED', protocol.state) + def test_create_datagram_endpoint_reuse_address_error(self): + # bpo-37228: Ensure that explicit passing of `reuse_address=True` + # raises an error, as it is not safe to use SO_REUSEADDR when using UDP + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + local_addr=('127.0.0.1', 0), + reuse_address=True) + + with self.assertRaises(ValueError): + self.loop.run_until_complete(coro) + + def test_create_datagram_endpoint_reuse_address_warning(self): + # bpo-37228: Deprecate *reuse_address* parameter + + coro = self.loop.create_datagram_endpoint( + lambda: MyDatagramProto(create_future=True, loop=self.loop), + local_addr=('127.0.0.1', 0), + reuse_address=False) + + with self.assertWarns(DeprecationWarning): + transport, protocol = self.loop.run_until_complete(coro) + transport.close() + self.loop.run_until_complete(protocol.done) + self.assertEqual('CLOSED', protocol.state) + @patch_socket def test_create_datagram_endpoint_nosoreuseport(self, m_socket): del m_socket.SO_REUSEPORT @@ -1787,7 +1797,6 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): coro = self.loop.create_datagram_endpoint( lambda: MyDatagramProto(loop=self.loop), local_addr=('127.0.0.1', 0), - reuse_address=False, reuse_port=True) self.assertRaises(ValueError, self.loop.run_until_complete, coro) @@ -1806,7 +1815,6 @@ class BaseEventLoopWithSelectorTests(test_utils.TestCase): coro = self.loop.create_datagram_endpoint( lambda: MyDatagramProto(loop=self.loop), local_addr=('1.2.3.4', 0), - reuse_address=False, reuse_port=reuseport_supported) t, p = self.loop.run_until_complete(coro) diff --git a/lib-python/3/test/test_asyncio/test_context.py b/lib-python/3/test/test_asyncio/test_context.py index 6abddd9f25..728ce808a2 100644 --- a/lib-python/3/test/test_asyncio/test_context.py +++ b/lib-python/3/test/test_asyncio/test_context.py @@ -3,6 +3,7 @@ import decimal import unittest +@unittest.skipUnless(decimal.HAVE_CONTEXTVAR, "decimal is built with a thread-local context") class DecimalContextTest(unittest.TestCase): def test_asyncio_task_decimal_context(self): diff --git a/lib-python/3/test/test_asyncio/test_futures.py b/lib-python/3/test/test_asyncio/test_futures.py index 9608a3a81c..8bc861d374 100644 --- a/lib-python/3/test/test_asyncio/test_futures.py +++ b/lib-python/3/test/test_asyncio/test_futures.py @@ -818,5 +818,44 @@ class PyFutureDoneCallbackTests(BaseFutureDoneCallbackTests, return futures._PyFuture(loop=self.loop) +class BaseFutureInheritanceTests: + + def _get_future_cls(self): + raise NotImplementedError + + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.addCleanup(self.loop.close) + + def test_inherit_without_calling_super_init(self): + # See https://bugs.python.org/issue38785 for the context + cls = self._get_future_cls() + + class MyFut(cls): + def __init__(self, *args, **kwargs): + # don't call super().__init__() + pass + + fut = MyFut(loop=self.loop) + with self.assertRaisesRegex( + RuntimeError, + "Future object is not initialized." + ): + fut.get_loop() + + +class PyFutureInheritanceTests(BaseFutureInheritanceTests, + test_utils.TestCase): + def _get_future_cls(self): + return futures._PyFuture + + +class CFutureInheritanceTests(BaseFutureInheritanceTests, + test_utils.TestCase): + def _get_future_cls(self): + return futures._CFuture + + if __name__ == '__main__': unittest.main() diff --git a/lib-python/3/test/test_asyncio/test_sslproto.py b/lib-python/3/test/test_asyncio/test_sslproto.py index 866ef81fb2..8720961442 100644 --- a/lib-python/3/test/test_asyncio/test_sslproto.py +++ b/lib-python/3/test/test_asyncio/test_sslproto.py @@ -495,14 +495,6 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): server_context = test_utils.simple_server_sslcontext() client_context = test_utils.simple_client_sslcontext() - if (sys.platform.startswith('freebsd') - or sys.platform.startswith('win') - or sys.platform.startswith('darwin')): - # bpo-35031: Some FreeBSD and Windows buildbots fail to run this test - # as the eof was not being received by the server if the payload - # size is not big enough. This behaviour only appears if the - # client is using TLS1.3. Also seen on macOS. - client_context.options |= ssl.OP_NO_TLSv1_3 answer = None def client(sock, addr): @@ -519,9 +511,10 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): sock.close() class ServerProto(asyncio.Protocol): - def __init__(self, on_con, on_con_lost): + def __init__(self, on_con, on_con_lost, on_got_hello): self.on_con = on_con self.on_con_lost = on_con_lost + self.on_got_hello = on_got_hello self.data = b'' self.transport = None @@ -535,7 +528,7 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): def data_received(self, data): self.data += data if len(self.data) >= len(HELLO_MSG): - self.transport.write(ANSWER) + self.on_got_hello.set_result(None) def connection_lost(self, exc): self.transport = None @@ -544,7 +537,7 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): else: self.on_con_lost.set_exception(exc) - async def main(proto, on_con, on_con_lost): + async def main(proto, on_con, on_con_lost, on_got_hello): tr = await on_con tr.write(HELLO_MSG) @@ -554,9 +547,11 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): tr, proto, server_context, server_side=True, ssl_handshake_timeout=self.TIMEOUT) - proto.replace_transport(new_tr) + await on_got_hello + new_tr.write(ANSWER) + await on_con_lost self.assertEqual(proto.data, HELLO_MSG) new_tr.close() @@ -564,7 +559,8 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): async def run_main(): on_con = self.loop.create_future() on_con_lost = self.loop.create_future() - proto = ServerProto(on_con, on_con_lost) + on_got_hello = self.loop.create_future() + proto = ServerProto(on_con, on_con_lost, on_got_hello) server = await self.loop.create_server( lambda: proto, '127.0.0.1', 0) @@ -573,7 +569,7 @@ class BaseStartTLS(func_tests.FunctionalTestCaseMixin): with self.tcp_client(lambda sock: client(sock, addr), timeout=self.TIMEOUT): await asyncio.wait_for( - main(proto, on_con, on_con_lost), + main(proto, on_con, on_con_lost, on_got_hello), loop=self.loop, timeout=self.TIMEOUT) server.close() diff --git a/lib-python/3/test/test_asyncio/test_unix_events.py b/lib-python/3/test/test_asyncio/test_unix_events.py index ec171fa83d..66a6bc1680 100644 --- a/lib-python/3/test/test_asyncio/test_unix_events.py +++ b/lib-python/3/test/test_asyncio/test_unix_events.py @@ -722,6 +722,7 @@ class UnixReadPipeTransportTests(test_utils.TestCase): @mock.patch('os.read') def test_resume_reading(self, m_read): tr = self.read_pipe_transport() + tr.pause_reading() tr.resume_reading() self.loop.assert_reader(5, tr._read_ready) @@ -776,6 +777,32 @@ class UnixReadPipeTransportTests(test_utils.TestCase): self.assertIsNone(tr._protocol) self.assertIsNone(tr._loop) + def test_pause_reading_on_closed_pipe(self): + tr = self.read_pipe_transport() + tr.close() + test_utils.run_briefly(self.loop) + self.assertIsNone(tr._loop) + tr.pause_reading() + + def test_pause_reading_on_paused_pipe(self): + tr = self.read_pipe_transport() + tr.pause_reading() + # the second call should do nothing + tr.pause_reading() + + def test_resume_reading_on_closed_pipe(self): + tr = self.read_pipe_transport() + tr.close() + test_utils.run_briefly(self.loop) + self.assertIsNone(tr._loop) + tr.resume_reading() + + def test_resume_reading_on_paused_pipe(self): + tr = self.read_pipe_transport() + # the pipe is not paused + # resuming should do nothing + tr.resume_reading() + class UnixWritePipeTransportTests(test_utils.TestCase): diff --git a/lib-python/3/test/test_base64.py b/lib-python/3/test/test_base64.py index 2a4cc2acad..7dba6635d4 100644 --- a/lib-python/3/test/test_base64.py +++ b/lib-python/3/test/test_base64.py @@ -250,6 +250,7 @@ class BaseXYTestCase(unittest.TestCase): (b'3d}==', b'\xdd'), (b'@@', b''), (b'!', b''), + (b"YWJj\n", b"abc"), (b'YWJj\nYWI=', b'abcab')) funcs = ( base64.b64decode, diff --git a/lib-python/3/test/test_buffer.py b/lib-python/3/test/test_buffer.py index f302da415d..2ae5ffaf61 100644 --- a/lib-python/3/test/test_buffer.py +++ b/lib-python/3/test/test_buffer.py @@ -10,6 +10,8 @@ # the same way as the original. Thus, a substantial part of the # memoryview tests is now in this module. # +# Written and designed by Stefan Krah for Python 3.3. +# import contextlib import unittest @@ -2728,6 +2730,10 @@ class TestBufferProtocol(unittest.TestCase): # be 1D, at least one format must be 'c', 'b' or 'B'. for _tshape in gencastshapes(): for char in fmtdict['@']: + # Casts to _Bool are undefined if the source contains values + # other than 0 or 1. + if char == "?": + continue tfmt = ('', '@')[randrange(2)] + char tsize = struct.calcsize(tfmt) n = prod(_tshape) * tsize diff --git a/lib-python/3/test/test_builtin.py b/lib-python/3/test/test_builtin.py index c3f04ec744..40f83de57d 100644 --- a/lib-python/3/test/test_builtin.py +++ b/lib-python/3/test/test_builtin.py @@ -154,6 +154,11 @@ class BuiltinTest(unittest.TestCase): self.assertRaises(TypeError, __import__, 1, 2, 3, 4) self.assertRaises(ValueError, __import__, '') self.assertRaises(TypeError, __import__, 'sys', name='sys') + # Relative import outside of a package with no __package__ or __spec__ (bpo-37409). + with self.assertWarns(ImportWarning): + self.assertRaises(ImportError, __import__, '', + {'__package__': None, '__spec__': None, '__name__': '__main__'}, + locals={}, fromlist=('foo',), level=1) # embedded null character self.assertRaises(ModuleNotFoundError, __import__, 'string\x00') @@ -1511,6 +1516,11 @@ class BuiltinTest(unittest.TestCase): self.assertRaises(ValueError, x.translate, b"1", 1) self.assertRaises(TypeError, x.translate, b"1"*256, 1) + def test_bytearray_extend_error(self): + array = bytearray() + bad_iter = map(int, "X") + self.assertRaises(ValueError, array.extend, bad_iter) + def test_construct_singletons(self): for const in None, Ellipsis, NotImplemented: tp = type(const) @@ -1629,7 +1639,21 @@ class PtyTests(unittest.TestCase): """Tests that use a pseudo terminal to guarantee stdin and stdout are terminals in the test environment""" + @staticmethod + def handle_sighup(signum, frame): + # bpo-40140: if the process is the session leader, os.close(fd) + # of "pid, fd = pty.fork()" can raise SIGHUP signal: + # just ignore the signal. + pass + def run_child(self, child, terminal_input): + old_sighup = signal.signal(signal.SIGHUP, self.handle_sighup) + try: + return self._run_child(child, terminal_input) + finally: + signal.signal(signal.SIGHUP, old_sighup) + + def _run_child(self, child, terminal_input): r, w = os.pipe() # Pipe test results from child back to parent try: pid, fd = pty.fork() @@ -1680,6 +1704,9 @@ class PtyTests(unittest.TestCase): child_output = child_output.decode("ascii", "ignore") self.fail("got %d lines in pipe but expected 2, child output was:\n%s" % (len(lines), child_output)) + + # bpo-40155: Close the PTY before waiting for the child process + # completion, otherwise the child process hangs on AIX. os.close(fd) # Wait until the child process completes diff --git a/lib-python/3/test/test_c_locale_coercion.py b/lib-python/3/test/test_c_locale_coercion.py index 134f07a17e..0355547436 100644 --- a/lib-python/3/test/test_c_locale_coercion.py +++ b/lib-python/3/test/test_c_locale_coercion.py @@ -100,11 +100,11 @@ _EncodingDetails = namedtuple("EncodingDetails", _fields) class EncodingDetails(_EncodingDetails): # XXX (ncoghlan): Using JSON for child state reporting may be less fragile CHILD_PROCESS_SCRIPT = ";".join([ - "import sys, os", - "print(sys.getfilesystemencoding())", - "print(sys.stdin.encoding + ':' + sys.stdin.errors)", - "print(sys.stdout.encoding + ':' + sys.stdout.errors)", - "print(sys.stderr.encoding + ':' + sys.stderr.errors)", + "import sys, os, codecs", + "print(codecs.lookup(sys.getfilesystemencoding()).name)", + "print(codecs.lookup(sys.stdin.encoding).name + ':' + sys.stdin.errors)", + "print(codecs.lookup(sys.stdout.encoding).name + ':' + sys.stdout.errors)", + "print(codecs.lookup(sys.stderr.encoding).name + ':' + sys.stderr.errors)", "print(os.environ.get('LANG', 'not set'))", "print(os.environ.get('LC_CTYPE', 'not set'))", "print(os.environ.get('LC_ALL', 'not set'))", @@ -119,28 +119,15 @@ class EncodingDetails(_EncodingDetails): stream_info = 2*[_stream.format("surrogateescape")] # stderr should always use backslashreplace stream_info.append(_stream.format("backslashreplace")) - expected_lang = env_vars.get("LANG", "not set").lower() + expected_lang = env_vars.get("LANG", "not set") if coercion_expected: - expected_lc_ctype = CLI_COERCION_TARGET.lower() + expected_lc_ctype = CLI_COERCION_TARGET else: - expected_lc_ctype = env_vars.get("LC_CTYPE", "not set").lower() - expected_lc_all = env_vars.get("LC_ALL", "not set").lower() + expected_lc_ctype = env_vars.get("LC_CTYPE", "not set") + expected_lc_all = env_vars.get("LC_ALL", "not set") env_info = expected_lang, expected_lc_ctype, expected_lc_all return dict(cls(fs_encoding, *stream_info, *env_info)._asdict()) - @staticmethod - def _handle_output_variations(data): - """Adjust the output to handle platform specific idiosyncrasies - - * Some platforms report ASCII as ANSI_X3.4-1968 - * Some platforms report ASCII as US-ASCII - * Some platforms report UTF-8 instead of utf-8 - """ - data = data.replace(b"ANSI_X3.4-1968", b"ascii") - data = data.replace(b"US-ASCII", b"ascii") - data = data.lower() - return data - @classmethod def get_child_details(cls, env_vars): """Retrieves fsencoding and standard stream details from a child process @@ -160,8 +147,7 @@ class EncodingDetails(_EncodingDetails): if not result.rc == 0: result.fail(py_cmd) # All subprocess outputs in this test case should be pure ASCII - adjusted_output = cls._handle_output_variations(result.out) - stdout_lines = adjusted_output.decode("ascii").splitlines() + stdout_lines = result.out.decode("ascii").splitlines() child_encoding_details = dict(cls(*stdout_lines)._asdict()) stderr_lines = result.err.decode("ascii").rstrip().splitlines() return child_encoding_details, stderr_lines diff --git a/lib-python/3/test/test_capi.py b/lib-python/3/test/test_capi.py index d94ee0227c..3ed2263fda 100644 --- a/lib-python/3/test/test_capi.py +++ b/lib-python/3/test/test_capi.py @@ -315,6 +315,20 @@ class CAPITest(unittest.TestCase): self.assertRaises(TypeError, _testcapi.get_mapping_values, bad_mapping) self.assertRaises(TypeError, _testcapi.get_mapping_items, bad_mapping) + def test_pynumber_tobase(self): + from _testcapi import pynumber_tobase + self.assertEqual(pynumber_tobase(123, 2), '0b1111011') + self.assertEqual(pynumber_tobase(123, 8), '0o173') + self.assertEqual(pynumber_tobase(123, 10), '123') + self.assertEqual(pynumber_tobase(123, 16), '0x7b') + self.assertEqual(pynumber_tobase(-123, 2), '-0b1111011') + self.assertEqual(pynumber_tobase(-123, 8), '-0o173') + self.assertEqual(pynumber_tobase(-123, 10), '-123') + self.assertEqual(pynumber_tobase(-123, 16), '-0x7b') + self.assertRaises(TypeError, pynumber_tobase, 123.0, 10) + self.assertRaises(TypeError, pynumber_tobase, '123', 10) + self.assertRaises(SystemError, pynumber_tobase, 123, 0) + class TestPendingCalls(unittest.TestCase): diff --git a/lib-python/3/test/test_cgi.py b/lib-python/3/test/test_cgi.py index f4e00c7a72..220268e14f 100644 --- a/lib-python/3/test/test_cgi.py +++ b/lib-python/3/test/test_cgi.py @@ -130,6 +130,20 @@ class CgiTests(unittest.TestCase): 'file': [b'Testing 123.\n'], 'title': ['']} self.assertEqual(result, expected) + def test_parse_multipart_without_content_length(self): + POSTDATA = '''--JfISa01 +Content-Disposition: form-data; name="submit-name" + +just a string + +--JfISa01-- +''' + fp = BytesIO(POSTDATA.encode('latin1')) + env = {'boundary': 'JfISa01'.encode('latin1')} + result = cgi.parse_multipart(fp, env) + expected = {'submit-name': ['just a string\n']} + self.assertEqual(result, expected) + def test_parse_multipart_invalid_encoding(self): BOUNDARY = "JfISa01" POSTDATA = """--JfISa01 @@ -363,6 +377,23 @@ Larry self.assertEqual(fs.list[0].name, 'submit-name') self.assertEqual(fs.list[0].value, 'Larry') + def test_field_storage_multipart_no_content_length(self): + fp = BytesIO(b"""--MyBoundary +Content-Disposition: form-data; name="my-arg"; filename="foo" + +Test + +--MyBoundary-- +""") + env = { + "REQUEST_METHOD": "POST", + "CONTENT_TYPE": "multipart/form-data; boundary=MyBoundary", + "wsgi.input": fp, + } + fields = cgi.FieldStorage(fp, environ=env) + + self.assertEqual(len(fields["my-arg"].file.read()), 5) + def test_fieldstorage_as_context_manager(self): fp = BytesIO(b'x' * 10) env = {'REQUEST_METHOD': 'PUT'} diff --git a/lib-python/3/test/test_codecs.py b/lib-python/3/test/test_codecs.py index b39ea54e4b..ea9e719e74 100644 --- a/lib-python/3/test/test_codecs.py +++ b/lib-python/3/test/test_codecs.py @@ -1206,6 +1206,7 @@ class UTF8SigTest(UTF8Test, unittest.TestCase): got = ostream.getvalue() self.assertEqual(got, unistring) + class EscapeDecodeTest(unittest.TestCase): def test_empty(self): self.assertEqual(codecs.escape_decode(b""), (b"", 0)) @@ -1406,6 +1407,18 @@ class PunycodeTest(unittest.TestCase): puny = puny.decode("ascii").encode("ascii") self.assertEqual(uni, puny.decode("punycode")) + def test_decode_invalid(self): + testcases = [ + (b"xn--w&", "strict", UnicodeError()), + (b"xn--w&", "ignore", "xn-"), + ] + for puny, errors, expected in testcases: + with self.subTest(puny=puny, errors=errors): + if isinstance(expected, Exception): + self.assertRaises(UnicodeError, puny.decode, "punycode", errors) + else: + self.assertEqual(puny.decode("punycode", errors), expected) + class UnicodeInternalTest(unittest.TestCase): @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') @@ -1857,6 +1870,14 @@ class CodecsModuleTest(unittest.TestCase): self.assertRaises(UnicodeError, codecs.decode, b'abc', 'undefined', errors) + def test_file_closes_if_lookup_error_raised(self): + mock_open = mock.mock_open() + with mock.patch('builtins.open', mock_open) as file: + with self.assertRaises(LookupError): + codecs.open(support.TESTFN, 'wt', 'invalid-encoding') + + file().close.assert_called() + class StreamReaderTest(unittest.TestCase): @@ -3229,13 +3250,13 @@ class CodePageTest(unittest.TestCase): self.assertEqual(codec.name, 'mbcs') @support.bigmemtest(size=2**31, memuse=7, dry_run=False) - def test_large_input(self): + def test_large_input(self, size): # Test input longer than INT_MAX. # Input should contain undecodable bytes before and after # the INT_MAX limit. - encoded = (b'01234567' * (2**28-1) + + encoded = (b'01234567' * ((size//8)-1) + b'\x85\x86\xea\xeb\xec\xef\xfc\xfd\xfe\xff') - self.assertEqual(len(encoded), 2**31+2) + self.assertEqual(len(encoded), size+2) decoded = codecs.code_page_decode(932, encoded, 'surrogateescape', True) self.assertEqual(decoded[1], len(encoded)) del encoded @@ -3246,6 +3267,20 @@ class CodePageTest(unittest.TestCase): '\udc85\udc86\udcea\udceb\udcec' '\udcef\udcfc\udcfd\udcfe\udcff') + @support.bigmemtest(size=2**31, memuse=6, dry_run=False) + def test_large_utf8_input(self, size): + # Test input longer than INT_MAX. + # Input should contain a decodable multi-byte character + # surrounding INT_MAX + encoded = (b'0123456\xed\x84\x80' * (size//8)) + self.assertEqual(len(encoded), size // 8 * 10) + decoded = codecs.code_page_decode(65001, encoded, 'ignore', True) + self.assertEqual(decoded[1], len(encoded)) + del encoded + self.assertEqual(len(decoded[0]), size) + self.assertEqual(decoded[0][:10], '0123456\ud10001') + self.assertEqual(decoded[0][-11:], '56\ud1000123456\ud100') + class ASCIITest(unittest.TestCase): def test_encode(self): diff --git a/lib-python/3/test/test_codeop.py b/lib-python/3/test/test_codeop.py index 98da26fa5d..8e278b9b23 100644 --- a/lib-python/3/test/test_codeop.py +++ b/lib-python/3/test/test_codeop.py @@ -3,12 +3,12 @@ Nick Mathewson """ import unittest -from test.support import is_jython +from test import support from codeop import compile_command, PyCF_DONT_IMPLY_DEDENT import io -if is_jython: +if support.is_jython: import sys def unify_callables(d): @@ -21,7 +21,7 @@ class CodeopTests(unittest.TestCase): def assertValid(self, str, symbol='single'): '''succeed iff str is a valid piece of code''' - if is_jython: + if support.is_jython: code = compile_command(str, "<input>", symbol) self.assertTrue(code) if symbol == "single": @@ -60,7 +60,7 @@ class CodeopTests(unittest.TestCase): av = self.assertValid # special case - if not is_jython: + if not support.is_jython: self.assertEqual(compile_command(""), compile("pass", "<input>", 'single', PyCF_DONT_IMPLY_DEDENT)) @@ -294,6 +294,11 @@ class CodeopTests(unittest.TestCase): self.assertNotEqual(compile_command("a = 1\n", "abc").co_filename, compile("a = 1\n", "def", 'single').co_filename) + def test_warning(self): + # Test that the warning is only returned once. + with support.check_warnings((".*invalid", DeprecationWarning)) as w: + compile_command("'\e'") + self.assertEqual(len(w.warnings), 1) if __name__ == "__main__": unittest.main() diff --git a/lib-python/3/test/test_compileall.py b/lib-python/3/test/test_compileall.py index 2e2552303f..07c31cedb1 100644 --- a/lib-python/3/test/test_compileall.py +++ b/lib-python/3/test/test_compileall.py @@ -578,6 +578,47 @@ class CommandLineTestsBase: self.assertTrue(compile_dir.called) self.assertEqual(compile_dir.call_args[-1]['workers'], None) + def _test_ddir_only(self, *, ddir, parallel=True): + """Recursive compile_dir ddir must contain package paths; bpo39769.""" + fullpath = ["test", "foo"] + path = self.directory + mods = [] + for subdir in fullpath: + path = os.path.join(path, subdir) + os.mkdir(path) + script_helper.make_script(path, "__init__", "") + mods.append(script_helper.make_script(path, "mod", + "def fn(): 1/0\nfn()\n")) + compileall.compile_dir( + self.directory, quiet=True, ddir=ddir, + workers=2 if parallel else 1) + self.assertTrue(mods) + for mod in mods: + self.assertTrue(mod.startswith(self.directory), mod) + modcode = importlib.util.cache_from_source(mod) + modpath = mod[len(self.directory+os.sep):] + _, _, err = script_helper.assert_python_failure(modcode) + expected_in = os.path.join(ddir, modpath) + mod_code_obj = test.test_importlib.util._get_code_from_pyc(modcode) + self.assertEqual(mod_code_obj.co_filename, expected_in) + self.assertIn(f'"{expected_in}"', os.fsdecode(err)) + + def test_ddir_only_one_worker(self): + """Recursive compile_dir ddir= contains package paths; bpo39769.""" + return self._test_ddir_only(ddir="<a prefix>", parallel=False) + + def test_ddir_multiple_workers(self): + """Recursive compile_dir ddir= contains package paths; bpo39769.""" + return self._test_ddir_only(ddir="<a prefix>", parallel=True) + + def test_ddir_empty_only_one_worker(self): + """Recursive compile_dir ddir='' contains package paths; bpo39769.""" + return self._test_ddir_only(ddir="", parallel=False) + + def test_ddir_empty_multiple_workers(self): + """Recursive compile_dir ddir='' contains package paths; bpo39769.""" + return self._test_ddir_only(ddir="", parallel=True) + class CommmandLineTestsWithSourceEpoch(CommandLineTestsBase, unittest.TestCase, diff --git a/lib-python/3/test/test_concurrent_futures.py b/lib-python/3/test/test_concurrent_futures.py index ad68909161..a261bfeb4d 100644 --- a/lib-python/3/test/test_concurrent_futures.py +++ b/lib-python/3/test/test_concurrent_futures.py @@ -26,6 +26,7 @@ from concurrent.futures._base import ( BrokenExecutor) from concurrent.futures.process import BrokenProcessPool from multiprocessing import get_context +import multiprocessing.util def create_future(state=PENDING, exception=None, result=None): @@ -84,8 +85,7 @@ class MyObject(object): class EventfulGCObj(): - def __init__(self, ctx): - mgr = get_context(ctx).Manager() + def __init__(self, mgr): self.event = mgr.Event() def __del__(self): @@ -818,12 +818,21 @@ class ProcessPoolExecutorTest(ExecutorTest): def test_ressources_gced_in_workers(self): # Ensure that argument for a job are correctly gc-ed after the job # is finished - obj = EventfulGCObj(self.ctx) + mgr = get_context(self.ctx).Manager() + obj = EventfulGCObj(mgr) future = self.executor.submit(id, obj) future.result() self.assertTrue(obj.event.wait(timeout=1)) + # explicitly destroy the object to ensure that EventfulGCObj.__del__() + # is called while manager is still running. + obj = None + test.support.gc_collect() + + mgr.shutdown() + mgr.join() + create_executor_tests(ProcessPoolExecutorTest, executor_mixins=(ProcessPoolForkMixin, @@ -1245,6 +1254,7 @@ def test_main(): test.support.run_unittest(__name__) finally: test.support.reap_children() + multiprocessing.util._cleanup_tests() if __name__ == "__main__": test_main() diff --git a/lib-python/3/test/test_context.py b/lib-python/3/test/test_context.py index efd7319a23..b9e991a400 100644 --- a/lib-python/3/test/test_context.py +++ b/lib-python/3/test/test_context.py @@ -38,9 +38,6 @@ class ContextTest(unittest.TestCase): self.assertNotEqual(hash(c), hash('aaa')) - def test_context_var_new_2(self): - self.assertIsNone(contextvars.ContextVar[int]) - @isolated_context def test_context_var_repr_1(self): c = contextvars.ContextVar('a') @@ -361,6 +358,10 @@ class ContextTest(unittest.TestCase): tp.shutdown() self.assertEqual(results, list(range(10))) + def test_contextvar_getitem(self): + clss = contextvars.ContextVar + self.assertEqual(clss[str], clss) + # HAMT Tests diff --git a/lib-python/3/test/test_contextlib_async.py b/lib-python/3/test/test_contextlib_async.py index cc38dcf8c4..9db40652f4 100644 --- a/lib-python/3/test/test_contextlib_async.py +++ b/lib-python/3/test/test_contextlib_async.py @@ -36,6 +36,28 @@ class TestAbstractAsyncContextManager(unittest.TestCase): async with manager as context: self.assertIs(manager, context) + @_async_test + async def test_async_gen_propagates_generator_exit(self): + # A regression test for https://bugs.python.org/issue33786. + + @asynccontextmanager + async def ctx(): + yield + + async def gen(): + async with ctx(): + yield 11 + + ret = [] + exc = ValueError(22) + with self.assertRaises(ValueError): + async with ctx(): + async for val in gen(): + ret.append(val) + raise exc + + self.assertEqual(ret, [11]) + def test_exit_is_abstract(self): class MissingAexit(AbstractAsyncContextManager): pass diff --git a/lib-python/3/test/test_copy.py b/lib-python/3/test/test_copy.py index 45a692022f..35f72fb216 100644 --- a/lib-python/3/test/test_copy.py +++ b/lib-python/3/test/test_copy.py @@ -99,7 +99,7 @@ class TestCopy(unittest.TestCase): 42, 2**100, 3.14, True, False, 1j, "hello", "hello\u1234", f.__code__, b"world", bytes(range(256)), range(10), slice(1, 10, 2), - NewStyle, Classic, max, WithMetaclass] + NewStyle, Classic, max, WithMetaclass, property()] for x in tests: self.assertIs(copy.copy(x), x) @@ -357,7 +357,7 @@ class TestCopy(unittest.TestCase): pass tests = [None, 42, 2**100, 3.14, True, False, 1j, "hello", "hello\u1234", f.__code__, - NewStyle, Classic, max] + NewStyle, Classic, max, property()] for x in tests: self.assertIs(copy.deepcopy(x), x) diff --git a/lib-python/3/test/test_dataclasses.py b/lib-python/3/test/test_dataclasses.py index a2c7ceea9d..48f48971b3 100755 --- a/lib-python/3/test/test_dataclasses.py +++ b/lib-python/3/test/test_dataclasses.py @@ -10,6 +10,7 @@ import builtins import unittest from unittest.mock import Mock from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional +from typing import get_type_hints from collections import deque, OrderedDict, namedtuple from functools import total_ordering @@ -44,6 +45,25 @@ class TestCase(unittest.TestCase): o = C(42) self.assertEqual(o.x, 42) + def test_field_default_default_factory_error(self): + msg = "cannot specify both default and default_factory" + with self.assertRaisesRegex(ValueError, msg): + @dataclass + class C: + x: int = field(default=1, default_factory=int) + + def test_field_repr(self): + int_field = field(default=1, init=True, repr=False) + int_field.name = "id" + repr_output = repr(int_field) + expected_output = "Field(name='id',type=None," \ + f"default=1,default_factory={MISSING!r}," \ + "init=True,repr=False,hash=None," \ + "compare=True,metadata=mappingproxy({})," \ + "_field_type=None)" + + self.assertEqual(repr_output, expected_output) + def test_named_init_params(self): @dataclass class C: @@ -1294,6 +1314,32 @@ class TestCase(unittest.TestCase): self.assertTrue(is_dataclass(d.d)) self.assertFalse(is_dataclass(d.e)) + def test_is_dataclass_when_getattr_always_returns(self): + # See bpo-37868. + class A: + def __getattr__(self, key): + return 0 + self.assertFalse(is_dataclass(A)) + a = A() + + # Also test for an instance attribute. + class B: + pass + b = B() + b.__dataclass_fields__ = [] + + for obj in a, b: + with self.subTest(obj=obj): + self.assertFalse(is_dataclass(obj)) + + # Indirect tests for _is_dataclass_instance(). + with self.assertRaisesRegex(TypeError, 'should be called on dataclass instances'): + asdict(obj) + with self.assertRaisesRegex(TypeError, 'should be called on dataclass instances'): + astuple(obj) + with self.assertRaisesRegex(TypeError, 'should be called on dataclass instances'): + replace(obj, x=0) + def test_helper_fields_with_class_instance(self): # Check that we can call fields() on either a class or instance, # and get back the same thing. @@ -2892,6 +2938,17 @@ class TestStringAnnotations(unittest.TestCase): # won't exist on the instance. self.assertNotIn('not_iv4', c.__dict__) + def test_text_annotations(self): + from test import dataclass_textanno + + self.assertEqual( + get_type_hints(dataclass_textanno.Bar), + {'foo': dataclass_textanno.Foo}) + self.assertEqual( + get_type_hints(dataclass_textanno.Bar.__init__), + {'foo': dataclass_textanno.Foo, + 'return': type(None)}) + class TestMakeDataclass(unittest.TestCase): def test_simple(self): @@ -3037,11 +3094,11 @@ class TestMakeDataclass(unittest.TestCase): def test_non_identifier_field_names(self): for field in ['()', 'x,y', '*', '2@3', '', 'little johnny tables']: with self.subTest(field=field): - with self.assertRaisesRegex(TypeError, 'must be valid identifers'): + with self.assertRaisesRegex(TypeError, 'must be valid identifiers'): make_dataclass('C', ['a', field]) - with self.assertRaisesRegex(TypeError, 'must be valid identifers'): + with self.assertRaisesRegex(TypeError, 'must be valid identifiers'): make_dataclass('C', [field]) - with self.assertRaisesRegex(TypeError, 'must be valid identifers'): + with self.assertRaisesRegex(TypeError, 'must be valid identifiers'): make_dataclass('C', [field, 'a']) def test_underscore_field_names(self): diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py index 675f9748e8..8730474997 100644 --- a/lib-python/3/test/test_descr.py +++ b/lib-python/3/test/test_descr.py @@ -2627,12 +2627,8 @@ order (MRO) for bases """ self.assertEqual(Sub.test(), Base.aProp) # Verify that super() doesn't allow keyword args - try: + with self.assertRaises(TypeError): super(Base, kw=1) - except TypeError: - pass - else: - self.assertEqual("super shouldn't accept keyword args") def test_basic_inheritance(self): # Testing inheritance from basic types... diff --git a/lib-python/3/test/test_dict.py b/lib-python/3/test/test_dict.py index 90c0a3131a..ea9dcb6a81 100644 --- a/lib-python/3/test/test_dict.py +++ b/lib-python/3/test/test_dict.py @@ -1138,7 +1138,7 @@ class DictTest(unittest.TestCase): support.check_free_after_iterating(self, lambda d: iter(d.items()), dict) def test_equal_operator_modifying_operand(self): - # test fix for seg fault reported in issue 27945 part 3. + # test fix for seg fault reported in bpo-27945 part 3. class X(): def __del__(self): dict_b.clear() @@ -1154,6 +1154,16 @@ class DictTest(unittest.TestCase): dict_b = {X(): X()} self.assertTrue(dict_a == dict_b) + # test fix for seg fault reported in bpo-38588 part 1. + class Y: + def __eq__(self, other): + dict_d.clear() + return True + + dict_c = {0: Y()} + dict_d = {0: set()} + self.assertTrue(dict_c == dict_d) + def test_fromkeys_operator_modifying_dict_operand(self): # test fix for seg fault reported in issue 27945 part 4a. class X(int): diff --git a/lib-python/3/test/test_doctest.py b/lib-python/3/test/test_doctest.py index 4a3c488738..c2f0568b55 100644 --- a/lib-python/3/test/test_doctest.py +++ b/lib-python/3/test/test_doctest.py @@ -8,8 +8,12 @@ import functools import os import sys import importlib +import importlib.abc +import importlib.util import unittest import tempfile +import shutil +import contextlib # NOTE: There are some additional tests relating to interaction with # zipimport in the test_zipimport_support test module. @@ -437,7 +441,7 @@ We'll simulate a __file__ attr that ends in pyc: >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [<DocTest sample_func from ...:21 (1 example)>] + [<DocTest sample_func from ...:25 (1 example)>] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -697,8 +701,12 @@ class TestDocTestFinder(unittest.TestCase): finally: support.forget(pkg_name) sys.path.pop() - assert doctest.DocTestFinder().find(mod) == [] + include_empty_finder = doctest.DocTestFinder(exclude_empty=False) + exclude_empty_finder = doctest.DocTestFinder(exclude_empty=True) + + self.assertEqual(len(include_empty_finder.find(mod)), 1) + self.assertEqual(len(exclude_empty_finder.find(mod)), 0) def test_DocTestParser(): r""" Unit tests for the `DocTestParser` class. @@ -2655,12 +2663,52 @@ Test the verbose output: >>> sys.argv = save_argv """ +class TestImporter(importlib.abc.MetaPathFinder, importlib.abc.ResourceLoader): + + def find_spec(self, fullname, path, target=None): + return importlib.util.spec_from_file_location(fullname, path, loader=self) + + def get_data(self, path): + with open(path, mode='rb') as f: + return f.read() + +class TestHook: + + def __init__(self, pathdir): + self.sys_path = sys.path[:] + self.meta_path = sys.meta_path[:] + self.path_hooks = sys.path_hooks[:] + sys.path.append(pathdir) + sys.path_importer_cache.clear() + self.modules_before = sys.modules.copy() + self.importer = TestImporter() + sys.meta_path.append(self.importer) + + def remove(self): + sys.path[:] = self.sys_path + sys.meta_path[:] = self.meta_path + sys.path_hooks[:] = self.path_hooks + sys.path_importer_cache.clear() + sys.modules.clear() + sys.modules.update(self.modules_before) + + +@contextlib.contextmanager +def test_hook(pathdir): + hook = TestHook(pathdir) + try: + yield hook + finally: + hook.remove() + + def test_lineendings(): r""" -*nix systems use \n line endings, while Windows systems use \r\n. Python +*nix systems use \n line endings, while Windows systems use \r\n, and +old Mac systems used \r, which Python still recognizes as a line ending. Python handles this using universal newline mode for reading files. Let's make sure doctest does so (issue 8473) by creating temporary test files using each -of the two line disciplines. One of the two will be the "wrong" one for the -platform the test is run on. +of the three line disciplines. At least one will not match either the universal +newline \n or os.linesep for the platform the test is run on. Windows line endings first: @@ -2683,6 +2731,47 @@ And now *nix line endings: TestResults(failed=0, attempted=1) >>> os.remove(fn) +And finally old Mac line endings: + + >>> fn = tempfile.mktemp() + >>> with open(fn, 'wb') as f: + ... f.write(b'Test:\r\r >>> x = 1 + 1\r\rDone.\r') + 30 + >>> doctest.testfile(fn, module_relative=False, verbose=False) + TestResults(failed=0, attempted=1) + >>> os.remove(fn) + +Now we test with a package loader that has a get_data method, since that +bypasses the standard universal newline handling so doctest has to do the +newline conversion itself; let's make sure it does so correctly (issue 1812). +We'll write a file inside the package that has all three kinds of line endings +in it, and use a package hook to install a custom loader; on any platform, +at least one of the line endings will raise a ValueError for inconsistent +whitespace if doctest does not correctly do the newline conversion. + + >>> dn = tempfile.mkdtemp() + >>> pkg = os.path.join(dn, "doctest_testpkg") + >>> os.mkdir(pkg) + >>> support.create_empty_file(os.path.join(pkg, "__init__.py")) + >>> fn = os.path.join(pkg, "doctest_testfile.txt") + >>> with open(fn, 'wb') as f: + ... f.write( + ... b'Test:\r\n\r\n' + ... b' >>> x = 1 + 1\r\n\r\n' + ... b'Done.\r\n' + ... b'Test:\n\n' + ... b' >>> x = 1 + 1\n\n' + ... b'Done.\n' + ... b'Test:\r\r' + ... b' >>> x = 1 + 1\r\r' + ... b'Done.\r' + ... ) + 95 + >>> with test_hook(dn): + ... doctest.testfile("doctest_testfile.txt", package="doctest_testpkg", verbose=False) + TestResults(failed=0, attempted=3) + >>> shutil.rmtree(dn) + """ def test_testmod(): r""" diff --git a/lib-python/3/test/test_docxmlrpc.py b/lib-python/3/test/test_docxmlrpc.py index f077f05f5b..38215659b6 100644 --- a/lib-python/3/test/test_docxmlrpc.py +++ b/lib-python/3/test/test_docxmlrpc.py @@ -1,5 +1,6 @@ from xmlrpc.server import DocXMLRPCServer import http.client +import re import sys import threading from test import support @@ -193,6 +194,21 @@ class DocXMLRPCHTTPGETServer(unittest.TestCase): b'method_annotation</strong></a>(x: bytes)</dt></dl>'), response.read()) + def test_server_title_escape(self): + # bpo-38243: Ensure that the server title and documentation + # are escaped for HTML. + self.serv.set_server_title('test_title<script>') + self.serv.set_server_documentation('test_documentation<script>') + self.assertEqual('test_title<script>', self.serv.server_title) + self.assertEqual('test_documentation<script>', + self.serv.server_documentation) + + generated = self.serv.generate_html_documentation() + title = re.search(r'<title>(.+?)</title>', generated).group() + documentation = re.search(r'<p><tt>(.+?)</tt></p>', generated).group() + self.assertEqual('<title>Python: test_title<script></title>', title) + self.assertEqual('<p><tt>test_documentation<script></tt></p>', documentation) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/3/test/test_eintr.py b/lib-python/3/test/test_eintr.py index f61efa3c64..a5f8f6465e 100644 --- a/lib-python/3/test/test_eintr.py +++ b/lib-python/3/test/test_eintr.py @@ -22,7 +22,7 @@ class EINTRTests(unittest.TestCase): print() print("--- run eintr_tester.py ---", flush=True) # In verbose mode, the child process inherit stdout and stdout, - # to see output in realtime and reduce the risk of loosing output. + # to see output in realtime and reduce the risk of losing output. args = [sys.executable, "-E", "-X", "faulthandler", *args] proc = subprocess.run(args) print(f"--- eintr_tester.py completed: " diff --git a/lib-python/3/test/test_email/test__encoded_words.py b/lib-python/3/test/test_email/test__encoded_words.py index 5a59aebba8..0b8b1de335 100644 --- a/lib-python/3/test/test_email/test__encoded_words.py +++ b/lib-python/3/test/test_email/test__encoded_words.py @@ -58,6 +58,8 @@ class TestDecode(TestEmailBase): _ew.decode('=?') with self.assertRaises(ValueError): _ew.decode('') + with self.assertRaises(KeyError): + _ew.decode('=?utf-8?X?somevalue?=') def _test(self, source, result, charset='us-ascii', lang='', defects=[]): res, char, l, d = _ew.decode(source) diff --git a/lib-python/3/test/test_email/test__header_value_parser.py b/lib-python/3/test/test_email/test__header_value_parser.py index 693487bc96..4894d8f22e 100644 --- a/lib-python/3/test/test_email/test__header_value_parser.py +++ b/lib-python/3/test/test_email/test__header_value_parser.py @@ -89,6 +89,10 @@ class TestParser(TestParserMixin, TestEmailBase): with self.assertRaises(errors.HeaderParseError): parser.get_encoded_word('=?abc?=') + def test_get_encoded_word_invalid_cte(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_encoded_word('=?utf-8?X?somevalue?=') + def test_get_encoded_word_valid_ew(self): self._test_get_x(parser.get_encoded_word, '=?us-ascii?q?this_is_a_test?= bird', @@ -383,6 +387,30 @@ class TestParser(TestParserMixin, TestEmailBase): [errors.InvalidHeaderDefect], '') + def test_get_unstructured_without_trailing_whitespace_hang_case(self): + self._test_get_x(self._get_unst, + '=?utf-8?q?somevalue?=aa', + 'somevalueaa', + 'somevalueaa', + [errors.InvalidHeaderDefect], + '') + + def test_get_unstructured_invalid_ew(self): + self._test_get_x(self._get_unst, + '=?utf-8?q?=somevalue?=', + '=?utf-8?q?=somevalue?=', + '=?utf-8?q?=somevalue?=', + [], + '') + + def test_get_unstructured_invalid_ew_cte(self): + self._test_get_x(self._get_unst, + '=?utf-8?X?=somevalue?=', + '=?utf-8?X?=somevalue?=', + '=?utf-8?X?=somevalue?=', + [], + '') + # get_qp_ctext def test_get_qp_ctext_only(self): @@ -522,6 +550,10 @@ class TestParser(TestParserMixin, TestEmailBase): self._test_get_x(parser.get_bare_quoted_string, '""', '""', '', [], '') + def test_get_bare_quoted_string_missing_endquotes(self): + self._test_get_x(parser.get_bare_quoted_string, + '"', '""', '', [errors.InvalidHeaderDefect], '') + def test_get_bare_quoted_string_following_wsp_preserved(self): self._test_get_x(parser.get_bare_quoted_string, '"foo"\t bar', '"foo"', 'foo', [], '\t bar') @@ -930,6 +962,12 @@ class TestParser(TestParserMixin, TestEmailBase): self.assertEqual(word.token_type, 'atom') self.assertEqual(word[0].token_type, 'cfws') + def test_get_word_all_CFWS(self): + # bpo-29412: Test that we don't raise IndexError when parsing CFWS only + # token. + with self.assertRaises(errors.HeaderParseError): + parser.get_word('(Recipients list suppressed') + def test_get_word_qs_yields_qs(self): word = self._test_get_x(parser.get_word, '"bar " (bang) ah', '"bar " (bang) ', 'bar ', [], 'ah') @@ -1438,6 +1476,16 @@ class TestParser(TestParserMixin, TestEmailBase): self.assertEqual(addr_spec.domain, 'example.com') self.assertEqual(addr_spec.addr_spec, 'star.a.star@example.com') + def test_get_addr_spec_multiple_domains(self): + with self.assertRaises(errors.HeaderParseError): + parser.get_addr_spec('star@a.star@example.com') + + with self.assertRaises(errors.HeaderParseError): + parser.get_addr_spec('star@a@example.com') + + with self.assertRaises(errors.HeaderParseError): + parser.get_addr_spec('star@172.17.0.1@example.com') + # get_obs_route def test_get_obs_route_simple(self): @@ -1680,6 +1728,14 @@ class TestParser(TestParserMixin, TestEmailBase): self.assertEqual(display_name[3].comments, ['with trailing comment']) self.assertEqual(display_name.display_name, 'simple phrase.') + def test_get_display_name_for_invalid_address_field(self): + # bpo-32178: Test that address fields starting with `:` don't cause + # IndexError when parsing the display name. + display_name = self._test_get_x( + parser.get_display_name, + ':Foo ', '', '', [errors.InvalidHeaderDefect], ':Foo ') + self.assertEqual(display_name.value, '') + # get_name_addr def test_get_name_addr_angle_addr_only(self): @@ -2343,6 +2399,17 @@ class TestParser(TestParserMixin, TestEmailBase): # get_address_list + def test_get_address_list_CFWS(self): + address_list = self._test_get_x(parser.get_address_list, + '(Recipient list suppressed)', + '(Recipient list suppressed)', + ' ', + [errors.ObsoleteHeaderDefect], # no content in address list + '') + self.assertEqual(address_list.token_type, 'address-list') + self.assertEqual(len(address_list.mailboxes), 0) + self.assertEqual(address_list.mailboxes, address_list.all_mailboxes) + def test_get_address_list_mailboxes_simple(self): address_list = self._test_get_x(parser.get_address_list, 'dinsdale@example.com', @@ -2621,6 +2688,13 @@ class Test_parse_mime_parameters(TestParserMixin, TestEmailBase): # Defects are apparent missing *0*, and two 'out of sequence'. [errors.InvalidHeaderDefect]*3), + # bpo-37461: Check that we don't go into an infinite loop. + 'extra_dquote': ( + 'r*="\'a\'\\"', + ' r="\\""', + 'r*=\'a\'"', + [('r', '"')], + [errors.InvalidHeaderDefect]*2), } @parameterize diff --git a/lib-python/3/test/test_email/test_email.py b/lib-python/3/test/test_email/test_email.py index c29cc56203..9e5c6adca8 100644 --- a/lib-python/3/test/test_email/test_email.py +++ b/lib-python/3/test/test_email/test_email.py @@ -11,8 +11,8 @@ import textwrap from io import StringIO, BytesIO from itertools import chain from random import choice -from socket import getfqdn from threading import Thread +from unittest.mock import patch import email import email.policy @@ -3041,6 +3041,20 @@ class TestMiscellaneous(TestEmailBase): self.assertEqual(utils.parseaddr('<>'), ('', '')) self.assertEqual(utils.formataddr(utils.parseaddr('<>')), '') + def test_parseaddr_multiple_domains(self): + self.assertEqual( + utils.parseaddr('a@b@c'), + ('', '') + ) + self.assertEqual( + utils.parseaddr('a@b.c@c'), + ('', '') + ) + self.assertEqual( + utils.parseaddr('a@172.17.0.1@c'), + ('', '') + ) + def test_noquote_dump(self): self.assertEqual( utils.formataddr(('A Silly Person', 'person@dom.ain')), @@ -3328,9 +3342,11 @@ multipart/report '.test-idstring@testdomain-string>') def test_make_msgid_default_domain(self): - self.assertTrue( - email.utils.make_msgid().endswith( - '@' + getfqdn() + '>')) + with patch('socket.getfqdn') as mock_getfqdn: + mock_getfqdn.return_value = domain = 'pythontest.example.com' + self.assertTrue( + email.utils.make_msgid().endswith( + '@' + domain + '>')) def test_Generator_linend(self): # Issue 14645. @@ -5367,6 +5383,27 @@ Content-Type: application/x-foo; eq(language, 'en-us') eq(s, 'My Document For You') + def test_should_not_hang_on_invalid_ew_messages(self): + messages = ["""From: user@host.com +To: user@host.com +Bad-Header: + =?us-ascii?Q?LCSwrV11+IB0rSbSker+M9vWR7wEDSuGqmHD89Gt=ea0nJFSaiz4vX3XMJPT4vrE?= + =?us-ascii?Q?xGUZeOnp0o22pLBB7CYLH74Js=wOlK6Tfru2U47qR?= + =?us-ascii?Q?72OfyEY2p2=2FrA9xNFyvH+fBTCmazxwzF8nGkK6D?= + +Hello! +""", """From: ����� �������� <xxx@xxx> +To: "xxx" <xxx@xxx> +Subject: ��� ���������� ����� ����� � ��������� �� ���� +MIME-Version: 1.0 +Content-Type: text/plain; charset="windows-1251"; +Content-Transfer-Encoding: 8bit + +�� ����� � ���� ������ ��� �������� +"""] + for m in messages: + with self.subTest(m=m): + msg = email.message_from_string(m) # Tests to ensure that signed parts of an email are completely preserved, as diff --git a/lib-python/3/test/test_email/test_headerregistry.py b/lib-python/3/test/test_email/test_headerregistry.py index e6db3acedc..a95b20aeca 100644 --- a/lib-python/3/test/test_email/test_headerregistry.py +++ b/lib-python/3/test/test_email/test_headerregistry.py @@ -872,6 +872,25 @@ class TestContentDisposition(TestHeaderBase): {'filename': 'foo'}, [errors.InvalidHeaderDefect]), + 'invalid_parameter_value_with_fws_between_ew': ( + 'attachment; filename="=?UTF-8?Q?Schulbesuchsbest=C3=A4ttigung=2E?=' + ' =?UTF-8?Q?pdf?="', + 'attachment', + {'filename': 'Schulbesuchsbestättigung.pdf'}, + [errors.InvalidHeaderDefect]*3, + ('attachment; filename="Schulbesuchsbestättigung.pdf"'), + ('Content-Disposition: attachment;\n' + ' filename*=utf-8\'\'Schulbesuchsbest%C3%A4ttigung.pdf\n'), + ), + + 'parameter_value_with_fws_between_tokens': ( + 'attachment; filename="File =?utf-8?q?Name?= With Spaces.pdf"', + 'attachment', + {'filename': 'File Name With Spaces.pdf'}, + [errors.InvalidHeaderDefect], + 'attachment; filename="File Name With Spaces.pdf"', + ('Content-Disposition: attachment; filename="File Name With Spaces.pdf"\n'), + ) } @@ -1436,6 +1455,25 @@ class TestAddressAndGroup(TestEmailBase): # with self.assertRaises(ValueError): # Address('foo', 'wők', 'example.com') + def test_crlf_in_constructor_args_raises(self): + cases = ( + dict(display_name='foo\r'), + dict(display_name='foo\n'), + dict(display_name='foo\r\n'), + dict(domain='example.com\r'), + dict(domain='example.com\n'), + dict(domain='example.com\r\n'), + dict(username='wok\r'), + dict(username='wok\n'), + dict(username='wok\r\n'), + dict(addr_spec='wok@example.com\r'), + dict(addr_spec='wok@example.com\n'), + dict(addr_spec='wok@example.com\r\n') + ) + for kwargs in cases: + with self.subTest(kwargs=kwargs), self.assertRaisesRegex(ValueError, "invalid arguments"): + Address(**kwargs) + def test_non_ascii_username_in_addr_spec_raises(self): with self.assertRaises(ValueError): Address('foo', addr_spec='wők@example.com') @@ -1528,6 +1566,30 @@ class TestAddressAndGroup(TestEmailBase): class TestFolding(TestHeaderBase): + def test_address_display_names(self): + """Test the folding and encoding of address headers.""" + for name, result in ( + ('Foo Bar, France', '"Foo Bar, France"'), + ('Foo Bar (France)', '"Foo Bar (France)"'), + ('Foo Bar, España', 'Foo =?utf-8?q?Bar=2C_Espa=C3=B1a?='), + ('Foo Bar (España)', 'Foo Bar =?utf-8?b?KEVzcGHDsWEp?='), + ('Foo, Bar España', '=?utf-8?q?Foo=2C_Bar_Espa=C3=B1a?='), + ('Foo, Bar [España]', '=?utf-8?q?Foo=2C_Bar_=5BEspa=C3=B1a=5D?='), + ('Foo Bär, France', 'Foo =?utf-8?q?B=C3=A4r=2C?= France'), + ('Foo Bär <France>', 'Foo =?utf-8?q?B=C3=A4r_=3CFrance=3E?='), + ( + 'Lôrem ipsum dôlôr sit amet, cônsectetuer adipiscing. ' + 'Suspendisse pôtenti. Aliquam nibh. Suspendisse pôtenti.', + '=?utf-8?q?L=C3=B4rem_ipsum_d=C3=B4l=C3=B4r_sit_amet=2C_c' + '=C3=B4nsectetuer?=\n =?utf-8?q?adipiscing=2E_Suspendisse' + '_p=C3=B4tenti=2E_Aliquam_nibh=2E?=\n Suspendisse =?utf-8' + '?q?p=C3=B4tenti=2E?=', + ), + ): + h = self.make_header('To', Address(name, addr_spec='a@b.com')) + self.assertEqual(h.fold(policy=policy.default), + 'To: %s <a@b.com>\n' % result) + def test_short_unstructured(self): h = self.make_header('subject', 'this is a test') self.assertEqual(h.fold(policy=policy.default), diff --git a/lib-python/3/test/test_email/test_message.py b/lib-python/3/test/test_email/test_message.py index 5dc46e1b81..fab97d9188 100644 --- a/lib-python/3/test/test_email/test_message.py +++ b/lib-python/3/test/test_email/test_message.py @@ -929,6 +929,15 @@ class TestMIMEPart(TestEmailMessageBase, TestEmailBase): m.set_content(content_manager=cm) self.assertNotIn('MIME-Version', m) + def test_string_payload_with_multipart_content_type(self): + msg = message_from_string(textwrap.dedent("""\ + Content-Type: multipart/mixed; charset="utf-8" + + sample text + """), policy=policy.default) + attachments = msg.iter_attachments() + self.assertEqual(list(attachments), []) + if __name__ == '__main__': unittest.main() diff --git a/lib-python/3/test/test_email/test_policy.py b/lib-python/3/test/test_email/test_policy.py index 0aea934df4..ebc3ce6f76 100644 --- a/lib-python/3/test/test_email/test_policy.py +++ b/lib-python/3/test/test_email/test_policy.py @@ -3,6 +3,7 @@ import sys import types import textwrap import unittest +import email.errors import email.policy import email.parser import email.generator @@ -258,6 +259,25 @@ class PolicyAPITests(unittest.TestCase): 'Subject: \n' + 12 * ' =?utf-8?q?=C4=85?=\n') + def test_short_maxlen_error(self): + # RFC 2047 chrome takes up 7 characters, plus the length of the charset + # name, so folding should fail if maxlen is lower than the minimum + # required length for a line. + + # Note: This is only triggered when there is a single word longer than + # max_line_length, hence the 1234567890 at the end of this whimsical + # subject. This is because when we encounter a word longer than + # max_line_length, it is broken down into encoded words to fit + # max_line_length. If the max_line_length isn't large enough to even + # contain the RFC 2047 chrome (`?=<charset>?q??=`), we fail. + subject = "Melt away the pounds with this one simple trick! 1234567890" + + for maxlen in [3, 7, 9]: + with self.subTest(maxlen=maxlen): + policy = email.policy.default.clone(max_line_length=maxlen) + with self.assertRaises(email.errors.HeaderParseError): + policy.fold("Subject", subject) + # XXX: Need subclassing tests. # For adding subclassed objects, make sure the usual rules apply (subclass # wins), but that the order still works (right overrides left). diff --git a/lib-python/3/test/test_ensurepip.py b/lib-python/3/test/test_ensurepip.py index 8996689309..4786d28f39 100644 --- a/lib-python/3/test/test_ensurepip.py +++ b/lib-python/3/test/test_ensurepip.py @@ -40,7 +40,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "setuptools", "pip", ], unittest.mock.ANY, @@ -54,7 +54,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "--root", "/foo/bar/", "setuptools", "pip", ], @@ -66,7 +66,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "--user", "setuptools", "pip", ], unittest.mock.ANY, @@ -77,7 +77,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "--upgrade", "setuptools", "pip", ], unittest.mock.ANY, @@ -88,7 +88,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "-v", "setuptools", "pip", ], unittest.mock.ANY, @@ -99,7 +99,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "-vv", "setuptools", "pip", ], unittest.mock.ANY, @@ -110,7 +110,7 @@ class TestBootstrap(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "-vvv", "setuptools", "pip", ], unittest.mock.ANY, @@ -260,7 +260,7 @@ class TestBootstrappingMainFunction(EnsurepipMixin, unittest.TestCase): self.run_pip.assert_called_once_with( [ - "install", "--no-index", "--find-links", + "install", "--no-cache-dir", "--no-index", "--find-links", unittest.mock.ANY, "setuptools", "pip", ], unittest.mock.ANY, diff --git a/lib-python/3/test/test_enum.py b/lib-python/3/test/test_enum.py index 29a429ccd9..d9260f4cb4 100644 --- a/lib-python/3/test/test_enum.py +++ b/lib-python/3/test/test_enum.py @@ -1710,6 +1710,16 @@ class TestEnum(unittest.TestCase): self.assertEqual(Color.blue.value, 2) self.assertEqual(Color.green.value, 3) + def test_auto_order(self): + with self.assertRaises(TypeError): + class Color(Enum): + red = auto() + green = auto() + blue = auto() + def _generate_next_value_(name, start, count, last): + return name + + def test_duplicate_auto(self): class Dupes(Enum): first = primero = auto() diff --git a/lib-python/3/test/test_fcntl.py b/lib-python/3/test/test_fcntl.py index acd5c7cc58..036bd64b40 100644 --- a/lib-python/3/test/test_fcntl.py +++ b/lib-python/3/test/test_fcntl.py @@ -5,6 +5,7 @@ import os import struct import sys import unittest +from multiprocessing import Process from test.support import (verbose, TESTFN, unlink, run_unittest, import_module, cpython_only) @@ -12,7 +13,6 @@ from test.support import (verbose, TESTFN, unlink, run_unittest, import_module, fcntl = import_module('fcntl') -# TODO - Write tests for flock() and lockf(). def get_lockdata(): try: @@ -51,6 +51,21 @@ class BadFile: def fileno(self): return self.fn +def try_lockf_on_other_process_fail(fname, cmd): + f = open(fname, 'wb+') + try: + fcntl.lockf(f, cmd) + except BlockingIOError: + pass + finally: + f.close() + +def try_lockf_on_other_process(fname, cmd): + f = open(fname, 'wb+') + fcntl.lockf(f, cmd) + fcntl.lockf(f, fcntl.LOCK_UN) + f.close() + class TestFcntl(unittest.TestCase): def setUp(self): @@ -138,6 +153,28 @@ class TestFcntl(unittest.TestCase): self.assertRaises(ValueError, fcntl.flock, -1, fcntl.LOCK_SH) self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH) + @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + def test_lockf_exclusive(self): + self.f = open(TESTFN, 'wb+') + cmd = fcntl.LOCK_EX | fcntl.LOCK_NB + fcntl.lockf(self.f, cmd) + p = Process(target=try_lockf_on_other_process_fail, args=(TESTFN, cmd)) + p.start() + p.join() + fcntl.lockf(self.f, fcntl.LOCK_UN) + self.assertEqual(p.exitcode, 0) + + @unittest.skipIf(platform.system() == "AIX", "AIX returns PermissionError") + def test_lockf_share(self): + self.f = open(TESTFN, 'wb+') + cmd = fcntl.LOCK_SH | fcntl.LOCK_NB + fcntl.lockf(self.f, cmd) + p = Process(target=try_lockf_on_other_process, args=(TESTFN, cmd)) + p.start() + p.join() + fcntl.lockf(self.f, fcntl.LOCK_UN) + self.assertEqual(p.exitcode, 0) + @cpython_only def test_flock_overflow(self): import _testcapi diff --git a/lib-python/3/test/test_fractions.py b/lib-python/3/test/test_fractions.py index 7905c367ba..d1a43870ea 100644 --- a/lib-python/3/test/test_fractions.py +++ b/lib-python/3/test/test_fractions.py @@ -6,6 +6,7 @@ import math import numbers import operator import fractions +import functools import sys import unittest import warnings @@ -335,6 +336,42 @@ class FractionTest(unittest.TestCase): self.assertTypedEquals(0.1+0j, complex(F(1,10))) + def testBoolGuarateesBoolReturn(self): + # Ensure that __bool__ is used on numerator which guarantees a bool + # return. See also bpo-39274. + @functools.total_ordering + class CustomValue: + denominator = 1 + + def __init__(self, value): + self.value = value + + def __bool__(self): + return bool(self.value) + + @property + def numerator(self): + # required to preserve `self` during instantiation + return self + + def __eq__(self, other): + raise AssertionError("Avoid comparisons in Fraction.__bool__") + + __lt__ = __eq__ + + # We did not implement all abstract methods, so register: + numbers.Rational.register(CustomValue) + + numerator = CustomValue(1) + r = F(numerator) + # ensure the numerator was not lost during instantiation: + self.assertIs(r.numerator, numerator) + self.assertIs(bool(r), True) + + numerator = CustomValue(0) + r = F(numerator) + self.assertIs(bool(r), False) + def testRound(self): self.assertTypedEquals(F(-200), round(F(-150), -2)) self.assertTypedEquals(F(-200), round(F(-250), -2)) diff --git a/lib-python/3/test/test_future.py b/lib-python/3/test/test_future.py index 70da0cf570..13a4f6f3cc 100644 --- a/lib-python/3/test/test_future.py +++ b/lib-python/3/test/test_future.py @@ -237,6 +237,10 @@ class AnnotationsFutureTestCase(unittest.TestCase): eq("dict[str, int]") eq("set[str,]") eq("tuple[str, ...]") + eq("tuple[(str, *types)]") + eq("tuple[xx:yy, (*types,)]") + eq("tuple[str, int, (str, int)]") + eq("tuple[(*int, str, str, (str, int))]") eq("tuple[str, int, float, dict[str, int]]") eq("slice[0]") eq("slice[0:1]") @@ -245,6 +249,11 @@ class AnnotationsFutureTestCase(unittest.TestCase): eq("slice[:-1]") eq("slice[1:]") eq("slice[::-1]") + eq("slice[:,]") + eq("slice[1:2,]") + eq("slice[1:2:3,]") + eq("slice[1:2, 1]") + eq("slice[1:2, 2, 3]") eq("slice[()]") eq("slice[a, b:c, d:e:f]") eq("slice[(x for x in a)]") diff --git a/lib-python/3/test/test_gc.py b/lib-python/3/test/test_gc.py index 8d806db3ba..a2fa8bbace 100644 --- a/lib-python/3/test/test_gc.py +++ b/lib-python/3/test/test_gc.py @@ -755,6 +755,77 @@ class GCTests(unittest.TestCase): gc.unfreeze() self.assertEqual(gc.get_freeze_count(), 0) + def test_38379(self): + # When a finalizer resurrects objects, stats were reporting them as + # having been collected. This affected both collect()'s return + # value and the dicts returned by get_stats(). + N = 100 + + class A: # simple self-loop + def __init__(self): + self.me = self + + class Z(A): # resurrecting __del__ + def __del__(self): + zs.append(self) + + zs = [] + + def getstats(): + d = gc.get_stats()[-1] + return d['collected'], d['uncollectable'] + + gc.collect() + gc.disable() + + # No problems if just collecting A() instances. + oldc, oldnc = getstats() + for i in range(N): + A() + t = gc.collect() + c, nc = getstats() + self.assertEqual(t, 2*N) # instance object & its dict + self.assertEqual(c - oldc, 2*N) + self.assertEqual(nc - oldnc, 0) + + # But Z() is not actually collected. + oldc, oldnc = c, nc + Z() + # Nothing is collected - Z() is merely resurrected. + t = gc.collect() + c, nc = getstats() + #self.assertEqual(t, 2) # before + self.assertEqual(t, 0) # after + #self.assertEqual(c - oldc, 2) # before + self.assertEqual(c - oldc, 0) # after + self.assertEqual(nc - oldnc, 0) + + # Unfortunately, a Z() prevents _anything_ from being collected. + # It should be possible to collect the A instances anyway, but + # that will require non-trivial code changes. + oldc, oldnc = c, nc + for i in range(N): + A() + Z() + # Z() prevents anything from being collected. + t = gc.collect() + c, nc = getstats() + #self.assertEqual(t, 2*N + 2) # before + self.assertEqual(t, 0) # after + #self.assertEqual(c - oldc, 2*N + 2) # before + self.assertEqual(c - oldc, 0) # after + self.assertEqual(nc - oldnc, 0) + + # But the A() trash is reclaimed on the next run. + oldc, oldnc = c, nc + t = gc.collect() + c, nc = getstats() + self.assertEqual(t, 2*N) + self.assertEqual(c - oldc, 2*N) + self.assertEqual(nc - oldnc, 0) + + gc.enable() + class GCCallbackTests(unittest.TestCase): def setUp(self): diff --git a/lib-python/3/test/test_gdb.py b/lib-python/3/test/test_gdb.py index 711fb69ebd..fe603be2bb 100644 --- a/lib-python/3/test/test_gdb.py +++ b/lib-python/3/test/test_gdb.py @@ -18,12 +18,18 @@ from test.support import run_unittest, findfile, python_is_optimized def get_gdb_version(): try: - proc = subprocess.Popen(["gdb", "-nx", "--version"], + cmd = ["gdb", "-nx", "--version"] + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) with proc: - version = proc.communicate()[0] + version, stderr = proc.communicate() + + if proc.returncode: + raise Exception(f"Command {' '.join(cmd)!r} failed " + f"with exit code {proc.returncode}: " + f"stdout={version!r} stderr={stderr!r}") except OSError: # This is what "no gdb" looks like. There may, however, be other # errors that manifest this way too. @@ -211,43 +217,31 @@ class DebuggerTests(unittest.TestCase): elif script: args += [script] - # print args - # print (' '.join(args)) - # Use "args" to invoke gdb, capturing stdout, stderr: out, err = run_gdb(*args, PYTHONHASHSEED=PYTHONHASHSEED) - errlines = err.splitlines() - unexpected_errlines = [] - - # Ignore some benign messages on stderr. - ignore_patterns = ( - 'Function "%s" not defined.' % breakpoint, - 'Do you need "set solib-search-path" or ' - '"set sysroot"?', - # BFD: /usr/lib/debug/(...): unable to initialize decompress - # status for section .debug_aranges - 'BFD: ', - # ignore all warnings - 'warning: ', - ) - for line in errlines: - if not line: - continue - # bpo34007: Sometimes some versions of the shared libraries that - # are part of the traceback are compiled in optimised mode and the - # Program Counter (PC) is not present, not allowing gdb to walk the - # frames back. When this happens, the Python bindings of gdb raise - # an exception, making the test impossible to succeed. - if "PC not saved" in line: - raise unittest.SkipTest("gdb cannot walk the frame object" - " because the Program Counter is" - " not present") - if not line.startswith(ignore_patterns): - unexpected_errlines.append(line) - - # Ensure no unexpected error messages: - self.assertEqual(unexpected_errlines, []) + for line in err.splitlines(): + print(line, file=sys.stderr) + + # bpo-34007: Sometimes some versions of the shared libraries that + # are part of the traceback are compiled in optimised mode and the + # Program Counter (PC) is not present, not allowing gdb to walk the + # frames back. When this happens, the Python bindings of gdb raise + # an exception, making the test impossible to succeed. + if "PC not saved" in err: + raise unittest.SkipTest("gdb cannot walk the frame object" + " because the Program Counter is" + " not present") + + # bpo-40019: Skip the test if gdb failed to read debug information + # because the Python binary is optimized. + for pattern in ( + '(frame information optimized out)', + 'Unable to read information on python frame', + ): + if pattern in out: + raise unittest.SkipTest(f"{pattern!r} found in gdb output") + return out def get_gdb_repr(self, source, @@ -273,8 +267,15 @@ class DebuggerTests(unittest.TestCase): # gdb can insert additional '\n' and space characters in various places # in its output, depending on the width of the terminal it's connected # to (using its "wrap_here" function) - m = re.match(r'.*#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)\)\s+at\s+\S*Python/bltinmodule.c.*', - gdb_output, re.DOTALL) + m = re.search( + # Match '#0 builtin_id(self=..., v=...)' + r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)' + # Match ' at Python/bltinmodule.c'. + # bpo-38239: builtin_id() is defined in Python/bltinmodule.c, + # but accept any "Directory\file.c" to support Link Time + # Optimization (LTO). + r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c', + gdb_output, re.DOTALL) if not m: self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output)) return m.group(1), gdb_output diff --git a/lib-python/3/test/test_gzip.py b/lib-python/3/test/test_gzip.py index 17ecda2089..0251914d8b 100644 --- a/lib-python/3/test/test_gzip.py +++ b/lib-python/3/test/test_gzip.py @@ -358,6 +358,26 @@ class TestGzip(BaseTest): isizeBytes = fRead.read(4) self.assertEqual(isizeBytes, struct.pack('<i', len(data1))) + def test_compresslevel_metadata(self): + # see RFC 1952: http://www.faqs.org/rfcs/rfc1952.html + # specifically, discussion of XFL in section 2.3.1 + cases = [ + ('fast', 1, b'\x04'), + ('best', 9, b'\x02'), + ('tradeoff', 6, b'\x00'), + ] + xflOffset = 8 + + for (name, level, expectedXflByte) in cases: + with self.subTest(name): + fWrite = gzip.GzipFile(self.filename, 'w', compresslevel=level) + with fWrite: + fWrite.write(data1) + with open(self.filename, 'rb') as fRead: + fRead.seek(xflOffset) + xflByte = fRead.read(1) + self.assertEqual(xflByte, expectedXflByte) + def test_with_open(self): # GzipFile supports the context management protocol with gzip.GzipFile(self.filename, "wb") as f: diff --git a/lib-python/3/test/test_heapq.py b/lib-python/3/test/test_heapq.py index 2f8c648d84..7c3fb0210f 100644 --- a/lib-python/3/test/test_heapq.py +++ b/lib-python/3/test/test_heapq.py @@ -414,6 +414,37 @@ class TestErrorHandling: with self.assertRaises((IndexError, RuntimeError)): self.module.heappop(heap) + def test_comparison_operator_modifiying_heap(self): + # See bpo-39421: Strong references need to be taken + # when comparing objects as they can alter the heap + class EvilClass(int): + def __lt__(self, o): + heap.clear() + return NotImplemented + + heap = [] + self.module.heappush(heap, EvilClass(0)) + self.assertRaises(IndexError, self.module.heappushpop, heap, 1) + + def test_comparison_operator_modifiying_heap_two_heaps(self): + + class h(int): + def __lt__(self, o): + list2.clear() + return NotImplemented + + class g(int): + def __lt__(self, o): + list1.clear() + return NotImplemented + + list1, list2 = [], [] + + self.module.heappush(list1, h(0)) + self.module.heappush(list2, g(0)) + + self.assertRaises((IndexError, RuntimeError), self.module.heappush, list1, g(1)) + self.assertRaises((IndexError, RuntimeError), self.module.heappush, list2, h(1)) class TestErrorHandlingPython(TestErrorHandling, TestCase): module = py_heapq diff --git a/lib-python/3/test/test_http_cookiejar.py b/lib-python/3/test/test_http_cookiejar.py index 16edf34a99..ff86ffa98c 100644 --- a/lib-python/3/test/test_http_cookiejar.py +++ b/lib-python/3/test/test_http_cookiejar.py @@ -6,6 +6,7 @@ import test.support import time import unittest import urllib.request +import warnings from http.cookiejar import (time2isoz, http2time, iso2time, time2netscape, parse_ns_headers, join_header_words, split_header_words, Cookie, @@ -122,6 +123,13 @@ class DateTimeTests(unittest.TestCase): "http2time(%s) is not None\n" "http2time(test) %s" % (test, http2time(test))) + def test_http2time_redos_regression_actually_completes(self): + # LOOSE_HTTP_DATE_RE was vulnerable to malicious input which caused catastrophic backtracking (REDoS). + # If we regress to cubic complexity, this test will take a very long time to succeed. + # If fixed, it should complete within a fraction of a second. + http2time("01 Jan 1970{}00:00:00 GMT!".format(" " * 10 ** 5)) + http2time("01 Jan 1970 00:00:00{}GMT!".format(" " * 10 ** 5)) + def test_iso2time(self): def parse_date(text): return time.gmtime(iso2time(text))[:6] @@ -179,6 +187,12 @@ class DateTimeTests(unittest.TestCase): self.assertIsNone(iso2time(test), "iso2time(%r)" % test) + def test_iso2time_performance_regression(self): + # If ISO_DATE_RE regresses to quadratic complexity, this test will take a very long time to succeed. + # If fixed, it should complete within a fraction of a second. + iso2time('1994-02-03{}14:15:29 -0100!'.format(' '*10**6)) + iso2time('1994-02-03 14:15:29{}-0100!'.format(' '*10**6)) + class HeaderTests(unittest.TestCase): @@ -560,6 +574,16 @@ class CookieTests(unittest.TestCase): # if expires is in future, keep cookie... c = CookieJar() future = time2netscape(time.time()+3600) + + with warnings.catch_warnings(record=True) as warns: + headers = [f"Set-Cookie: FOO=BAR; path=/; expires={future}"] + req = urllib.request.Request("http://www.coyote.com/") + res = FakeResponse(headers, "http://www.coyote.com/") + cookies = c.make_cookies(res, req) + self.assertEqual(len(cookies), 1) + self.assertEqual(time2netscape(cookies[0].expires), future) + self.assertEqual(len(warns), 0) + interact_netscape(c, "http://www.acme.com/", 'spam="bar"; expires=%s' % future) self.assertEqual(len(c), 1) diff --git a/lib-python/3/test/test_httplib.py b/lib-python/3/test/test_httplib.py index c424667158..3fa0691d3a 100644 --- a/lib-python/3/test/test_httplib.py +++ b/lib-python/3/test/test_httplib.py @@ -363,6 +363,28 @@ class HeaderTests(TestCase): self.assertEqual(lines[3], "header: Second: val2") +class HttpMethodTests(TestCase): + def test_invalid_method_names(self): + methods = ( + 'GET\r', + 'POST\n', + 'PUT\n\r', + 'POST\nValue', + 'POST\nHOST:abc', + 'GET\nrHost:abc\n', + 'POST\rRemainder:\r', + 'GET\rHOST:\n', + '\nPUT' + ) + + for method in methods: + with self.assertRaisesRegex( + ValueError, "method can't contain control characters"): + conn = client.HTTPConnection('example.com') + conn.sock = FakeSocket(None) + conn.request(method=method, url="/") + + class TransferEncodingTest(TestCase): expected_body = b"It's just a flesh wound" @@ -1157,6 +1179,45 @@ class BasicTest(TestCase): thread.join() self.assertEqual(result, b"proxied data\n") + def test_putrequest_override_domain_validation(self): + """ + It should be possible to override the default validation + behavior in putrequest (bpo-38216). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _validate_path(self, url): + pass + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/\x00') + + def test_putrequest_override_host_validation(self): + class UnsafeHTTPConnection(client.HTTPConnection): + def _validate_host(self, url): + pass + + conn = UnsafeHTTPConnection('example.com\r\n') + conn.sock = FakeSocket('') + # set skip_host so a ValueError is not raised upon adding the + # invalid URL as the value of the "Host:" header + conn.putrequest('GET', '/', skip_host=1) + + def test_putrequest_override_encoding(self): + """ + It should be possible to override the default encoding + to transmit bytes in another encoding even if invalid + (bpo-36274). + """ + class UnsafeHTTPConnection(client.HTTPConnection): + def _encode_request(self, str_url): + return str_url.encode('utf-8') + + conn = UnsafeHTTPConnection('example.com') + conn.sock = FakeSocket('') + conn.putrequest('GET', '/☃') + + class ExtendedReadTest(TestCase): """ Test peek(), read1(), readline() @@ -1281,6 +1342,7 @@ class ExtendedReadTest(TestCase): p = self.resp.peek(0) self.assertLessEqual(0, len(p)) + class ExtendedReadTestChunked(ExtendedReadTest): """ Test peek(), read1(), readline() in chunked mode diff --git a/lib-python/3/test/test_imaplib.py b/lib-python/3/test/test_imaplib.py index 9305e47ee9..300a6d7b65 100644 --- a/lib-python/3/test/test_imaplib.py +++ b/lib-python/3/test/test_imaplib.py @@ -908,6 +908,7 @@ class ThreadedNetworkedTestsSSL(ThreadedNetworkedTests): @unittest.skipUnless( support.is_resource_enabled('network'), 'network resource disabled') +@unittest.skip('cyrus.andrew.cmu.edu blocks connections') class RemoteIMAPTest(unittest.TestCase): host = 'cyrus.andrew.cmu.edu' port = 143 @@ -943,6 +944,7 @@ class RemoteIMAPTest(unittest.TestCase): @unittest.skipUnless(ssl, "SSL not available") @unittest.skipUnless( support.is_resource_enabled('network'), 'network resource disabled') +@unittest.skip('cyrus.andrew.cmu.edu blocks connections') class RemoteIMAP_STARTTLSTest(RemoteIMAPTest): def setUp(self): @@ -958,6 +960,7 @@ class RemoteIMAP_STARTTLSTest(RemoteIMAPTest): @unittest.skipUnless(ssl, "SSL not available") +@unittest.skip('cyrus.andrew.cmu.edu blocks connections') class RemoteIMAP_SSLTest(RemoteIMAPTest): port = 993 imap_class = IMAP4_SSL diff --git a/lib-python/3/test/test_import/__init__.py b/lib-python/3/test/test_import/__init__.py index 1fc4de11e1..4b9907d532 100644 --- a/lib-python/3/test/test_import/__init__.py +++ b/lib-python/3/test/test_import/__init__.py @@ -711,6 +711,11 @@ class RelativeImportTests(unittest.TestCase): ns = dict(__package__=object()) self.assertRaises(TypeError, check_relative) + def test_parentless_import_shadowed_by_global(self): + # Test as if this were done from the REPL where this error most commonly occurs (bpo-37409). + script_helper.assert_python_failure('-W', 'ignore', '-c', + "foo = 1; from . import foo") + def test_absolute_import_without_future(self): # If explicit relative import syntax is used, then do not try # to perform an absolute import in the face of failure. diff --git a/lib-python/3/test/test_importlib/test_api.py b/lib-python/3/test/test_importlib/test_api.py index edb745c2cd..0fb1346f9e 100644 --- a/lib-python/3/test/test_importlib/test_api.py +++ b/lib-python/3/test/test_importlib/test_api.py @@ -363,7 +363,7 @@ class ReloadTests: def test_module_missing_spec(self): #Test that reload() throws ModuleNotFounderror when reloading - # a module who's missing a spec. (bpo-29851) + # a module whose missing a spec. (bpo-29851) name = 'spam' with test_util.uncache(name): module = sys.modules[name] = types.ModuleType(name) diff --git a/lib-python/3/test/test_importlib/util.py b/lib-python/3/test/test_importlib/util.py index b0badebc2b..101b7d22bf 100644 --- a/lib-python/3/test/test_importlib/util.py +++ b/lib-python/3/test/test_importlib/util.py @@ -7,6 +7,7 @@ import importlib from importlib import machinery, util, invalidate_caches from importlib.abc import ResourceReader import io +import marshal import os import os.path from pathlib import Path, PurePath @@ -118,6 +119,16 @@ def submodule(parent, name, pkg_dir, content=''): return '{}.{}'.format(parent, name), path +def _get_code_from_pyc(pyc_path): + """Reads a pyc file and returns the unmarshalled code object within. + + No header validation is performed. + """ + with open(pyc_path, 'rb') as pyc_f: + pyc_f.seek(16) + return marshal.load(pyc_f) + + @contextlib.contextmanager def uncache(*names): """Uncache a module from sys.modules. diff --git a/lib-python/3/test/test_inspect.py b/lib-python/3/test/test_inspect.py index f3eb2f62c3..75ac40ba4f 100644 --- a/lib-python/3/test/test_inspect.py +++ b/lib-python/3/test/test_inspect.py @@ -3050,14 +3050,21 @@ class TestSignatureObject(unittest.TestCase): class MySignature(inspect.Signature): pass def foo(a, *, b:1): pass foo_sig = MySignature.from_callable(foo) - self.assertTrue(isinstance(foo_sig, MySignature)) + self.assertIsInstance(foo_sig, MySignature) + + def test_signature_from_callable_class(self): + # A regression test for a class inheriting its signature from `object`. + class MySignature(inspect.Signature): pass + class foo: pass + foo_sig = MySignature.from_callable(foo) + self.assertIsInstance(foo_sig, MySignature) @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_from_callable_builtin_obj(self): class MySignature(inspect.Signature): pass sig = MySignature.from_callable(_pickle.Pickler) - self.assertTrue(isinstance(sig, MySignature)) + self.assertIsInstance(sig, MySignature) def test_signature_definition_order_preserved_on_kwonly(self): for fn in signatures_with_lexicographic_keyword_only_parameters(): diff --git a/lib-python/3/test/test_io.py b/lib-python/3/test/test_io.py index d6bf43d0a0..7d50037bd7 100644 --- a/lib-python/3/test/test_io.py +++ b/lib-python/3/test/test_io.py @@ -726,6 +726,11 @@ class IOTest(unittest.TestCase): file.seek(0) file.close() self.assertRaises(ValueError, file.read) + with self.open(support.TESTFN, "rb") as f: + file = self.open(f.fileno(), "rb", closefd=False) + self.assertEqual(file.read()[:3], b"egg") + file.close() + self.assertRaises(ValueError, file.readinto, bytearray(1)) def test_no_closefd_with_filename(self): # can't use closefd in combination with a file name @@ -3830,6 +3835,17 @@ class MiscIOTest(unittest.TestCase): f.close() g.close() + def test_open_pipe_with_append(self): + # bpo-27805: Ignore ESPIPE from lseek() in open(). + r, w = os.pipe() + self.addCleanup(os.close, r) + f = self.open(w, 'a') + self.addCleanup(f.close) + # Check that the file is marked non-seekable. On Windows, however, lseek + # somehow succeeds on pipes. + if sys.platform != 'win32': + self.assertFalse(f.seekable()) + def test_io_after_close(self): for kwargs in [ {"mode": "w"}, diff --git a/lib-python/3/test/test_ipaddress.py b/lib-python/3/test/test_ipaddress.py index 3c50eec456..1fb6a929dc 100644 --- a/lib-python/3/test/test_ipaddress.py +++ b/lib-python/3/test/test_ipaddress.py @@ -12,6 +12,7 @@ import operator import pickle import ipaddress import weakref +from test.support import LARGEST, SMALLEST class BaseTestCase(unittest.TestCase): @@ -405,7 +406,13 @@ class NetmaskTestMixin_v4(CommonTestMixin_v4): """Input validation on interfaces and networks is very similar""" def test_no_mask(self): - self.assertEqual(str(self.factory('1.2.3.4')), '1.2.3.4/32') + for address in ('1.2.3.4', 0x01020304, b'\x01\x02\x03\x04'): + net = self.factory(address) + self.assertEqual(str(net), '1.2.3.4/32') + self.assertEqual(str(net.netmask), '255.255.255.255') + self.assertEqual(str(net.hostmask), '0.0.0.0') + # IPv4Network has prefixlen, but IPv4Interface doesn't. + # Should we add it to IPv4Interface too? (bpo-36392) def test_split_netmask(self): addr = "1.2.3.4/32/24" @@ -541,6 +548,15 @@ class NetworkTestCase_v4(BaseTestCase, NetmaskTestMixin_v4): class NetmaskTestMixin_v6(CommonTestMixin_v6): """Input validation on interfaces and networks is very similar""" + def test_no_mask(self): + for address in ('::1', 1, b'\x00'*15 + b'\x01'): + net = self.factory(address) + self.assertEqual(str(net), '::1/128') + self.assertEqual(str(net.netmask), 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff') + self.assertEqual(str(net.hostmask), '::') + # IPv6Network has prefixlen, but IPv6Interface doesn't. + # Should we add it to IPv4Interface too? (bpo-36392) + def test_split_netmask(self): addr = "cafe:cafe::/128/190" with self.assertAddressError("Only one '/' permitted in %r" % addr): @@ -664,20 +680,6 @@ class FactoryFunctionErrors(BaseTestCase): self.assertFactoryError(ipaddress.ip_network, "network") -@functools.total_ordering -class LargestObject: - def __eq__(self, other): - return isinstance(other, LargestObject) - def __lt__(self, other): - return False - -@functools.total_ordering -class SmallestObject: - def __eq__(self, other): - return isinstance(other, SmallestObject) - def __gt__(self, other): - return False - class ComparisonTests(unittest.TestCase): v4addr = ipaddress.IPv4Address(1) @@ -766,8 +768,6 @@ class ComparisonTests(unittest.TestCase): def test_foreign_type_ordering(self): other = object() - smallest = SmallestObject() - largest = LargestObject() for obj in self.objects: with self.assertRaises(TypeError): obj < other @@ -777,14 +777,14 @@ class ComparisonTests(unittest.TestCase): obj <= other with self.assertRaises(TypeError): obj >= other - self.assertTrue(obj < largest) - self.assertFalse(obj > largest) - self.assertTrue(obj <= largest) - self.assertFalse(obj >= largest) - self.assertFalse(obj < smallest) - self.assertTrue(obj > smallest) - self.assertFalse(obj <= smallest) - self.assertTrue(obj >= smallest) + self.assertTrue(obj < LARGEST) + self.assertFalse(obj > LARGEST) + self.assertTrue(obj <= LARGEST) + self.assertFalse(obj >= LARGEST) + self.assertFalse(obj < SMALLEST) + self.assertTrue(obj > SMALLEST) + self.assertFalse(obj <= SMALLEST) + self.assertTrue(obj >= SMALLEST) def test_mixed_type_key(self): # with get_mixed_type_key, you can sort addresses and network. @@ -2091,6 +2091,17 @@ class IpaddrUnitTest(unittest.TestCase): sixtofouraddr.sixtofour) self.assertFalse(bad_addr.sixtofour) + # issue41004 Hash collisions in IPv4Interface and IPv6Interface + def testV4HashIsNotConstant(self): + ipv4_address1 = ipaddress.IPv4Interface("1.2.3.4") + ipv4_address2 = ipaddress.IPv4Interface("2.3.4.5") + self.assertNotEqual(ipv4_address1.__hash__(), ipv4_address2.__hash__()) + + # issue41004 Hash collisions in IPv4Interface and IPv6Interface + def testV6HashIsNotConstant(self): + ipv6_address1 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:1") + ipv6_address2 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:2") + self.assertNotEqual(ipv6_address1.__hash__(), ipv6_address2.__hash__()) if __name__ == '__main__': unittest.main() diff --git a/lib-python/3/test/test_isinstance.py b/lib-python/3/test/test_isinstance.py index 65751ab916..53639e984e 100644 --- a/lib-python/3/test/test_isinstance.py +++ b/lib-python/3/test/test_isinstance.py @@ -251,6 +251,27 @@ class TestIsInstanceIsSubclass(unittest.TestCase): # blown self.assertRaises(RecursionError, blowstack, isinstance, '', str) + def test_issubclass_refcount_handling(self): + # bpo-39382: abstract_issubclass() didn't hold item reference while + # peeking in the bases tuple, in the single inheritance case. + class A: + @property + def __bases__(self): + return (int, ) + + class B: + def __init__(self): + # setting this here increases the chances of exhibiting the bug, + # probably due to memory layout changes. + self.x = 1 + + @property + def __bases__(self): + return (A(), ) + + self.assertEqual(True, issubclass(B(), int)) + + def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. diff --git a/lib-python/3/test/test_itertools.py b/lib-python/3/test/test_itertools.py index cbbb4c4f71..721a17556f 100644 --- a/lib-python/3/test/test_itertools.py +++ b/lib-python/3/test/test_itertools.py @@ -11,6 +11,7 @@ import pickle from functools import reduce import sys import struct +import threading maxsize = support.MAX_Py_ssize_t minsize = -maxsize-1 @@ -1476,6 +1477,42 @@ class TestBasicOps(unittest.TestCase): del forward, backward raise + def test_tee_reenter(self): + class I: + first = True + def __iter__(self): + return self + def __next__(self): + first = self.first + self.first = False + if first: + return next(b) + + a, b = tee(I()) + with self.assertRaisesRegex(RuntimeError, "tee"): + next(a) + + def test_tee_concurrent(self): + start = threading.Event() + finish = threading.Event() + class I: + def __iter__(self): + return self + def __next__(self): + start.set() + finish.wait() + + a, b = tee(I()) + thread = threading.Thread(target=next, args=[a]) + thread.start() + try: + start.wait() + with self.assertRaisesRegex(RuntimeError, "tee"): + next(b) + finally: + finish.set() + thread.join() + def test_StopIteration(self): self.assertRaises(StopIteration, next, zip()) diff --git a/lib-python/3/test/test_json/test_dump.py b/lib-python/3/test/test_json/test_dump.py index fd0d86b392..13b4002078 100644 --- a/lib-python/3/test/test_json/test_dump.py +++ b/lib-python/3/test/test_json/test_dump.py @@ -12,6 +12,16 @@ class TestDump: def test_dumps(self): self.assertEqual(self.dumps({}), '{}') + def test_dump_skipkeys(self): + v = {b'invalid_key': False, 'valid_key': True} + with self.assertRaises(TypeError): + self.json.dumps(v) + + s = self.json.dumps(v, skipkeys=True) + o = self.json.loads(s) + self.assertIn('valid_key', o) + self.assertNotIn(b'invalid_key', o) + def test_encode_truefalse(self): self.assertEqual(self.dumps( {True: False, False: True}, sort_keys=True), diff --git a/lib-python/3/test/test_json/test_tool.py b/lib-python/3/test/test_json/test_tool.py index 9d93f931ca..8dfbe8ad53 100644 --- a/lib-python/3/test/test_json/test_tool.py +++ b/lib-python/3/test/test_json/test_tool.py @@ -1,7 +1,9 @@ +import errno import os import sys import textwrap import unittest + from subprocess import Popen, PIPE from test import support from test.support.script_helper import assert_python_ok @@ -67,11 +69,11 @@ class TestTool(unittest.TestCase): self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) self.assertEqual(err, b'') - def _create_infile(self): + def _create_infile(self, data=None): infile = support.TESTFN - with open(infile, "w") as fp: + with open(infile, "w", encoding="utf-8") as fp: self.addCleanup(os.remove, infile) - fp.write(self.data) + fp.write(data or self.data) return infile def test_infile_stdout(self): @@ -81,6 +83,21 @@ class TestTool(unittest.TestCase): self.assertEqual(out.splitlines(), self.expect.encode().splitlines()) self.assertEqual(err, b'') + def test_non_ascii_infile(self): + data = '{"msg": "\u3053\u3093\u306b\u3061\u306f"}' + expect = textwrap.dedent('''\ + { + "msg": "\\u3053\\u3093\\u306b\\u3061\\u306f" + } + ''').encode() + + infile = self._create_infile(data) + rc, out, err = assert_python_ok('-m', 'json.tool', infile) + + self.assertEqual(rc, 0) + self.assertEqual(out.splitlines(), expect.splitlines()) + self.assertEqual(err, b'') + def test_infile_outfile(self): infile = self._create_infile() outfile = support.TESTFN + '.out' @@ -105,3 +122,12 @@ class TestTool(unittest.TestCase): self.assertEqual(out.splitlines(), self.expect_without_sort_keys.encode().splitlines()) self.assertEqual(err, b'') + + @unittest.skipIf(sys.platform =="win32", "The test is failed with ValueError on Windows") + def test_broken_pipe_error(self): + cmd = [sys.executable, '-m', 'json.tool'] + proc = Popen(cmd, stdout=PIPE, stdin=PIPE) + # bpo-39828: Closing before json.tool attempts to write into stdout. + proc.stdout.close() + proc.communicate(b'"{}"') + self.assertEqual(proc.returncode, errno.EPIPE) diff --git a/lib-python/3/test/test_list.py b/lib-python/3/test/test_list.py index def4badbf5..32bf17564c 100644 --- a/lib-python/3/test/test_list.py +++ b/lib-python/3/test/test_list.py @@ -149,6 +149,11 @@ class ListTest(list_tests.CommonTest): a[:] = data self.assertEqual(list(it), []) + def test_step_overflow(self): + a = [0, 1, 2, 3, 4] + a[1::sys.maxsize] = [0] + self.assertEqual(a[3::sys.maxsize], [3]) + def test_no_comdat_folding(self): # Issue 8847: In the PGO build, the MSVC linker's COMDAT folding # optimization causes failures in code that relies on distinct @@ -157,5 +162,63 @@ class ListTest(list_tests.CommonTest): with self.assertRaises(TypeError): (3,) + L([1,2]) + def test_equal_operator_modifying_operand(self): + # test fix for seg fault reported in bpo-38588 part 2. + class X: + def __eq__(self,other) : + list2.clear() + return NotImplemented + + class Y: + def __eq__(self, other): + list1.clear() + return NotImplemented + + class Z: + def __eq__(self, other): + list3.clear() + return NotImplemented + + list1 = [X()] + list2 = [Y()] + self.assertTrue(list1 == list2) + + list3 = [Z()] + list4 = [1] + self.assertFalse(list3 == list4) + + def test_count_index_remove_crashes(self): + # bpo-38610: The count(), index(), and remove() methods were not + # holding strong references to list elements while calling + # PyObject_RichCompareBool(). + class X: + def __eq__(self, other): + lst.clear() + return NotImplemented + + lst = [X()] + with self.assertRaises(ValueError): + lst.index(lst) + + class L(list): + def __eq__(self, other): + str(other) + return NotImplemented + + lst = L([X()]) + lst.count(lst) + + lst = L([X()]) + with self.assertRaises(ValueError): + lst.remove(lst) + + # bpo-39453: list.__contains__ was not holding strong references + # to list elements while calling PyObject_RichCompareBool(). + lst = [X(), X()] + 3 in lst + lst = [X(), X()] + X() in lst + + if __name__ == "__main__": unittest.main() diff --git a/lib-python/3/test/test_locale.py b/lib-python/3/test/test_locale.py index e2c2178ae6..9a05029b42 100644 --- a/lib-python/3/test/test_locale.py +++ b/lib-python/3/test/test_locale.py @@ -493,6 +493,42 @@ class NormalizeTest(unittest.TestCase): class TestMiscellaneous(unittest.TestCase): + def test_defaults_UTF8(self): + # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is + # valid. Futhermore LC_CTYPE=UTF is used by the UTF-8 locale coercing + # during interpreter startup (on macOS). + import _locale + import os + + self.assertEqual(locale._parse_localename('UTF-8'), (None, 'UTF-8')) + + if hasattr(_locale, '_getdefaultlocale'): + orig_getlocale = _locale._getdefaultlocale + del _locale._getdefaultlocale + else: + orig_getlocale = None + + orig_env = {} + try: + for key in ('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE'): + if key in os.environ: + orig_env[key] = os.environ[key] + del os.environ[key] + + os.environ['LC_CTYPE'] = 'UTF-8' + + self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) + + finally: + for k in orig_env: + os.environ[k] = orig_env[k] + + if 'LC_CTYPE' not in orig_env: + del os.environ['LC_CTYPE'] + + if orig_getlocale is not None: + _locale._getdefaultlocale = orig_getlocale + def test_getpreferredencoding(self): # Invoke getpreferredencoding to make sure it does not cause exceptions. enc = locale.getpreferredencoding() diff --git a/lib-python/3/test/test_logging.py b/lib-python/3/test/test_logging.py index 97c13a4c52..c24a302868 100644 --- a/lib-python/3/test/test_logging.py +++ b/lib-python/3/test/test_logging.py @@ -3223,6 +3223,37 @@ class ConfigDictTest(BaseTest): self.assertRaises(ValueError, bc.convert, 'cfg://!') self.assertRaises(KeyError, bc.convert, 'cfg://adict[2]') + def test_namedtuple(self): + # see bpo-39142 + from collections import namedtuple + + class MyHandler(logging.StreamHandler): + def __init__(self, resource, *args, **kwargs): + super().__init__(*args, **kwargs) + self.resource: namedtuple = resource + + def emit(self, record): + record.msg += f' {self.resource.type}' + return super().emit(record) + + Resource = namedtuple('Resource', ['type', 'labels']) + resource = Resource(type='my_type', labels=['a']) + + config = { + 'version': 1, + 'handlers': { + 'myhandler': { + '()': MyHandler, + 'resource': resource + } + }, + 'root': {'level': 'INFO', 'handlers': ['myhandler']}, + } + with support.captured_stderr() as stderr: + self.apply_config(config) + logging.info('some log') + self.assertEqual(stderr.getvalue(), 'some log my_type\n') + class ManagerTest(BaseTest): def test_manager_loggerclass(self): logged = [] @@ -3877,6 +3908,37 @@ class ModuleLevelMiscTest(BaseTest): logging.setLoggerClass(logging.Logger) self.assertEqual(logging.getLoggerClass(), logging.Logger) + def test_subclass_logger_cache(self): + # bpo-37258 + message = [] + + class MyLogger(logging.getLoggerClass()): + def __init__(self, name='MyLogger', level=logging.NOTSET): + super().__init__(name, level) + message.append('initialized') + + logging.setLoggerClass(MyLogger) + logger = logging.getLogger('just_some_logger') + self.assertEqual(message, ['initialized']) + stream = io.StringIO() + h = logging.StreamHandler(stream) + logger.addHandler(h) + try: + logger.setLevel(logging.DEBUG) + logger.debug("hello") + self.assertEqual(stream.getvalue().strip(), "hello") + + stream.truncate(0) + stream.seek(0) + + logger.setLevel(logging.INFO) + logger.debug("hello") + self.assertEqual(stream.getvalue(), "") + finally: + logger.removeHandler(h) + h.close() + logging.setLoggerClass(logging.Logger) + @support.requires_type_collecting def test_logging_at_shutdown(self): # Issue #20037 @@ -3989,7 +4051,7 @@ class BasicConfigTest(unittest.TestCase): logging._handlers.clear() logging._handlers.update(self.saved_handlers) logging._handlerList[:] = self.saved_handler_list - logging.root.level = self.original_logging_level + logging.root.setLevel(self.original_logging_level) def test_no_kwargs(self): logging.basicConfig() diff --git a/lib-python/3/test/test_lzma.py b/lib-python/3/test/test_lzma.py index 3dc2c1e7e3..49758ac7e7 100644 --- a/lib-python/3/test/test_lzma.py +++ b/lib-python/3/test/test_lzma.py @@ -1195,6 +1195,36 @@ class FileTestCase(unittest.TestCase): f.close() self.assertRaises(ValueError, f.tell) + def test_issue21872(self): + # sometimes decompress data incompletely + + # --------------------- + # when max_length == -1 + # --------------------- + d1 = LZMADecompressor() + entire = d1.decompress(ISSUE_21872_DAT, max_length=-1) + self.assertEqual(len(entire), 13160) + self.assertTrue(d1.eof) + + # --------------------- + # when max_length > 0 + # --------------------- + d2 = LZMADecompressor() + + # When this value of max_length is used, the input and output + # buffers are exhausted at the same time, and lzs's internal + # state still have 11 bytes can be output. + out1 = d2.decompress(ISSUE_21872_DAT, max_length=13149) + self.assertFalse(d2.needs_input) # ensure needs_input mechanism works + self.assertFalse(d2.eof) + + # simulate needs_input mechanism + # output internal state's 11 bytes + out2 = d2.decompress(b'') + self.assertEqual(len(out2), 11) + self.assertTrue(d2.eof) + self.assertEqual(out1 + out2, entire) + class OpenTestCase(unittest.TestCase): @@ -1761,6 +1791,139 @@ COMPRESSED_RAW_4 = ( b"\x00" ) +ISSUE_21872_DAT = ( + b']\x00\x00@\x00h3\x00\x00\x00\x00\x00\x00\x00\x00`D\x0c\x99\xc8' + b'\xd1\xbbZ^\xc43+\x83\xcd\xf1\xc6g\xec-\x061F\xb1\xbb\xc7\x17%-\xea' + b'\xfap\xfb\x8fs\x128\xb2,\x88\xe4\xc0\x12|*x\xd0\xa2\xc4b\x1b!\x02c' + b'\xab\xd9\x87U\xb8n \xfaVJ\x9a"\xb78\xff%_\x17`?@*\xc2\x82' + b"\xf2^\x1b\xb8\x04&\xc0\xbb\x03g\x9d\xca\xe9\xa4\xc9\xaf'\xe5\x8e}" + b'F\xdd\x11\xf3\x86\xbe\x1fN\x95\\\xef\xa2Mz-\xcb\x9a\xe3O@' + b"\x19\x07\xf6\xee\x9e\x9ag\xc6\xa5w\rnG'\x99\xfd\xfeGI\xb0" + b'\xbb\xf9\xc2\xe1\xff\xc5r\xcf\x85y[\x01\xa1\xbd\xcc/\xa3\x1b\x83\xaa' + b'\xc6\xf9\x99\x0c\xb6_\xc9MQ+x\xa2F\xda]\xdd\xe8\xfb\x1a&' + b',\xc4\x19\x1df\x81\x1e\x90\xf3\xb8Hgr\x85v\xbe\xa3qx\x01Y\xb5\x9fF' + b"\x13\x18\x01\xe69\x9b\xc8'\x1e\x9d\xd6\xe4F\x84\xac\xf8d<\x11\xd5" + b'\\\x0b\xeb\x0e\x82\xab\xb1\xe6\x1fka\xe1i\xc4 C\xb1"4)\xd6\xa7`\x02' + b'\xec\x11\x8c\xf0\x14\xb0\x1d\x1c\xecy\xf0\xb7|\x11j\x85X\xb2!\x1c' + b'\xac\xb5N\xc7\x85j\x9ev\xf5\xe6\x0b\xc1]c\xc15\x16\x9f\xd5\x99' + b"\xfei^\xd2G\x9b\xbdl\xab:\xbe,\xa9'4\x82\xe5\xee\xb3\xc1" + b'$\x93\x95\xa8Y\x16\xf5\xbf\xacw\x91\x04\x1d\x18\x06\xe9\xc5\xfdk\x06' + b'\xe8\xfck\xc5\x86>\x8b~\xa4\xcb\xf1\xb3\x04\xf1\x04G5\xe2\xcc]' + b'\x16\xbf\x140d\x18\xe2\xedw#(3\xca\xa1\x80bX\x7f\xb3\x84' + b'\x9d\xdb\xe7\x08\x97\xcd\x16\xb9\xf1\xd5r+m\x1e\xcb3q\xc5\x9e\x92' + b"\x7f\x8e*\xc7\xde\xe9\xe26\xcds\xb1\x10-\xf6r\x02?\x9d\xddCgJN'" + b'\x11M\xfa\nQ\n\xe6`m\xb8N\xbbq\x8el\x0b\x02\xc7:q\x04G\xa1T' + b'\xf1\xfe!0\x85~\xe5\x884\xe9\x89\xfb\x13J8\x15\xe42\xb6\xad' + b'\x877A\x9a\xa6\xbft]\xd0\xe35M\xb0\x0cK\xc8\xf6\x88\xae\xed\xa9,j7' + b'\x81\x13\xa0(\xcb\xe1\xe9l2\x7f\xcd\xda\x95(\xa70B\xbd\xf4\xe3' + b'hp\x94\xbdJ\xd7\t\xc7g\xffo?\x89?\xf8}\x7f\xbc\x1c\x87' + b'\x14\xc0\xcf\x8cV:\x9a\x0e\xd0\xb2\x1ck\xffk\xb9\xe0=\xc7\x8d/' + b'\xb8\xff\x7f\x1d\x87`\x19.\x98X*~\xa7j\xb9\x0b"\xf4\xe4;V`\xb9\xd7' + b'\x03\x1e\xd0t0\xd3\xde\x1fd\xb9\xe2)\x16\x81}\xb1\\b\x7fJ' + b'\x92\xf4\xff\n+V!\xe9\xde\x98\xa0\x8fK\xdf7\xb9\xc0\x12\x1f\xe2' + b'\xe9\xb0`\xae\x14\r\xa7\xc4\x81~\xd8\x8d\xc5\x06\xd8m\xb0Y\x8a)' + b'\x06/\xbb\xf9\xac\xeaP\xe0\x91\x05m[\xe5z\xe6Z\xf3\x9f\xc7\xd0' + b'\xd3\x8b\xf3\x8a\x1b\xfa\xe4Pf\xbc0\x17\x10\xa9\xd0\x95J{\xb3\xc3' + b'\xfdW\x9bop\x0f\xbe\xaee\xa3]\x93\x9c\xda\xb75<\xf6g!\xcc\xb1\xfc\\' + b'7\x152Mc\x17\x84\x9d\xcd35\r0\xacL-\xf3\xfb\xcb\x96\x1e\xe9U\x7f' + b'\xd7\xca\xb0\xcc\x89\x0c*\xce\x14\xd1P\xf1\x03\xb6.~9o?\xe8' + b'\r\x86\xe0\x92\x87}\xa3\x84\x03P\xe0\xc2\x7f\n;m\x9d\x9e\xb4|' + b'\x8c\x18\xc0#0\xfe3\x07<\xda\xd8\xcf^\xd4Hi\xd6\xb3\x0bT' + b'\x1dF\x88\x85q}\x02\xc6&\xc4\xae\xce\x9cU\xfa\x0f\xcc\xb6\x1f\x11' + b'drw\x9eN\x19\xbd\xffz\x0f\xf0\x04s\xadR\xc1\xc0\xbfl\xf1\xba\xf95^' + b'e\xb1\xfbVY\xd9\x9f\x1c\xbf*\xc4\xa86\x08+\xd6\x88[\xc4_rc\xf0f' + b'\xb8\xd4\xec\x1dx\x19|\xbf\xa7\xe0\x82\x0b\x8c~\x10L/\x90\xd6\xfb' + b'\x81\xdb\x98\xcc\x02\x14\xa5C\xb2\xa7i\xfd\xcd\x1fO\xf7\xe9\x89t\xf0' + b'\x17\xa5\x1c\xad\xfe<Q`%\x075k\n7\x9eI\x82<#)&\x04\xc2\xf0C\xd4`!' + b'\xcb\xa9\xf9\xb3F\x86\xb5\xc3M\xbeu\x12\xb2\xca\x95e\x10\x0b\xb1\xcc' + b'\x01b\x9bXa\x1b[B\x8c\x07\x11Of;\xeaC\xebr\x8eb\xd9\x9c\xe4i]<z\x9a' + b'\x03T\x8b9pF\x10\x8c\x84\xc7\x0e\xeaPw\xe5\xa0\x94\x1f\x84\xdd' + b'a\xe8\x85\xc2\x00\xebq\xe7&Wo5q8\xc2t\x98\xab\xb7\x7f\xe64-H' + b'\t\xb4d\xbe\x06\xe3Q\x8b\xa9J\xb0\x00\xd7s.\x85"\xc0p\x05' + b'\x1c\x06N\x87\xa5\xf8\xc3g\x1b}\x0f\x0f\xc3|\x90\xea\xefd3X' + b'[\xab\x04E\xf2\xf2\xc9\x08\x8a\xa8+W\xa2v\xec\x15G\x08/I<L\\1' + b'\xff\x15O\xaa\x89{\xd1mW\x13\xbd~\xe1\x90^\xc4@\r\xed\xb5D@\xb4\x08' + b'A\x90\xe69;\xc7BO\xdb\xda\xebu\x9e\xa9tN\xae\x8aJ5\xcd\x11\x1d\xea' + b"\xe5\xa7\x04\xe6\x82Z\xc7O\xe46[7\xdco*[\xbe\x0b\xc9\xb7a\xab'\xf6" + b"\xd1u\xdb\xd9q\xf5+y\x1b\x00\xb4\xf3a\xae\xf1M\xc4\xbc\xd00'\x06pQ" + b'\x8dH\xaa\xaa\xc4\xd2K\x9b\xc0\xe9\xec=n\xa9\x1a\x8a\xc2\xe8\x18\xbc' + b'\x93\xb8F\xa1\x8fOY\xe7\xda\xcf0\t\xff|\xd9\xe5\xcf\xe7\xf6\xbe' + b'\xf8\x04\x17\xf2\xe5P\xa7y~\xce\x11h0\x81\x80d[\x00_v\xbbc\xdbI' + b'3\xbc`W\xc0yrkB\xf5\x9f\xe9i\xc5\x8a^\x8d\xd4\x81\xd9\x05\xc1\xfc>' + b'"\xd1v`\x82\xd5$\x89\xcf^\xd52.\xafd\xe8d@\xaa\xd5Y|\x90\x84' + b'j\xdb}\x84riV\x8e\xf0X4rB\xf2NPS[\x8e\x88\xd4\x0fI\xb8' + b'\xdd\xcb\x1d\xf2(\xdf;9\x9e|\xef^0;.*[\x9fl\x7f\xa2_X\xaff!\xbb\x03' + b'\xff\x19\x8f\x88\xb5\xb6\x884\xa3\x05\xde3D{\xe3\xcb\xce\xe4t]' + b'\x875\xe3Uf\xae\xea\x88\x1c\x03b\n\xb1,Q\xec\xcf\x08\t\xde@\x83\xaa<' + b',-\xe4\xee\x9b\x843\xe5\x007\tK\xac\x057\xd6*X\xa3\xc6~\xba\xe6O' + b'\x81kz"\xbe\xe43sL\xf1\xfa;\xf4^\x1e\xb4\x80\xe2\xbd\xaa\x17Z\xe1f' + b'\xda\xa6\xb9\x07:]}\x9fa\x0b?\xba\xe7\xf15\x04M\xe3\n}M\xa4\xcb\r' + b'2\x8a\x88\xa9\xa7\x92\x93\x84\x81Yo\x00\xcc\xc4\xab\x9aT\x96\x0b\xbe' + b'U\xac\x1d\x8d\x1b\x98"\xf8\x8f\xf1u\xc1n\xcc\xfcA\xcc\x90\xb7i' + b'\x83\x9c\x9c~\x1d4\xa2\xf0*J\xe7t\x12\xb4\xe3\xa0u\xd7\x95Z' + b'\xf7\xafG\x96~ST,\xa7\rC\x06\xf4\xf0\xeb`2\x9e>Q\x0e\xf6\xf5\xc5' + b'\x9b\xb5\xaf\xbe\xa3\x8f\xc0\xa3hu\x14\x12 \x97\x99\x04b\x8e\xc7\x1b' + b'VKc\xc1\xf3 \xde\x85-:\xdc\x1f\xac\xce*\x06\xb3\x80;`' + b'\xdb\xdd\x97\xfdg\xbf\xe7\xa8S\x08}\xf55e7\xb8/\xf0!\xc8' + b"Y\xa8\x9a\x07'\xe2\xde\r\x02\xe1\xb2\x0c\xf4C\xcd\xf9\xcb(\xe8\x90" + b'\xd3bTD\x15_\xf6\xc3\xfb\xb3E\xfc\xd6\x98{\xc6\\fz\x81\xa99\x85\xcb' + b'\xa5\xb1\x1d\x94bqW\x1a!;z~\x18\x88\xe8i\xdb\x1b\x8d\x8d' + b'\x06\xaa\x0e\x99s+5k\x00\xe4\xffh\xfe\xdbt\xa6\x1bU\xde\xa3' + b'\xef\xcb\x86\x9e\x81\x16j\n\x9d\xbc\xbbC\x80?\x010\xc7Jj;' + b'\xc4\xe5\x86\xd5\x0e0d#\xc6;\xb8\xd1\xc7c\xb5&8?\xd9J\xe5\xden\xb9' + b'\xe9cb4\xbb\xe6\x14\xe0\xe7l\x1b\x85\x94\x1fh\xf1n\xdeZ\xbe' + b'\x88\xff\xc2e\xca\xdc,B-\x8ac\xc9\xdf\xf5|&\xe4LL\xf0\x1f\xaa8\xbd' + b'\xc26\x94bVi\xd3\x0c\x1c\xb6\xbb\x99F\x8f\x0e\xcc\x8e4\xc6/^W\xf5?' + b'\xdc\x84(\x14dO\x9aD6\x0f4\xa3,\x0c\x0bS\x9fJ\xe1\xacc^\x8a0\t\x80D[' + b'\xb8\xe6\x86\xb0\xe8\xd4\xf9\x1en\xf1\xf5^\xeb\xb8\xb8\xf8' + b')\xa8\xbf\xaa\x84\x86\xb1a \x95\x16\x08\x1c\xbb@\xbd+\r/\xfb' + b'\x92\xfbh\xf1\x8d3\xf9\x92\xde`\xf1\x86\x03\xaa+\xd9\xd9\xc6P\xaf' + b'\xe3-\xea\xa5\x0fB\xca\xde\xd5n^\xe3/\xbf\xa6w\xc8\x0e<M' + b'\xc2\x1e!\xd4\xc6E\xf2\xad\x0c\xbc\x1d\x88Y\x03\x98<\x92\xd9\xa6B' + b'\xc7\x83\xb5"\x97D|&\xc4\xd4\xfad\x0e\xde\x06\xa3\xc2\x9c`\xf2' + b'7\x03\x1a\xed\xd80\x10\xe9\x0co\x10\xcf\x18\x16\xa7\x1c' + b"\xe5\x96\xa4\xd9\xe1\xa5v;]\xb7\xa9\xdc'hA\xe3\x9c&\x98\x0b9\xdf~@" + b'\xf8\xact\x87<\xf94\x0c\x9d\x93\xb0)\xe1\xa2\x0f\x1e=:&\xd56\xa5A+' + b'\xab\xc4\x00\x8d\x81\x93\xd4\xd8<\x82k\\d\xd8v\xab\xbd^l5C?\xd4\xa0' + b'M\x12C\xc8\x80\r\xc83\xe8\xc0\xf5\xdf\xca\x05\xf4BPjy\xbe\x91\x9bzE' + b"\xd8[\x93oT\r\x13\x16'\x1a\xbd*H\xd6\xfe\r\xf3\x91M\x8b\xee\x8f7f" + b"\x0b;\xaa\x85\xf2\xdd'\x0fwM \xbd\x13\xb9\xe5\xb8\xb7 D+P\x1c\xe4g" + b'n\xd2\xf1kc\x15\xaf\xc6\x90V\x03\xc2UovfZ\xcc\xd23^\xb3\xe7\xbf' + b'\xacv\x1d\x82\xedx\xa3J\xa9\xb7\xcf\x0c\xe6j\x96n*o\x18>' + b'\xc6\xfd\x97_+D{\x03\x15\xe8s\xb1\xc8HAG\xcf\xf4\x1a\xdd' + b'\xad\x11\xbf\x157q+\xdeW\x89g"X\x82\xfd~\xf7\xab4\xf6`\xab\xf1q' + b')\x82\x10K\xe9sV\xfe\xe45\xafs*\x14\xa7;\xac{\x06\x9d<@\x93G' + b'j\x1d\xefL\xe9\xd8\x92\x19&\xa1\x16\x19\x04\tu5\x01]\xf6\xf4' + b'\xcd\\\xd8A|I\xd4\xeb\x05\x88C\xc6e\xacQ\xe9*\x97~\x9au\xf8Xy' + b"\x17P\x10\x9f\n\x8c\xe2fZEu>\x9b\x1e\x91\x0b'`\xbd\xc0\xa8\x86c\x1d" + b'Z\xe2\xdc8j\x95\xffU\x90\x1e\xf4o\xbc\xe5\xe3e>\xd2R\xc0b#\xbc\x15' + b'H-\xb9!\xde\x9d\x90k\xdew\x9b{\x99\xde\xf7/K)A\xfd\xf5\xe6:\xda' + b'UM\xcc\xbb\xa2\x0b\x9a\x93\xf5{9\xc0 \xd2((6i\xc0\xbbu\xd8\x9e\x8d' + b'\xf8\x04q\x10\xd4\x14\x9e7-\xb9B\xea\x01Q8\xc8v\x9a\x12A\x88Cd\x92' + b"\x1c\x8c!\xf4\x94\x87'\xe3\xcd\xae\xf7\xd8\x93\xfa\xde\xa8b\x9e\xee2" + b'K\xdb\x00l\x9d\t\xb1|D\x05U\xbb\xf4>\xf1w\x887\xd1}W\x9d|g|1\xb0\x13' + b"\xa3 \xe5\xbfm@\xc06+\xb7\t\xcf\x15D\x9a \x1fM\x1f\xd2\xb5'\xa9\xbb" + b'~Co\x82\xfa\xc2\t\xe6f\xfc\xbeI\xae1\x8e\xbe\xb8\xcf\x86\x17' + b'\x9f\xe2`\xbd\xaf\xba\xb9\xbc\x1b\xa3\xcd\x82\x8fwc\xefd\xa9\xd5\x14' + b'\xe2C\xafUE\xb6\x11MJH\xd0=\x05\xd4*I\xff"\r\x1b^\xcaS6=\xec@\xd5' + b'\x11,\xe0\x87Gr\xaa[\xb8\xbc>n\xbd\x81\x0c\x07<\xe9\x92(' + b'\xb2\xff\xac}\xe7\xb6\x15\x90\x9f~4\x9a\xe6\xd6\xd8s\xed\x99tf' + b'\xa0f\xf8\xf1\x87\t\x96/)\x85\xb6\n\xd7\xb2w\x0b\xbc\xba\x99\xee' + b'Q\xeen\x1d\xad\x03\xc3s\xd1\xfd\xa2\xc6\xb7\x9a\x9c(G<6\xad[~H ' + b'\x16\x89\x89\xd0\xc3\xd2\xca~\xac\xea\xa5\xed\xe5\xfb\r:' + b'\x8e\xa6\xf1e\xbb\xba\xbd\xe0(\xa3\x89_\x01(\xb5c\xcc\x9f\x1fg' + b'v\xfd\x17\xb3\x08S=S\xee\xfc\x85>\x91\x8d\x8d\nYR\xb3G\xd1A\xa2\xb1' + b'\xec\xb0\x01\xd2\xcd\xf9\xfe\x82\x06O\xb3\xecd\xa9c\xe0\x8eP\x90\xce' + b'\xe0\xcd\xd8\xd8\xdc\x9f\xaa\x01"[Q~\xe4\x88\xa1#\xc1\x12C\xcf' + b'\xbe\x80\x11H\xbf\x86\xd8\xbem\xcfWFQ(X\x01DK\xdfB\xaa\x10.-' + b'\xd5\x9e|\x86\x15\x86N]\xc7Z\x17\xcd=\xd7)M\xde\x15\xa4LTi\xa0\x15' + b'\xd1\xe7\xbdN\xa4?\xd1\xe7\x02\xfe4\xe4O\x89\x98&\x96\x0f\x02\x9c' + b'\x9e\x19\xaa\x13u7\xbd0\xdc\xd8\x93\xf4BNE\x1d\x93\x82\x81\x16' + b'\xe5y\xcf\x98D\xca\x9a\xe2\xfd\xcdL\xcc\xd1\xfc_\x0b\x1c\xa0]\xdc' + b'\xa91 \xc9c\xd8\xbf\x97\xcfp\xe6\x19-\xad\xff\xcc\xd1N(\xe8' + b'\xeb#\x182\x96I\xf7l\xf3r\x00' +) + def test_main(): run_unittest( diff --git a/lib-python/3/test/test_math.py b/lib-python/3/test/test_math.py index 44785d3e49..4a61260f04 100644 --- a/lib-python/3/test/test_math.py +++ b/lib-python/3/test/test_math.py @@ -1415,6 +1415,22 @@ class MathTests(unittest.TestCase): self.fail('Failures in test_mtestfile:\n ' + '\n '.join(failures)) + def test_issue39871(self): + # A SystemError should not be raised if the first arg to atan2(), + # copysign(), or remainder() cannot be converted to a float. + class F: + def __float__(self): + self.converted = True + 1/0 + for func in math.atan2, math.copysign, math.remainder: + y = F() + with self.assertRaises(TypeError): + func("not a number", y) + + # There should not have been any attempt to convert the second + # argument to a float. + self.assertFalse(getattr(y, "converted", False)) + # Custom assertions. def assertIsNaN(self, value): diff --git a/lib-python/3/test/test_mimetypes.py b/lib-python/3/test/test_mimetypes.py index 4e2c9089b5..f29de8c3a1 100644 --- a/lib-python/3/test/test_mimetypes.py +++ b/lib-python/3/test/test_mimetypes.py @@ -1,6 +1,7 @@ import io import locale import mimetypes +import pathlib import sys import unittest @@ -49,6 +50,21 @@ class MimeTypesTestCase(unittest.TestCase): eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') + def test_filename_with_url_delimiters(self): + # bpo-38449: URL delimiters cases should be handled also. + # They would have different mime types if interpreted as URL as + # compared to when interpreted as filename because of the semicolon. + eq = self.assertEqual + gzip_expected = ('application/x-tar', 'gzip') + eq(self.db.guess_type(";1.tar.gz"), gzip_expected) + eq(self.db.guess_type("?1.tar.gz"), gzip_expected) + eq(self.db.guess_type("#1.tar.gz"), gzip_expected) + eq(self.db.guess_type("#1#.tar.gz"), gzip_expected) + eq(self.db.guess_type(";1#.tar.gz"), gzip_expected) + eq(self.db.guess_type(";&1=123;?.tar.gz"), gzip_expected) + eq(self.db.guess_type("?k1=v1&k2=v2.tar.gz"), gzip_expected) + eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + def test_guess_all_types(self): eq = self.assertEqual unless = self.assertTrue @@ -77,6 +93,80 @@ class MimeTypesTestCase(unittest.TestCase): strict=True) self.assertEqual(exts, ['.g3', '.g\xb3']) + def test_init_reinitializes(self): + # Issue 4936: make sure an init starts clean + # First, put some poison into the types table + mimetypes.add_type('foo/bar', '.foobar') + self.assertEqual(mimetypes.guess_extension('foo/bar'), '.foobar') + # Reinitialize + mimetypes.init() + # Poison should be gone. + self.assertEqual(mimetypes.guess_extension('foo/bar'), None) + + def test_preferred_extension(self): + def check_extensions(): + self.assertEqual(mimetypes.guess_extension('application/octet-stream'), '.bin') + self.assertEqual(mimetypes.guess_extension('application/postscript'), '.ps') + self.assertEqual(mimetypes.guess_extension('application/vnd.apple.mpegurl'), '.m3u') + self.assertEqual(mimetypes.guess_extension('application/vnd.ms-excel'), '.xls') + self.assertEqual(mimetypes.guess_extension('application/vnd.ms-powerpoint'), '.ppt') + self.assertEqual(mimetypes.guess_extension('application/x-texinfo'), '.texi') + self.assertEqual(mimetypes.guess_extension('application/x-troff'), '.roff') + self.assertEqual(mimetypes.guess_extension('application/xml'), '.xsl') + self.assertEqual(mimetypes.guess_extension('audio/mpeg'), '.mp3') + self.assertEqual(mimetypes.guess_extension('image/jpeg'), '.jpg') + self.assertEqual(mimetypes.guess_extension('image/tiff'), '.tiff') + self.assertEqual(mimetypes.guess_extension('message/rfc822'), '.eml') + self.assertEqual(mimetypes.guess_extension('text/html'), '.html') + self.assertEqual(mimetypes.guess_extension('text/plain'), '.txt') + self.assertEqual(mimetypes.guess_extension('video/mpeg'), '.mpeg') + self.assertEqual(mimetypes.guess_extension('video/quicktime'), '.mov') + + check_extensions() + mimetypes.init() + check_extensions() + + def test_init_stability(self): + mimetypes.init() + + suffix_map = mimetypes.suffix_map + encodings_map = mimetypes.encodings_map + types_map = mimetypes.types_map + common_types = mimetypes.common_types + + mimetypes.init() + self.assertIsNot(suffix_map, mimetypes.suffix_map) + self.assertIsNot(encodings_map, mimetypes.encodings_map) + self.assertIsNot(types_map, mimetypes.types_map) + self.assertIsNot(common_types, mimetypes.common_types) + self.assertEqual(suffix_map, mimetypes.suffix_map) + self.assertEqual(encodings_map, mimetypes.encodings_map) + self.assertEqual(types_map, mimetypes.types_map) + self.assertEqual(common_types, mimetypes.common_types) + + def test_path_like_ob(self): + filename = "LICENSE.txt" + filepath = pathlib.Path(filename) + filepath_with_abs_dir = pathlib.Path('/dir/'+filename) + filepath_relative = pathlib.Path('../dir/'+filename) + path_dir = pathlib.Path('./') + + expected = self.db.guess_type(filename) + + self.assertEqual(self.db.guess_type(filepath), expected) + self.assertEqual(self.db.guess_type( + filepath_with_abs_dir), expected) + self.assertEqual(self.db.guess_type(filepath_relative), expected) + self.assertEqual(self.db.guess_type(path_dir), (None, None)) + + def test_keywords_args_api(self): + self.assertEqual(self.db.guess_type( + url="foo.html", strict=True), ("text/html", None)) + self.assertEqual(self.db.guess_all_extensions( + type='image/jpg', strict=True), []) + self.assertEqual(self.db.guess_extension( + type='image/jpg', strict=False), '.jpg') + @unittest.skipUnless(sys.platform.startswith("win"), "Windows only") class Win32MimeTypesTestCase(unittest.TestCase): diff --git a/lib-python/3/test/test_msilib.py b/lib-python/3/test/test_msilib.py index 265eaea59b..392886c6a9 100644 --- a/lib-python/3/test/test_msilib.py +++ b/lib-python/3/test/test_msilib.py @@ -86,6 +86,7 @@ class MsiDatabaseTestCase(unittest.TestCase): def test_directory_start_component_keyfile(self): db, db_path = init_database() self.addCleanup(db.Close) + self.addCleanup(msilib._directories.clear) feature = msilib.Feature(db, 0, 'Feature', 'A feature', 'Python') cab = msilib.CAB('CAB') dir = msilib.Directory(db, cab, None, TESTFN, 'TARGETDIR', diff --git a/lib-python/3/test/test_nntplib.py b/lib-python/3/test/test_nntplib.py index 618b403bfb..fbd7db03de 100644 --- a/lib-python/3/test/test_nntplib.py +++ b/lib-python/3/test/test_nntplib.py @@ -633,7 +633,7 @@ class NNTPv1Handler: "\tSat, 19 Jun 2010 18:04:08 -0400" "\t<4FD05F05-F98B-44DC-8111-C6009C925F0C@gmail.com>" "\t<hvalf7$ort$1@dough.gmane.org>\t7103\t16" - "\tXref: news.gmane.org gmane.comp.python.authors:57" + "\tXref: news.gmane.io gmane.comp.python.authors:57" "\n" "58\tLooking for a few good bloggers" "\tDoug Hellmann <doug.hellmann-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org>" @@ -1119,7 +1119,7 @@ class NNTPv1v2TestsMixin: "references": "<hvalf7$ort$1@dough.gmane.org>", ":bytes": "7103", ":lines": "16", - "xref": "news.gmane.org gmane.comp.python.authors:57" + "xref": "news.gmane.io gmane.comp.python.authors:57" }) art_num, over = overviews[1] self.assertEqual(over["xref"], None) diff --git a/lib-python/3/test/test_ntpath.py b/lib-python/3/test/test_ntpath.py index 6e31b64411..89b0d5303d 100644 --- a/lib-python/3/test/test_ntpath.py +++ b/lib-python/3/test/test_ntpath.py @@ -14,10 +14,19 @@ except ImportError: # but for those that require it we import here. nt = None + +def _norm(path): + if isinstance(path, (bytes, str, os.PathLike)): + return ntpath.normcase(os.fsdecode(path)) + elif hasattr(path, "__iter__"): + return tuple(ntpath.normcase(os.fsdecode(p)) for p in path) + return path + + def tester(fn, wantResult): fn = fn.replace("\\", "\\\\") gotResult = eval(fn) - if wantResult != gotResult: + if wantResult != gotResult and _norm(wantResult) != _norm(gotResult): raise TestFailed("%s should return: %s but returned: %s" \ %(str(fn), str(wantResult), str(gotResult))) @@ -33,18 +42,22 @@ def tester(fn, wantResult): with warnings.catch_warnings(): warnings.simplefilter("ignore", DeprecationWarning) gotResult = eval(fn) - if isinstance(wantResult, str): - wantResult = os.fsencode(wantResult) - elif isinstance(wantResult, tuple): - wantResult = tuple(os.fsencode(r) for r in wantResult) - - gotResult = eval(fn) - if wantResult != gotResult: + if _norm(wantResult) != _norm(gotResult): raise TestFailed("%s should return: %s but returned: %s" \ %(str(fn), str(wantResult), repr(gotResult))) -class TestNtpath(unittest.TestCase): +class NtpathTestCase(unittest.TestCase): + def assertPathEqual(self, path1, path2): + if path1 == path2 or _norm(path1) == _norm(path2): + return + self.assertEqual(path1, path2) + + def assertPathIn(self, path, pathset): + self.assertIn(_norm(path), _norm(pathset)) + + +class TestNtpath(NtpathTestCase): def test_splitext(self): tester('ntpath.splitext("foo.ext")', ('foo', '.ext')) tester('ntpath.splitext("/foo/foo.ext")', ('/foo/foo', '.ext')) @@ -461,7 +474,7 @@ class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase): attributes = ['relpath'] -class PathLikeTests(unittest.TestCase): +class PathLikeTests(NtpathTestCase): path = ntpath @@ -472,67 +485,67 @@ class PathLikeTests(unittest.TestCase): with open(self.file_name, 'xb', 0) as file: file.write(b"test_ntpath.PathLikeTests") - def assertPathEqual(self, func): - self.assertEqual(func(self.file_path), func(self.file_name)) + def _check_function(self, func): + self.assertPathEqual(func(self.file_path), func(self.file_name)) def test_path_normcase(self): - self.assertPathEqual(self.path.normcase) + self._check_function(self.path.normcase) def test_path_isabs(self): - self.assertPathEqual(self.path.isabs) + self._check_function(self.path.isabs) def test_path_join(self): self.assertEqual(self.path.join('a', FakePath('b'), 'c'), self.path.join('a', 'b', 'c')) def test_path_split(self): - self.assertPathEqual(self.path.split) + self._check_function(self.path.split) def test_path_splitext(self): - self.assertPathEqual(self.path.splitext) + self._check_function(self.path.splitext) def test_path_splitdrive(self): - self.assertPathEqual(self.path.splitdrive) + self._check_function(self.path.splitdrive) def test_path_basename(self): - self.assertPathEqual(self.path.basename) + self._check_function(self.path.basename) def test_path_dirname(self): - self.assertPathEqual(self.path.dirname) + self._check_function(self.path.dirname) def test_path_islink(self): - self.assertPathEqual(self.path.islink) + self._check_function(self.path.islink) def test_path_lexists(self): - self.assertPathEqual(self.path.lexists) + self._check_function(self.path.lexists) def test_path_ismount(self): - self.assertPathEqual(self.path.ismount) + self._check_function(self.path.ismount) def test_path_expanduser(self): - self.assertPathEqual(self.path.expanduser) + self._check_function(self.path.expanduser) def test_path_expandvars(self): - self.assertPathEqual(self.path.expandvars) + self._check_function(self.path.expandvars) def test_path_normpath(self): - self.assertPathEqual(self.path.normpath) + self._check_function(self.path.normpath) def test_path_abspath(self): - self.assertPathEqual(self.path.abspath) + self._check_function(self.path.abspath) def test_path_realpath(self): - self.assertPathEqual(self.path.realpath) + self._check_function(self.path.realpath) def test_path_relpath(self): - self.assertPathEqual(self.path.relpath) + self._check_function(self.path.relpath) def test_path_commonpath(self): common_path = self.path.commonpath([self.file_path, self.file_name]) - self.assertEqual(common_path, self.file_name) + self.assertPathEqual(common_path, self.file_name) def test_path_isdir(self): - self.assertPathEqual(self.path.isdir) + self._check_function(self.path.isdir) if __name__ == "__main__": diff --git a/lib-python/3/test/test_ordered_dict.py b/lib-python/3/test/test_ordered_dict.py index b1d7f86a67..0e5c7fc750 100644 --- a/lib-python/3/test/test_ordered_dict.py +++ b/lib-python/3/test/test_ordered_dict.py @@ -749,6 +749,26 @@ class CPythonOrderedDictTests(OrderedDictTests, unittest.TestCase): self.assertEqual(list(unpickled), expected) self.assertEqual(list(it), expected) + @support.cpython_only + def test_weakref_list_is_not_traversed(self): + # Check that the weakref list is not traversed when collecting + # OrderedDict objects. See bpo-39778 for more information. + + gc.collect() + + x = self.OrderedDict() + x.cycle = x + + cycle = [] + cycle.append(cycle) + + x_ref = weakref.ref(x) + cycle.append(x_ref) + + del x, cycle, x_ref + + gc.collect() + class PurePythonOrderedDictSubclassTests(PurePythonOrderedDictTests): diff --git a/lib-python/3/test/test_os.py b/lib-python/3/test/test_os.py index a917050400..3a76940a08 100644 --- a/lib-python/3/test/test_os.py +++ b/lib-python/3/test/test_os.py @@ -42,15 +42,6 @@ try: except ImportError: _winapi = None try: - import grp - groups = [g.gr_gid for g in grp.getgrall() if getpass.getuser() in g.gr_mem] - if hasattr(os, 'getgid'): - process_gid = os.getgid() - if process_gid not in groups: - groups.append(process_gid) -except ImportError: - groups = [] -try: import pwd all_users = [u.pw_uid for u in pwd.getpwall()] except (ImportError, AttributeError): @@ -1238,13 +1229,19 @@ class ChownFileTests(unittest.TestCase): self.assertIsNone(os.chown(support.TESTFN, uid, gid)) self.assertIsNone(os.chown(support.TESTFN, -1, -1)) - @unittest.skipUnless(len(groups) > 1, "test needs more than one group") - def test_chown(self): + @unittest.skipUnless(hasattr(os, 'getgroups'), 'need os.getgroups') + def test_chown_gid(self): + groups = os.getgroups() + if len(groups) < 2: + self.skipTest("test needs at least 2 groups") + gid_1, gid_2 = groups[:2] uid = os.stat(support.TESTFN).st_uid + os.chown(support.TESTFN, uid, gid_1) gid = os.stat(support.TESTFN).st_gid self.assertEqual(gid, gid_1) + os.chown(support.TESTFN, uid, gid_2) gid = os.stat(support.TESTFN).st_gid self.assertEqual(gid, gid_2) @@ -2420,12 +2417,37 @@ class PidTests(unittest.TestCase): # We are the parent of our subprocess self.assertEqual(int(stdout), os.getpid()) + def check_waitpid(self, code, exitcode): + if sys.platform == 'win32': + # On Windows, os.spawnv() simply joins arguments with spaces: + # arguments need to be quoted + args = [f'"{sys.executable}"', '-c', f'"{code}"'] + else: + args = [sys.executable, '-c', code] + pid = os.spawnv(os.P_NOWAIT, sys.executable, args) + + pid2, status = os.waitpid(pid, 0) + if sys.platform == 'win32': + self.assertEqual(status, exitcode << 8) + else: + self.assertTrue(os.WIFEXITED(status), status) + self.assertEqual(os.WEXITSTATUS(status), exitcode) + self.assertEqual(pid2, pid) + def test_waitpid(self): - args = [sys.executable, '-c', 'pass'] - # Add an implicit test for PyUnicode_FSConverter(). - pid = os.spawnv(os.P_NOWAIT, FakePath(args[0]), args) - status = os.waitpid(pid, 0) - self.assertEqual(status, (pid, 0)) + self.check_waitpid(code='pass', exitcode=0) + + def test_waitpid_exitcode(self): + exitcode = 23 + code = f'import sys; sys.exit({exitcode})' + self.check_waitpid(code, exitcode=exitcode) + + @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') + def test_waitpid_windows(self): + # bpo-40138: test os.waitpid() with exit code larger than INT_MAX. + STATUS_CONTROL_C_EXIT = 0xC000013A + code = f'import _winapi; _winapi.ExitProcess({STATUS_CONTROL_C_EXIT})' + self.check_waitpid(code, exitcode=STATUS_CONTROL_C_EXIT) class SpawnTests(unittest.TestCase): @@ -3240,6 +3262,11 @@ class FDInheritanceTests(unittest.TestCase): self.addCleanup(os.close, fd2) self.assertEqual(os.get_inheritable(fd2), False) + def test_dup_standard_stream(self): + fd = os.dup(1) + self.addCleanup(os.close, fd) + self.assertGreater(fd, 0) + @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_dup_nul(self): # os.dup() was creating inheritable fds for character files. @@ -3743,6 +3770,14 @@ class TestPEP519(unittest.TestCase): self.assertRaises(ZeroDivisionError, self.fspath, FakePath(ZeroDivisionError())) + def test_pathlike_subclasshook(self): + # bpo-38878: subclasshook causes subclass checks + # true on abstract implementation. + class A(os.PathLike): + pass + self.assertFalse(issubclass(FakePath, A)) + self.assertTrue(issubclass(FakePath, os.PathLike)) + class TimesTests(unittest.TestCase): def test_times(self): diff --git a/lib-python/3/test/test_pathlib.py b/lib-python/3/test/test_pathlib.py index f7ed1d1e48..83181b4322 100644 --- a/lib-python/3/test/test_pathlib.py +++ b/lib-python/3/test/test_pathlib.py @@ -1271,10 +1271,13 @@ class _BasePathTest(object): func(*args, **kwargs) self.assertEqual(cm.exception.errno, errno.ENOENT) + def assertEqualNormCase(self, path_a, path_b): + self.assertEqual(os.path.normcase(path_a), os.path.normcase(path_b)) + def _test_cwd(self, p): q = self.cls(os.getcwd()) self.assertEqual(p, q) - self.assertEqual(str(p), str(q)) + self.assertEqualNormCase(str(p), str(q)) self.assertIs(type(p), type(q)) self.assertTrue(p.is_absolute()) @@ -1285,7 +1288,7 @@ class _BasePathTest(object): def _test_home(self, p): q = self.cls(os.path.expanduser('~')) self.assertEqual(p, q) - self.assertEqual(str(p), str(q)) + self.assertEqualNormCase(str(p), str(q)) self.assertIs(type(p), type(q)) self.assertTrue(p.is_absolute()) @@ -1475,6 +1478,42 @@ class _BasePathTest(object): self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) self.assertEqual(set(p.glob("../xyzzy")), set()) + @support.skip_unless_symlink + def test_glob_permissions(self): + # See bpo-38894 + P = self.cls + base = P(BASE) / 'permissions' + base.mkdir() + + file1 = base / "file1" + file1.touch() + file2 = base / "file2" + file2.touch() + + subdir = base / "subdir" + + file3 = base / "file3" + file3.symlink_to(subdir / "other") + + # Patching is needed to avoid relying on the filesystem + # to return the order of the files as the error will not + # happen if the symlink is the last item. + + with mock.patch("os.scandir") as scandir: + scandir.return_value = sorted(os.scandir(base)) + self.assertEqual(len(set(base.glob("*"))), 3) + + subdir.mkdir() + + with mock.patch("os.scandir") as scandir: + scandir.return_value = sorted(os.scandir(base)) + self.assertEqual(len(set(base.glob("*"))), 4) + + subdir.chmod(000) + + with mock.patch("os.scandir") as scandir: + scandir.return_value = sorted(os.scandir(base)) + self.assertEqual(len(set(base.glob("*"))), 4) def _check_resolve(self, p, expected, strict=True): q = p.resolve(strict) @@ -1491,15 +1530,15 @@ class _BasePathTest(object): p.resolve(strict=True) self.assertEqual(cm.exception.errno, errno.ENOENT) # Non-strict - self.assertEqual(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo')) + self.assertEqualNormCase(str(p.resolve(strict=False)), + os.path.join(BASE, 'foo')) p = P(BASE, 'foo', 'in', 'spam') - self.assertEqual(str(p.resolve(strict=False)), - os.path.join(BASE, 'foo', 'in', 'spam')) + self.assertEqualNormCase(str(p.resolve(strict=False)), + os.path.join(BASE, 'foo', 'in', 'spam')) p = P(BASE, '..', 'foo', 'in', 'spam') - self.assertEqual(str(p.resolve(strict=False)), - os.path.abspath(os.path.join('foo', 'in', 'spam'))) - # These are all relative symlinks + self.assertEqualNormCase(str(p.resolve(strict=False)), + os.path.abspath(os.path.join('foo', 'in', 'spam'))) + # These are all relative symlinks. p = P(BASE, 'dirB', 'fileB') self._check_resolve_relative(p, p) p = P(BASE, 'linkA') @@ -1996,16 +2035,16 @@ class _BasePathTest(object): # Resolve absolute paths p = (P / 'link0').resolve() self.assertEqual(p, P) - self.assertEqual(str(p), BASE) + self.assertEqualNormCase(str(p), BASE) p = (P / 'link1').resolve() self.assertEqual(p, P) - self.assertEqual(str(p), BASE) + self.assertEqualNormCase(str(p), BASE) p = (P / 'link2').resolve() self.assertEqual(p, P) - self.assertEqual(str(p), BASE) + self.assertEqualNormCase(str(p), BASE) p = (P / 'link3').resolve() self.assertEqual(p, P) - self.assertEqual(str(p), BASE) + self.assertEqualNormCase(str(p), BASE) # Resolve relative paths old_path = os.getcwd() @@ -2013,16 +2052,16 @@ class _BasePathTest(object): try: p = self.cls('link0').resolve() self.assertEqual(p, P) - self.assertEqual(str(p), BASE) + self.assertEqualNormCase(str(p), BASE) p = self.cls('link1').resolve() self.assertEqual(p, P) - self.assertEqual(str(p), BASE) + self.assertEqualNormCase(str(p), BASE) p = self.cls('link2').resolve() self.assertEqual(p, P) - self.assertEqual(str(p), BASE) + self.assertEqualNormCase(str(p), BASE) p = self.cls('link3').resolve() self.assertEqual(p, P) - self.assertEqual(str(p), BASE) + self.assertEqualNormCase(str(p), BASE) finally: os.chdir(old_path) @@ -2213,11 +2252,15 @@ class WindowsPathTest(_BasePathTest, unittest.TestCase): P = self.cls p = P(BASE) self.assertEqual(set(p.glob("FILEa")), { P(BASE, "fileA") }) + self.assertEqual(set(p.glob("F*a")), { P(BASE, "fileA") }) + self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\FILEa"}) + self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) def test_rglob(self): P = self.cls p = P(BASE, "dirC") self.assertEqual(set(p.rglob("FILEd")), { P(BASE, "dirC/dirD/fileD") }) + self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\FILEd"}) def test_expanduser(self): P = self.cls diff --git a/lib-python/3/test/test_pdb.py b/lib-python/3/test/test_pdb.py index de6c651b37..63909e21f2 100644 --- a/lib-python/3/test/test_pdb.py +++ b/lib-python/3/test/test_pdb.py @@ -1350,6 +1350,19 @@ class PdbTestCase(unittest.TestCase): if save_home is not None: os.environ['HOME'] = save_home + def test_readrc_homedir(self): + save_home = os.environ.pop("HOME", None) + with support.temp_dir() as temp_dir, patch("os.path.expanduser"): + rc_path = os.path.join(temp_dir, ".pdbrc") + os.path.expanduser.return_value = rc_path + try: + with open(rc_path, "w") as f: + f.write("invalid") + self.assertEqual(pdb.Pdb().rcLines[0], "invalid") + finally: + if save_home is not None: + os.environ["HOME"] = save_home + def test_header(self): stdout = StringIO() header = 'Nobody expects... blah, blah, blah' diff --git a/lib-python/3/test/test_platform.py b/lib-python/3/test/test_platform.py index d91e978a79..452a56e8b4 100644 --- a/lib-python/3/test/test_platform.py +++ b/lib-python/3/test/test_platform.py @@ -236,6 +236,11 @@ class PlatformTest(unittest.TestCase): fd.close() self.assertFalse(real_ver is None) result_list = res[0].split('.') + # macOS 11.0 (Big Sur) may report its version number + # as 10.16 if the executable is built with an older + # SDK target but sw_vers reports 11.0. + if result_list == ['10', '16']: + result_list = ['11', '0'] expect_list = real_ver.split('.') len_diff = len(result_list) - len(expect_list) # On Snow Leopard, sw_vers reports 10.6.0 as 10.6 diff --git a/lib-python/3/test/test_posix.py b/lib-python/3/test/test_posix.py index 233fea4d57..1cd9e567b3 100644 --- a/lib-python/3/test/test_posix.py +++ b/lib-python/3/test/test_posix.py @@ -1375,6 +1375,7 @@ class PosixTester(unittest.TestCase): self.assertEqual(posix.sched_getaffinity(0), mask) self.assertRaises(OSError, posix.sched_setaffinity, 0, []) self.assertRaises(ValueError, posix.sched_setaffinity, 0, [-10]) + self.assertRaises(ValueError, posix.sched_setaffinity, 0, map(int, "0X")) self.assertRaises(OverflowError, posix.sched_setaffinity, 0, [1<<128]) self.assertRaises(OSError, posix.sched_setaffinity, -1, mask) diff --git a/lib-python/3/test/test_pty.py b/lib-python/3/test/test_pty.py index 3b448569a2..dfb3a3fc32 100644 --- a/lib-python/3/test/test_pty.py +++ b/lib-python/3/test/test_pty.py @@ -66,16 +66,27 @@ def _readline(fd): # XXX(nnorwitz): these tests leak fds when there is an error. class PtyTest(unittest.TestCase): def setUp(self): - # isatty() and close() can hang on some platforms. Set an alarm - # before running the test to make sure we don't hang forever. old_alarm = signal.signal(signal.SIGALRM, self.handle_sig) self.addCleanup(signal.signal, signal.SIGALRM, old_alarm) + + old_sighup = signal.signal(signal.SIGHUP, self.handle_sighup) + self.addCleanup(signal.signal, signal.SIGHUP, old_sighup) + + # isatty() and close() can hang on some platforms. Set an alarm + # before running the test to make sure we don't hang forever. self.addCleanup(signal.alarm, 0) signal.alarm(10) def handle_sig(self, sig, frame): self.fail("isatty hung") + @staticmethod + def handle_sighup(signum, frame): + # bpo-38547: if the process is the session leader, os.close(master_fd) + # of "master_fd, slave_name = pty.master_open()" raises SIGHUP + # signal: just ignore the signal. + pass + def test_basic(self): try: debug("Calling master_open()") @@ -122,9 +133,11 @@ class PtyTest(unittest.TestCase): self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2)) os.close(slave_fd) + # closing master_fd can raise a SIGHUP if the process is + # the session leader: we installed a SIGHUP signal handler + # to ignore this signal. os.close(master_fd) - def test_fork(self): debug("calling pty.fork()") pid, master_fd = pty.fork() diff --git a/lib-python/3/test/test_pwd.py b/lib-python/3/test/test_pwd.py index c13a7c9294..85740cecd8 100644 --- a/lib-python/3/test/test_pwd.py +++ b/lib-python/3/test/test_pwd.py @@ -21,7 +21,7 @@ class PwdTest(unittest.TestCase): self.assertEqual(e[3], e.pw_gid) self.assertIsInstance(e.pw_gid, int) self.assertEqual(e[4], e.pw_gecos) - self.assertIsInstance(e.pw_gecos, str) + self.assertIn(type(e.pw_gecos), (str, type(None))) self.assertEqual(e[5], e.pw_dir) self.assertIsInstance(e.pw_dir, str) self.assertEqual(e[6], e.pw_shell) diff --git a/lib-python/3/test/test_py_compile.py b/lib-python/3/test/test_py_compile.py index f86abe26f9..df45764f04 100644 --- a/lib-python/3/test/test_py_compile.py +++ b/lib-python/3/test/test_py_compile.py @@ -51,7 +51,7 @@ class SourceDateEpochTestMeta(type(unittest.TestCase)): class PyCompileTestsBase: def setUp(self): - self.directory = tempfile.mkdtemp() + self.directory = tempfile.mkdtemp(dir=os.getcwd()) self.source_path = os.path.join(self.directory, '_test.py') self.pyc_path = self.source_path + 'c' self.cache_path = importlib.util.cache_from_source(self.source_path) diff --git a/lib-python/3/test/test_random.py b/lib-python/3/test/test_random.py index f0822cddec..6b9e90594c 100644 --- a/lib-python/3/test/test_random.py +++ b/lib-python/3/test/test_random.py @@ -228,7 +228,7 @@ class TestBasicOps: choices([], cum_weights=[], k=5) def test_choices_subnormal(self): - # Subnormal weights would occassionally trigger an IndexError + # Subnormal weights would occasionally trigger an IndexError # in choices() when the value returned by random() was large # enough to make `random() * total` round up to the total. # See https://bugs.python.org/msg275594 for more detail. diff --git a/lib-python/3/test/test_regrtest.py b/lib-python/3/test/test_regrtest.py index 4c61521536..a5ac810200 100644 --- a/lib-python/3/test/test_regrtest.py +++ b/lib-python/3/test/test_regrtest.py @@ -24,6 +24,7 @@ from test.libregrtest import utils Py_DEBUG = hasattr(sys, 'gettotalrefcount') ROOT_DIR = os.path.join(os.path.dirname(__file__), '..', '..') ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR)) +LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?' TEST_INTERRUPTED = textwrap.dedent(""" from signal import SIGINT @@ -156,6 +157,24 @@ class ParseArgsTestCase(unittest.TestCase): self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") + def test_ignore(self): + for opt in '-i', '--ignore': + with self.subTest(opt=opt): + ns = libregrtest._parse_args([opt, 'pattern']) + self.assertEqual(ns.ignore_tests, ['pattern']) + self.checkError([opt], 'expected one argument') + + self.addCleanup(support.unlink, support.TESTFN) + with open(support.TESTFN, "w") as fp: + print('matchfile1', file=fp) + print('matchfile2', file=fp) + + filename = os.path.abspath(support.TESTFN) + ns = libregrtest._parse_args(['-m', 'match', + '--ignorefile', filename]) + self.assertEqual(ns.ignore_tests, + ['matchfile1', 'matchfile2']) + def test_match(self): for opt in '-m', '--match': with self.subTest(opt=opt): @@ -390,8 +409,8 @@ class BaseTestCase(unittest.TestCase): self.assertRegex(output, regex) def parse_executed_tests(self, output): - regex = (r'^[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?\[ *[0-9]+(?:/ *[0-9]+)*\] (%s)' - % self.TESTNAME_REGEX) + regex = (r'^%s\[ *[0-9]+(?:/ *[0-9]+)*\] (%s)' + % (LOG_PREFIX, self.TESTNAME_REGEX)) parser = re.finditer(regex, output, re.MULTILINE) return list(match.group(1) for match in parser) @@ -451,9 +470,10 @@ class BaseTestCase(unittest.TestCase): if rerun: regex = list_regex('%s re-run test%s', rerun) self.check_line(output, regex) - self.check_line(output, "Re-running failed tests in verbose mode") + regex = LOG_PREFIX + r"Re-running failed tests in verbose mode" + self.check_line(output, regex) for test_name in rerun: - regex = f"Re-running {test_name} in verbose mode" + regex = LOG_PREFIX + f"Re-running {test_name} in verbose mode" self.check_line(output, regex) if no_test_ran: @@ -929,6 +949,42 @@ class ArgsTestCase(BaseTestCase): regex = re.compile("^(test[^ ]+).*ok$", flags=re.MULTILINE) return [match.group(1) for match in regex.finditer(output)] + def test_ignorefile(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_method1(self): + pass + def test_method2(self): + pass + def test_method3(self): + pass + def test_method4(self): + pass + """) + all_methods = ['test_method1', 'test_method2', + 'test_method3', 'test_method4'] + testname = self.create_test(code=code) + + # only run a subset + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + subset = [ + # only ignore the method name + 'test_method1', + # ignore the full identifier + '%s.Tests.test_method3' % testname] + with open(filename, "w") as fp: + for name in subset: + print(name, file=fp) + + output = self.run_tests("-v", "--ignorefile", filename, testname) + methods = self.parse_methods(output) + subset = ['test_method2', 'test_method4'] + self.assertEqual(methods, subset) + def test_matchfile(self): code = textwrap.dedent(""" import unittest @@ -1121,6 +1177,48 @@ class ArgsTestCase(BaseTestCase): env_changed=[testname], fail_env_changed=True) + def test_multiprocessing_timeout(self): + code = textwrap.dedent(r""" + import time + import unittest + try: + import faulthandler + except ImportError: + faulthandler = None + + class Tests(unittest.TestCase): + # test hangs and so should be stopped by the timeout + def test_sleep(self): + # we want to test regrtest multiprocessing timeout, + # not faulthandler timeout + if faulthandler is not None: + faulthandler.cancel_dump_traceback_later() + + time.sleep(60 * 5) + """) + testname = self.create_test(code=code) + + output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2) + self.check_executed_tests(output, [testname], + failed=testname) + self.assertRegex(output, + re.compile('%s timed out' % testname, re.MULTILINE)) + + def test_cleanup(self): + dirname = os.path.join(self.tmptestdir, "test_python_123") + os.mkdir(dirname) + filename = os.path.join(self.tmptestdir, "test_python_456") + open(filename, "wb").close() + names = [dirname, filename] + + cmdargs = ['-m', 'test', + '--tempdir=%s' % self.tmptestdir, + '--cleanup'] + self.run_python(cmdargs) + + for name in names: + self.assertFalse(os.path.exists(name), name) + class TestUtils(unittest.TestCase): def test_format_duration(self): @@ -1131,9 +1229,9 @@ class TestUtils(unittest.TestCase): self.assertEqual(utils.format_duration(10e-3), '10 ms') self.assertEqual(utils.format_duration(1.5), - '1 sec 500 ms') + '1.5 sec') self.assertEqual(utils.format_duration(1), - '1 sec') + '1.0 sec') self.assertEqual(utils.format_duration(2 * 60), '2 min') self.assertEqual(utils.format_duration(2 * 60 + 1), diff --git a/lib-python/3/test/test_shlex.py b/lib-python/3/test/test_shlex.py index fd35788e81..6d4627d7f4 100644 --- a/lib-python/3/test/test_shlex.py +++ b/lib-python/3/test/test_shlex.py @@ -308,6 +308,14 @@ class ShlexTest(unittest.TestCase): self.assertEqual(shlex.quote("test%s'name'" % u), "'test%s'\"'\"'name'\"'\"''" % u) + def testPunctuationCharsReadOnly(self): + punctuation_chars = "/|$%^" + shlex_instance = shlex.shlex(punctuation_chars=punctuation_chars) + self.assertEqual(shlex_instance.punctuation_chars, punctuation_chars) + with self.assertRaises(AttributeError): + shlex_instance.punctuation_chars = False + + # Allow this test to be used with old shlex.py if not getattr(shlex, "split", None): for methname in dir(ShlexTest): diff --git a/lib-python/3/test/test_site.py b/lib-python/3/test/test_site.py index 568f81da40..8815c83998 100644 --- a/lib-python/3/test/test_site.py +++ b/lib-python/3/test/test_site.py @@ -10,6 +10,7 @@ from test import support from test.support import (captured_stderr, TESTFN, EnvironmentVarGuard, change_cwd) import builtins +import glob import os import sys import re @@ -504,6 +505,23 @@ class ImportSideEffectTests(unittest.TestCase): class StartupImportTests(unittest.TestCase): def test_startup_imports(self): + # Get sys.path in isolated mode (python3 -I) + popen = subprocess.Popen([sys.executable, '-I', '-c', + 'import sys; print(repr(sys.path))'], + stdout=subprocess.PIPE, + encoding='utf-8') + stdout = popen.communicate()[0] + self.assertEqual(popen.returncode, 0, repr(stdout)) + isolated_paths = eval(stdout) + + # bpo-27807: Even with -I, the site module executes all .pth files + # found in sys.path (see site.addpackage()). Skip the test if at least + # one .pth file is found. + for path in isolated_paths: + pth_files = glob.glob(os.path.join(path, "*.pth")) + if pth_files: + self.skipTest(f"found {len(pth_files)} .pth files in: {path}") + # This tests checks which modules are loaded by Python when it # initially starts upon startup. popen = subprocess.Popen([sys.executable, '-I', '-v', '-c', @@ -512,6 +530,7 @@ class StartupImportTests(unittest.TestCase): stderr=subprocess.PIPE, encoding='utf-8') stdout, stderr = popen.communicate() + self.assertEqual(popen.returncode, 0, (stdout, stderr)) modules = eval(stdout) self.assertIn('site', modules) @@ -554,12 +573,19 @@ class StartupImportTests(unittest.TestCase): @unittest.skipUnless(sys.platform == 'win32', "only supported on Windows") class _pthFileTests(unittest.TestCase): - def _create_underpth_exe(self, lines): + def _create_underpth_exe(self, lines, exe_pth=True): + import _winapi temp_dir = tempfile.mkdtemp() self.addCleanup(test.support.rmtree, temp_dir) exe_file = os.path.join(temp_dir, os.path.split(sys.executable)[1]) + dll_src_file = _winapi.GetModuleFileName(sys.dllhandle) + dll_file = os.path.join(temp_dir, os.path.split(dll_src_file)[1]) shutil.copy(sys.executable, exe_file) - _pth_file = os.path.splitext(exe_file)[0] + '._pth' + shutil.copy(dll_src_file, dll_file) + if exe_pth: + _pth_file = os.path.splitext(exe_file)[0] + '._pth' + else: + _pth_file = os.path.splitext(dll_file)[0] + '._pth' with open(_pth_file, 'w') as f: for line in lines: print(line, file=f) @@ -627,5 +653,30 @@ class _pthFileTests(unittest.TestCase): self.assertTrue(rc, "sys.path is incorrect") + def test_underpth_dll_file(self): + libpath = os.path.dirname(os.path.dirname(encodings.__file__)) + exe_prefix = os.path.dirname(sys.executable) + exe_file = self._create_underpth_exe([ + 'fake-path-name', + *[libpath for _ in range(200)], + '', + '# comment', + 'import site' + ], exe_pth=False) + sys_prefix = os.path.dirname(exe_file) + env = os.environ.copy() + env['PYTHONPATH'] = 'from-env' + env['PATH'] = '{};{}'.format(exe_prefix, os.getenv('PATH')) + rc = subprocess.call([exe_file, '-c', + 'import sys; sys.exit(not sys.flags.no_site and ' + '%r in sys.path and %r in sys.path and %r not in sys.path and ' + 'all("\\r" not in p and "\\n" not in p for p in sys.path))' % ( + os.path.join(sys_prefix, 'fake-path-name'), + libpath, + os.path.join(sys_prefix, 'from-env'), + )], env=env) + self.assertTrue(rc, "sys.path is incorrect") + + if __name__ == "__main__": unittest.main() diff --git a/lib-python/3/test/test_socket.py b/lib-python/3/test/test_socket.py index 43929b355e..41ba53b481 100644 --- a/lib-python/3/test/test_socket.py +++ b/lib-python/3/test/test_socket.py @@ -47,6 +47,8 @@ except ImportError: def get_cid(): if fcntl is None: return None + if not hasattr(socket, 'IOCTL_VM_SOCKETS_GET_LOCAL_CID'): + return None try: with open("/dev/vsock", "rb") as f: r = fcntl.ioctl(f, socket.IOCTL_VM_SOCKETS_GET_LOCAL_CID, " ") diff --git a/lib-python/3/test/test_source_encoding.py b/lib-python/3/test/test_source_encoding.py index 38734009c0..a0bd741c36 100644 --- a/lib-python/3/test/test_source_encoding.py +++ b/lib-python/3/test/test_source_encoding.py @@ -31,7 +31,7 @@ class MiscSourceEncodingTest(unittest.TestCase): try: compile(b"# coding: cp932\nprint '\x94\x4e'", "dummy", "exec") except SyntaxError as v: - self.assertEqual(v.text, "print '\u5e74'\n") + self.assertEqual(v.text.rstrip('\n'), "print '\u5e74'") else: self.fail() diff --git a/lib-python/3/test/test_ssl.py b/lib-python/3/test/test_ssl.py index 4a61711f0e..1018259603 100644 --- a/lib-python/3/test/test_ssl.py +++ b/lib-python/3/test/test_ssl.py @@ -19,6 +19,7 @@ import weakref import platform import functools import sysconfig +import functools try: import ctypes except ImportError: @@ -142,6 +143,87 @@ OP_CIPHER_SERVER_PREFERENCE = getattr(ssl, "OP_CIPHER_SERVER_PREFERENCE", 0) OP_ENABLE_MIDDLEBOX_COMPAT = getattr(ssl, "OP_ENABLE_MIDDLEBOX_COMPAT", 0) +def has_tls_protocol(protocol): + """Check if a TLS protocol is available and enabled + + :param protocol: enum ssl._SSLMethod member or name + :return: bool + """ + if isinstance(protocol, str): + assert protocol.startswith('PROTOCOL_') + protocol = getattr(ssl, protocol, None) + if protocol is None: + return False + if protocol in { + ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS_SERVER, + ssl.PROTOCOL_TLS_CLIENT + }: + # auto-negotiate protocols are always available + return True + name = protocol.name + return has_tls_version(name[len('PROTOCOL_'):]) + + +@functools.lru_cache() +def has_tls_version(version): + """Check if a TLS/SSL version is enabled + + :param version: TLS version name or ssl.TLSVersion member + :return: bool + """ + if version == "SSLv2": + # never supported and not even in TLSVersion enum + return False + + if isinstance(version, str): + version = ssl.TLSVersion.__members__[version] + + # check compile time flags like ssl.HAS_TLSv1_2 + if not getattr(ssl, f'HAS_{version.name}'): + return False + + # check runtime and dynamic crypto policy settings. A TLS version may + # be compiled in but disabled by a policy or config option. + ctx = ssl.SSLContext() + if ( + hasattr(ctx, 'minimum_version') and + ctx.minimum_version != ssl.TLSVersion.MINIMUM_SUPPORTED and + version < ctx.minimum_version + ): + return False + if ( + hasattr(ctx, 'maximum_version') and + ctx.maximum_version != ssl.TLSVersion.MAXIMUM_SUPPORTED and + version > ctx.maximum_version + ): + return False + + return True + + +def requires_tls_version(version): + """Decorator to skip tests when a required TLS version is not available + + :param version: TLS version name or ssl.TLSVersion member + :return: + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kw): + if not has_tls_version(version): + raise unittest.SkipTest(f"{version} is not available.") + else: + return func(*args, **kw) + return wrapper + return decorator + + +requires_minimum_version = unittest.skipUnless( + hasattr(ssl.SSLContext, 'minimum_version'), + "required OpenSSL >= 1.1.0g" +) + + def handle_error(prefix): exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) if support.verbose: @@ -418,7 +500,7 @@ class BasicSocketTests(unittest.TestCase): ('email', 'null@python.org\x00user@example.org'), ('URI', 'http://null.python.org\x00http://example.org'), ('IP Address', '192.0.2.1'), - ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + ('IP Address', '2001:DB8:0:0:0:0:0:1')) else: # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName san = (('DNS', 'altnull.python.org\x00example.com'), @@ -445,7 +527,7 @@ class BasicSocketTests(unittest.TestCase): (('commonName', 'dirname example'),))), ('URI', 'https://www.python.org/'), ('IP Address', '127.0.0.1'), - ('IP Address', '0:0:0:0:0:0:0:1\n'), + ('IP Address', '0:0:0:0:0:0:0:1'), ('Registered ID', '1.2.3.4.5') ) ) @@ -472,11 +554,11 @@ class BasicSocketTests(unittest.TestCase): # Some sanity checks follow # >= 0.9 self.assertGreaterEqual(n, 0x900000) - # < 3.0 - self.assertLess(n, 0x30000000) + # < 4.0 + self.assertLess(n, 0x40000000) major, minor, fix, patch, status = t - self.assertGreaterEqual(major, 0) - self.assertLess(major, 3) + self.assertGreaterEqual(major, 1) + self.assertLess(major, 4) self.assertGreaterEqual(minor, 0) self.assertLess(minor, 256) self.assertGreaterEqual(fix, 0) @@ -849,8 +931,8 @@ class BasicSocketTests(unittest.TestCase): cert, enc, trust = element self.assertIsInstance(cert, bytes) self.assertIn(enc, {"x509_asn", "pkcs_7_asn"}) - self.assertIsInstance(trust, (set, bool)) - if isinstance(trust, set): + self.assertIsInstance(trust, (frozenset, set, bool)) + if isinstance(trust, (frozenset, set)): trust_oids.update(trust) serverAuth = "1.3.6.1.5.5.7.3.1" @@ -1124,22 +1206,32 @@ class ContextTests(unittest.TestCase): with self.assertRaises(AttributeError): ctx.hostname_checks_common_name = True - @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'), - "required OpenSSL 1.1.0g") + @requires_minimum_version + @unittest.skipIf(IS_LIBRESSL, "see bpo-34001") def test_min_max_version(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) # OpenSSL default is MINIMUM_SUPPORTED, however some vendors like # Fedora override the setting to TLS 1.0. + minimum_range = { + # stock OpenSSL + ssl.TLSVersion.MINIMUM_SUPPORTED, + # Fedora 29 uses TLS 1.0 by default + ssl.TLSVersion.TLSv1, + # RHEL 8 uses TLS 1.2 by default + ssl.TLSVersion.TLSv1_2 + } + maximum_range = { + # stock OpenSSL + ssl.TLSVersion.MAXIMUM_SUPPORTED, + # Fedora 32 uses TLS 1.3 by default + ssl.TLSVersion.TLSv1_3 + } + self.assertIn( - ctx.minimum_version, - {ssl.TLSVersion.MINIMUM_SUPPORTED, - # Fedora 29 uses TLS 1.0 by default - ssl.TLSVersion.TLSv1, - # RHEL 8 uses TLS 1.2 by default - ssl.TLSVersion.TLSv1_2} + ctx.minimum_version, minimum_range ) - self.assertEqual( - ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED + self.assertIn( + ctx.maximum_version, maximum_range ) ctx.minimum_version = ssl.TLSVersion.TLSv1_1 @@ -1182,8 +1274,8 @@ class ContextTests(unittest.TestCase): ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) - self.assertEqual( - ctx.minimum_version, ssl.TLSVersion.MINIMUM_SUPPORTED + self.assertIn( + ctx.minimum_version, minimum_range ) self.assertEqual( ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED @@ -2723,6 +2815,8 @@ class ThreadedTests(unittest.TestCase): for protocol in PROTOCOLS: if protocol in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER}: continue + if not has_tls_protocol(protocol): + continue with self.subTest(protocol=ssl._PROTOCOL_NAMES[protocol]): context = ssl.SSLContext(protocol) context.load_cert_chain(CERTFILE) @@ -3014,7 +3108,7 @@ class ThreadedTests(unittest.TestCase): else: self.fail("Use of invalid cert should have failed!") - @unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3") + @requires_tls_version('TLSv1_3') def test_wrong_cert_tls13(self): client_context, server_context, hostname = testing_context() # load client cert that is not signed by trusted CA @@ -3109,9 +3203,7 @@ class ThreadedTests(unittest.TestCase): self.assertIn(msg, repr(e)) self.assertIn('certificate verify failed', repr(e)) - @skip_if_broken_ubuntu_ssl - @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), - "OpenSSL is compiled without SSLv2 support") + @requires_tls_version('SSLv2') def test_protocol_sslv2(self): """Connecting to an SSLv2 server with various client options""" if support.verbose: @@ -3120,7 +3212,7 @@ class ThreadedTests(unittest.TestCase): try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_OPTIONAL) try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv2, True, ssl.CERT_REQUIRED) try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLS, False) - if hasattr(ssl, 'PROTOCOL_SSLv3'): + if has_tls_version('SSLv3'): try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_SSLv2, ssl.PROTOCOL_TLSv1, False) # SSLv23 client with specific SSL options @@ -3138,7 +3230,7 @@ class ThreadedTests(unittest.TestCase): """Connecting to an SSLv23 server with various client options""" if support.verbose: sys.stdout.write("\n") - if hasattr(ssl, 'PROTOCOL_SSLv2'): + if has_tls_version('SSLv2'): try: try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv2, True) except OSError as x: @@ -3147,35 +3239,36 @@ class ThreadedTests(unittest.TestCase): sys.stdout.write( " SSL2 client to SSL23 server test unexpectedly failed:\n %s\n" % str(x)) - if hasattr(ssl, 'PROTOCOL_SSLv3'): + if has_tls_version('SSLv3'): try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True) - try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1') + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1') - if hasattr(ssl, 'PROTOCOL_SSLv3'): + if has_tls_version('SSLv3'): try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_OPTIONAL) - try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) - if hasattr(ssl, 'PROTOCOL_SSLv3'): + if has_tls_version('SSLv3'): try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_REQUIRED) - try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) # Server with specific SSL options - if hasattr(ssl, 'PROTOCOL_SSLv3'): + if has_tls_version('SSLv3'): try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, server_options=ssl.OP_NO_SSLv3) # Will choose TLSv1 try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) - try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False, - server_options=ssl.OP_NO_TLSv1) - + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False, + server_options=ssl.OP_NO_TLSv1) - @skip_if_broken_ubuntu_ssl - @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv3'), - "OpenSSL is compiled without SSLv3 support") + @requires_tls_version('SSLv3') def test_protocol_sslv3(self): """Connecting to an SSLv3 server with various client options""" if support.verbose: @@ -3183,7 +3276,7 @@ class ThreadedTests(unittest.TestCase): try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) - if hasattr(ssl, 'PROTOCOL_SSLv2'): + if has_tls_version('SSLv2'): try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv2, False) try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS, False, client_options=ssl.OP_NO_SSLv3) @@ -3193,7 +3286,7 @@ class ThreadedTests(unittest.TestCase): try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS, False, client_options=ssl.OP_NO_SSLv2) - @skip_if_broken_ubuntu_ssl + @requires_tls_version('TLSv1') def test_protocol_tlsv1(self): """Connecting to a TLSv1 server with various client options""" if support.verbose: @@ -3201,36 +3294,32 @@ class ThreadedTests(unittest.TestCase): try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) - if hasattr(ssl, 'PROTOCOL_SSLv2'): + if has_tls_version('SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv2, False) - if hasattr(ssl, 'PROTOCOL_SSLv3'): + if has_tls_version('SSLv3'): try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLS, False, client_options=ssl.OP_NO_TLSv1) - @skip_if_broken_ubuntu_ssl - @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_1"), - "TLS version 1.1 not supported.") + @requires_tls_version('TLSv1_1') def test_protocol_tlsv1_1(self): """Connecting to a TLSv1.1 server with various client options. Testing against older TLS versions.""" if support.verbose: sys.stdout.write("\n") try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') - if hasattr(ssl, 'PROTOCOL_SSLv2'): + if has_tls_version('SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv2, False) - if hasattr(ssl, 'PROTOCOL_SSLv3'): + if has_tls_version('SSLv3'): try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLS, False, client_options=ssl.OP_NO_TLSv1_1) try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') - try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1, False) - try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) - @skip_if_broken_ubuntu_ssl - @unittest.skipUnless(hasattr(ssl, "PROTOCOL_TLSv1_2"), - "TLS version 1.2 not supported.") + @requires_tls_version('TLSv1_2') def test_protocol_tlsv1_2(self): """Connecting to a TLSv1.2 server with various client options. Testing against older TLS versions.""" @@ -3239,9 +3328,9 @@ class ThreadedTests(unittest.TestCase): try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) - if hasattr(ssl, 'PROTOCOL_SSLv2'): + if has_tls_version('SSLv2'): try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv2, False) - if hasattr(ssl, 'PROTOCOL_SSLv3'): + if has_tls_version('SSLv3'): try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLS, False, client_options=ssl.OP_NO_TLSv1_2) @@ -3684,7 +3773,7 @@ class ThreadedTests(unittest.TestCase): self.assertIs(s.version(), None) self.assertIs(s._sslobj, None) s.connect((HOST, server.port)) - if IS_OPENSSL_1_1_1 and ssl.HAS_TLSv1_3: + if IS_OPENSSL_1_1_1 and has_tls_version('TLSv1_3'): self.assertEqual(s.version(), 'TLSv1.3') elif ssl.OPENSSL_VERSION_INFO >= (1, 0, 2): self.assertEqual(s.version(), 'TLSv1.2') @@ -3693,8 +3782,7 @@ class ThreadedTests(unittest.TestCase): self.assertIs(s._sslobj, None) self.assertIs(s.version(), None) - @unittest.skipUnless(ssl.HAS_TLSv1_3, - "test requires TLSv1.3 enabled OpenSSL") + @requires_tls_version('TLSv1_3') def test_tls1_3(self): context = ssl.SSLContext(ssl.PROTOCOL_TLS) context.load_cert_chain(CERTFILE) @@ -3711,9 +3799,9 @@ class ThreadedTests(unittest.TestCase): }) self.assertEqual(s.version(), 'TLSv1.3') - @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'), - "required OpenSSL 1.1.0g") - def test_min_max_version(self): + @requires_minimum_version + @requires_tls_version('TLSv1_2') + def test_min_max_version_tlsv1_2(self): client_context, server_context, hostname = testing_context() # client TLSv1.0 to 1.2 client_context.minimum_version = ssl.TLSVersion.TLSv1 @@ -3728,7 +3816,13 @@ class ThreadedTests(unittest.TestCase): s.connect((HOST, server.port)) self.assertEqual(s.version(), 'TLSv1.2') + @requires_minimum_version + @requires_tls_version('TLSv1_1') + def test_min_max_version_tlsv1_1(self): + client_context, server_context, hostname = testing_context() # client 1.0 to 1.2, server 1.0 to 1.1 + client_context.minimum_version = ssl.TLSVersion.TLSv1 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 server_context.minimum_version = ssl.TLSVersion.TLSv1 server_context.maximum_version = ssl.TLSVersion.TLSv1_1 @@ -3738,6 +3832,10 @@ class ThreadedTests(unittest.TestCase): s.connect((HOST, server.port)) self.assertEqual(s.version(), 'TLSv1.1') + @requires_minimum_version + @requires_tls_version('TLSv1_2') + def test_min_max_version_mismatch(self): + client_context, server_context, hostname = testing_context() # client 1.0, server 1.2 (mismatch) server_context.minimum_version = ssl.TLSVersion.TLSv1_2 server_context.maximum_version = ssl.TLSVersion.TLSv1_2 @@ -3750,10 +3848,8 @@ class ThreadedTests(unittest.TestCase): s.connect((HOST, server.port)) self.assertIn("alert", str(e.exception)) - - @unittest.skipUnless(hasattr(ssl.SSLContext, 'minimum_version'), - "required OpenSSL 1.1.0g") - @unittest.skipUnless(ssl.HAS_SSLv3, "requires SSLv3 support") + @requires_minimum_version + @requires_tls_version('SSLv3') def test_min_max_version_sslv3(self): client_context, server_context, hostname = testing_context() server_context.minimum_version = ssl.TLSVersion.SSLv3 @@ -4272,7 +4368,7 @@ class ThreadedTests(unittest.TestCase): 'Session refers to a different SSLContext.') -@unittest.skipUnless(ssl.HAS_TLSv1_3, "Test needs TLS 1.3") +@unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") class TestPostHandshakeAuth(unittest.TestCase): def test_pha_setter(self): protocols = [ diff --git a/lib-python/3/test/test_stat.py b/lib-python/3/test/test_stat.py index 73cd901bdb..4ee0dc2ea3 100644 --- a/lib-python/3/test/test_stat.py +++ b/lib-python/3/test/test_stat.py @@ -14,10 +14,10 @@ class TestFilemode: 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE'} formats = {'S_IFBLK', 'S_IFCHR', 'S_IFDIR', 'S_IFIFO', 'S_IFLNK', - 'S_IFREG', 'S_IFSOCK'} + 'S_IFREG', 'S_IFSOCK', 'S_IFDOOR', 'S_IFPORT', 'S_IFWHT'} format_funcs = {'S_ISBLK', 'S_ISCHR', 'S_ISDIR', 'S_ISFIFO', 'S_ISLNK', - 'S_ISREG', 'S_ISSOCK'} + 'S_ISREG', 'S_ISSOCK', 'S_ISDOOR', 'S_ISPORT', 'S_ISWHT'} stat_struct = { 'ST_MODE': 0, @@ -221,10 +221,6 @@ class TestFilemode: class TestFilemodeCStat(TestFilemode, unittest.TestCase): statmod = c_stat - formats = TestFilemode.formats | {'S_IFDOOR', 'S_IFPORT', 'S_IFWHT'} - format_funcs = TestFilemode.format_funcs | {'S_ISDOOR', 'S_ISPORT', - 'S_ISWHT'} - class TestFilemodePyStat(TestFilemode, unittest.TestCase): statmod = py_stat diff --git a/lib-python/3/test/test_struct.py b/lib-python/3/test/test_struct.py index 8fd56c91cb..104f4d30c9 100644 --- a/lib-python/3/test/test_struct.py +++ b/lib-python/3/test/test_struct.py @@ -626,6 +626,13 @@ class StructTest(unittest.TestCase): s2 = struct.Struct(s.format.encode()) self.assertEqual(s2.format, s.format) + def test_issue35714(self): + # Embedded null characters should not be allowed in format strings. + for s in '\0', '2\0i', b'\0': + with self.assertRaisesRegex(struct.error, + 'embedded null character'): + struct.calcsize(s) + class UnpackIteratorTest(unittest.TestCase): """ diff --git a/lib-python/3/test/test_subprocess.py b/lib-python/3/test/test_subprocess.py index 8419061b2a..89dcb8f602 100644 --- a/lib-python/3/test/test_subprocess.py +++ b/lib-python/3/test/test_subprocess.py @@ -11,6 +11,7 @@ import os import errno import tempfile import time +import traceback import selectors import sysconfig import select @@ -59,10 +60,14 @@ class BaseTestCase(unittest.TestCase): support.reap_children() def tearDown(self): - for inst in subprocess._active: - inst.wait() - subprocess._cleanup() - self.assertFalse(subprocess._active, "subprocess._active not empty") + if not mswindows: + # subprocess._active is not used on Windows and is set to None. + for inst in subprocess._active: + inst.wait() + subprocess._cleanup() + self.assertFalse( + subprocess._active, "subprocess._active not empty" + ) self.doCleanups() support.reap_children() @@ -646,7 +651,6 @@ class ProcessTestCase(BaseTestCase): # on adding even when the environment in exec is empty. # Gentoo sandboxes also force LD_PRELOAD and SANDBOX_* to exist. return ('VERSIONER' in n or '__CF' in n or # MacOS - '__PYVENV_LAUNCHER__' in n or # MacOS framework build n == 'LD_PRELOAD' or n.startswith('SANDBOX') or # Gentoo n == 'LC_CTYPE') # Locale coercion triggered @@ -1504,6 +1508,26 @@ class RunFuncTestCase(BaseTestCase): self.assertIn('stderr', c.exception.args[0]) self.assertIn('capture_output', c.exception.args[0]) + # This test _might_ wind up a bit fragile on loaded build+test machines + # as it depends on the timing with wide enough margins for normal situations + # but does assert that it happened "soon enough" to believe the right thing + # happened. + @unittest.skipIf(mswindows, "requires posix like 'sleep' shell command") + def test_run_with_shell_timeout_and_capture_output(self): + """Output capturing after a timeout mustn't hang forever on open filehandles.""" + before_secs = time.monotonic() + try: + subprocess.run('sleep 3', shell=True, timeout=0.1, + capture_output=True) # New session unspecified. + except subprocess.TimeoutExpired as exc: + after_secs = time.monotonic() + stacks = traceback.format_exc() # assertRaises doesn't give this. + else: + self.fail("TimeoutExpired not raised.") + self.assertLess(after_secs - before_secs, 1.5, + msg="TimeoutExpired was delayed! Bad traceback:\n```\n" + f"{stacks}```") + @unittest.skipIf(mswindows, "POSIX specific tests") class POSIXProcessTestCase(BaseTestCase): @@ -2622,8 +2646,12 @@ class POSIXProcessTestCase(BaseTestCase): with support.check_warnings(('', ResourceWarning)): p = None - # check that p is in the active processes list - self.assertIn(ident, [id(o) for o in subprocess._active]) + if mswindows: + # subprocess._active is not used on Windows and is set to None. + self.assertIsNone(subprocess._active) + else: + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) def test_leak_fast_process_del_killed(self): # Issue #12650: on Unix, if Popen.__del__() was called before the @@ -2644,8 +2672,12 @@ class POSIXProcessTestCase(BaseTestCase): p = None os.kill(pid, signal.SIGKILL) - # check that p is in the active processes list - self.assertIn(ident, [id(o) for o in subprocess._active]) + if mswindows: + # subprocess._active is not used on Windows and is set to None. + self.assertIsNone(subprocess._active) + else: + # check that p is in the active processes list + self.assertIn(ident, [id(o) for o in subprocess._active]) # let some time for the process to exit, and create a new Popen: this # should trigger the wait() of p @@ -2657,7 +2689,11 @@ class POSIXProcessTestCase(BaseTestCase): pass # p should have been wait()ed on, and removed from the _active list self.assertRaises(OSError, os.waitpid, pid, 0) - self.assertNotIn(ident, [id(o) for o in subprocess._active]) + if mswindows: + # subprocess._active is not used on Windows and is set to None. + self.assertIsNone(subprocess._active) + else: + self.assertNotIn(ident, [id(o) for o in subprocess._active]) def test_close_fds_after_preexec(self): fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -2811,6 +2847,17 @@ class POSIXProcessTestCase(BaseTestCase): self.assertEqual(returncode, -3) + def test_communicate_repeated_call_after_stdout_close(self): + proc = subprocess.Popen([sys.executable, '-c', + 'import os, time; os.close(1), time.sleep(2)'], + stdout=subprocess.PIPE) + while True: + try: + proc.communicate(timeout=0.1) + return + except subprocess.TimeoutExpired: + pass + @unittest.skipUnless(mswindows, "Windows specific tests") class Win32ProcessTestCase(BaseTestCase): diff --git a/lib-python/3/test/test_support.py b/lib-python/3/test/test_support.py index e29846efe5..7c1ddb87f1 100644 --- a/lib-python/3/test/test_support.py +++ b/lib-python/3/test/test_support.py @@ -529,6 +529,7 @@ class TestSupport(unittest.TestCase): test_access = Test('test.test_os.FileTests.test_access') test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + # Test acceptance with support.swap_attr(support, '_match_test_func', None): # match all support.set_match_tests([]) @@ -536,45 +537,92 @@ class TestSupport(unittest.TestCase): self.assertTrue(support.match_test(test_chdir)) # match all using None - support.set_match_tests(None) + support.set_match_tests(None, None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # match the full test identifier - support.set_match_tests([test_access.id()]) + support.set_match_tests([test_access.id()], None) self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) # match the module name - support.set_match_tests(['test_os']) + support.set_match_tests(['test_os'], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Test '*' pattern - support.set_match_tests(['test_*']) + support.set_match_tests(['test_*'], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Test case sensitivity - support.set_match_tests(['filetests']) + support.set_match_tests(['filetests'], None) self.assertFalse(support.match_test(test_access)) - support.set_match_tests(['FileTests']) + support.set_match_tests(['FileTests'], None) self.assertTrue(support.match_test(test_access)) # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(['*test_os.*.test_*']) + support.set_match_tests(['*test_os.*.test_*'], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Multiple patterns - support.set_match_tests([test_access.id(), test_chdir.id()]) + support.set_match_tests([test_access.id(), test_chdir.id()], None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) - support.set_match_tests(['test_access', 'DONTMATCH']) + support.set_match_tests(['test_access', 'DONTMATCH'], None) self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) + # Test rejection + with support.swap_attr(support, '_match_test_func', None): + # match all + support.set_match_tests(ignore_patterns=[]) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match all using None + support.set_match_tests(None, None) + self.assertTrue(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match the full test identifier + support.set_match_tests(None, [test_access.id()]) + self.assertFalse(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # match the module name + support.set_match_tests(None, ['test_os']) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # Test '*' pattern + support.set_match_tests(None, ['test_*']) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # Test case sensitivity + support.set_match_tests(None, ['filetests']) + self.assertTrue(support.match_test(test_access)) + support.set_match_tests(None, ['FileTests']) + self.assertFalse(support.match_test(test_access)) + + # Test pattern containing '.' and a '*' metacharacter + support.set_match_tests(None, ['*test_os.*.test_*']) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + # Multiple patterns + support.set_match_tests(None, [test_access.id(), test_chdir.id()]) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + + support.set_match_tests(None, ['test_access', 'DONTMATCH']) + self.assertFalse(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + def test_fd_count(self): # We cannot test the absolute value of fd_count(): on old Linux # kernel or glibc versions, os.urandom() keeps a FD open on diff --git a/lib-python/3/test/test_symtable.py b/lib-python/3/test/test_symtable.py index dfaee173ef..aa99e86e4b 100644 --- a/lib-python/3/test/test_symtable.py +++ b/lib-python/3/test/test_symtable.py @@ -92,10 +92,14 @@ class SymtableTest(unittest.TestCase): self.assertTrue(self.spam.lookup("bar").is_declared_global()) self.assertFalse(self.internal.lookup("x").is_global()) self.assertFalse(self.Mine.lookup("instance_var").is_global()) + self.assertTrue(self.spam.lookup("bar").is_global()) def test_local(self): self.assertTrue(self.spam.lookup("x").is_local()) - self.assertFalse(self.internal.lookup("x").is_local()) + self.assertFalse(self.spam.lookup("bar").is_local()) + + def test_free(self): + self.assertTrue(self.internal.lookup("x").is_free()) def test_referenced(self): self.assertTrue(self.internal.lookup("x").is_referenced()) diff --git a/lib-python/3/test/test_sys.py b/lib-python/3/test/test_sys.py index ef3fee13b9..84927a393f 100644 --- a/lib-python/3/test/test_sys.py +++ b/lib-python/3/test/test_sys.py @@ -1,82 +1,103 @@ -import unittest, test.support +from test import support from test.support.script_helper import assert_python_ok, assert_python_failure -import sys, io, os +import builtins +import codecs +import gc +import locale +import operator +import os import struct import subprocess +import sys +import sysconfig +import test.support import textwrap +import unittest import warnings -import operator -import codecs -import gc -import sysconfig -import locale -import threading + # count the number of test runs, used to create unique # strings to intern in test_intern() -numruns = 0 +INTERN_NUMRUNS = 0 -class SysModuleTest(unittest.TestCase): +class DisplayHookTest(unittest.TestCase): - def setUp(self): - self.orig_stdout = sys.stdout - self.orig_stderr = sys.stderr - self.orig_displayhook = sys.displayhook + def test_original_displayhook(self): + dh = sys.__displayhook__ - def tearDown(self): - sys.stdout = self.orig_stdout - sys.stderr = self.orig_stderr - sys.displayhook = self.orig_displayhook - test.support.reap_children() + with support.captured_stdout() as out: + dh(42) - def test_original_displayhook(self): - import builtins - out = io.StringIO() - sys.stdout = out + self.assertEqual(out.getvalue(), "42\n") + self.assertEqual(builtins._, 42) - dh = sys.__displayhook__ + del builtins._ - self.assertRaises(TypeError, dh) - if hasattr(builtins, "_"): - del builtins._ + with support.captured_stdout() as out: + dh(None) - dh(None) self.assertEqual(out.getvalue(), "") self.assertTrue(not hasattr(builtins, "_")) - dh(42) - self.assertEqual(out.getvalue(), "42\n") - self.assertEqual(builtins._, 42) - del sys.stdout - self.assertRaises(RuntimeError, dh, 42) + # sys.displayhook() requires arguments + self.assertRaises(TypeError, dh) + + stdout = sys.stdout + try: + del sys.stdout + self.assertRaises(RuntimeError, dh, 42) + finally: + sys.stdout = stdout def test_lost_displayhook(self): - del sys.displayhook - code = compile("42", "<string>", "single") - self.assertRaises(RuntimeError, eval, code) + displayhook = sys.displayhook + try: + del sys.displayhook + code = compile("42", "<string>", "single") + self.assertRaises(RuntimeError, eval, code) + finally: + sys.displayhook = displayhook def test_custom_displayhook(self): def baddisplayhook(obj): raise ValueError - sys.displayhook = baddisplayhook - code = compile("42", "<string>", "single") - self.assertRaises(ValueError, eval, code) - def test_original_excepthook(self): - err = io.StringIO() - sys.stderr = err + with support.swap_attr(sys, 'displayhook', baddisplayhook): + code = compile("42", "<string>", "single") + self.assertRaises(ValueError, eval, code) + - eh = sys.__excepthook__ +class ExceptHookTest(unittest.TestCase): - self.assertRaises(TypeError, eh) + def test_original_excepthook(self): try: raise ValueError(42) except ValueError as exc: - eh(*sys.exc_info()) + with support.captured_stderr() as err: + sys.__excepthook__(*sys.exc_info()) self.assertTrue(err.getvalue().endswith("ValueError: 42\n")) + self.assertRaises(TypeError, sys.__excepthook__) + + def test_excepthook_bytes_filename(self): + # bpo-37467: sys.excepthook() must not crash if a filename + # is a bytes string + with warnings.catch_warnings(): + warnings.simplefilter('ignore', BytesWarning) + + try: + raise SyntaxError("msg", (b"bytes_filename", 123, 0, "text")) + except SyntaxError as exc: + with support.captured_stderr() as err: + sys.__excepthook__(*sys.exc_info()) + + err = err.getvalue() + self.assertIn(""" File "b'bytes_filename'", line 123\n""", err) + self.assertIn(""" text\n""", err) + self.assertTrue(err.endswith("SyntaxError: msg\n")) + def test_excepthook(self): with test.support.captured_output("stderr") as stderr: sys.excepthook(1, '1', 1) @@ -86,6 +107,12 @@ class SysModuleTest(unittest.TestCase): # FIXME: testing the code for a lost or replaced excepthook in # Python/pythonrun.c::PyErr_PrintEx() is tricky. + +class SysModuleTest(unittest.TestCase): + + def tearDown(self): + test.support.reap_children() + def test_exit(self): # call with two arguments self.assertRaises(TypeError, sys.exit, 42, 42) @@ -502,10 +529,10 @@ class SysModuleTest(unittest.TestCase): self.assertEqual(sys.__stdout__.encoding, sys.__stderr__.encoding) def test_intern(self): - global numruns - numruns += 1 + global INTERN_NUMRUNS + INTERN_NUMRUNS += 1 self.assertRaises(TypeError, sys.intern) - s = "never interned before" + str(numruns) + s = "never interned before" + str(INTERN_NUMRUNS) self.assertTrue(sys.intern(s) is s) s2 = s.swapcase().swapcase() self.assertTrue(sys.intern(s2) is s) diff --git a/lib-python/3/test/test_tarfile.py b/lib-python/3/test/test_tarfile.py index 5e4d75ecfc..9133d60e49 100644 --- a/lib-python/3/test/test_tarfile.py +++ b/lib-python/3/test/test_tarfile.py @@ -395,6 +395,13 @@ class CommonReadTest(ReadTest): with self.assertRaisesRegex(tarfile.ReadError, "unexpected end of data"): tar.extractfile(t).read() + def test_length_zero_header(self): + # bpo-39017 (CVE-2019-20907): reading a zero-length header should fail + # with an exception + with self.assertRaisesRegex(tarfile.ReadError, "file could not be opened successfully"): + with tarfile.open(support.findfile('recursion.tar')) as tar: + pass + class MiscReadTestBase(CommonReadTest): def requires_name_attribute(self): pass diff --git a/lib-python/3/test/test_tcl.py b/lib-python/3/test/test_tcl.py index 80f1668bce..3183ea850f 100644 --- a/lib-python/3/test/test_tcl.py +++ b/lib-python/3/test/test_tcl.py @@ -429,9 +429,12 @@ class TclTest(unittest.TestCase): self.assertEqual(passValue(False), False if self.wantobjects else '0') self.assertEqual(passValue('string'), 'string') self.assertEqual(passValue('string\u20ac'), 'string\u20ac') + self.assertEqual(passValue('string\U0001f4bb'), 'string\U0001f4bb') self.assertEqual(passValue('str\x00ing'), 'str\x00ing') self.assertEqual(passValue('str\x00ing\xbd'), 'str\x00ing\xbd') self.assertEqual(passValue('str\x00ing\u20ac'), 'str\x00ing\u20ac') + self.assertEqual(passValue('str\x00ing\U0001f4bb'), + 'str\x00ing\U0001f4bb') self.assertEqual(passValue(b'str\x00ing'), b'str\x00ing' if self.wantobjects else 'str\x00ing') self.assertEqual(passValue(b'str\xc0\x80ing'), @@ -490,6 +493,7 @@ class TclTest(unittest.TestCase): check('string') check('string\xbd') check('string\u20ac') + check('string\U0001f4bb') check('') check(b'string', 'string') check(b'string\xe2\x82\xac', 'string\xe2\x82\xac') @@ -531,6 +535,7 @@ class TclTest(unittest.TestCase): ('a\n b\t\r c\n ', ('a', 'b', 'c')), (b'a\n b\t\r c\n ', ('a', 'b', 'c')), ('a \u20ac', ('a', '\u20ac')), + ('a \U0001f4bb', ('a', '\U0001f4bb')), (b'a \xe2\x82\xac', ('a', '\u20ac')), (b'a\xc0\x80b c\xc0\x80d', ('a\x00b', 'c\x00d')), ('a {b c}', ('a', 'b c')), diff --git a/lib-python/3/test/test_tempfile.py b/lib-python/3/test/test_tempfile.py index 710756bde6..2d47e70bd0 100644 --- a/lib-python/3/test/test_tempfile.py +++ b/lib-python/3/test/test_tempfile.py @@ -3,6 +3,7 @@ import tempfile import errno import io import os +import pathlib import signal import sys import re @@ -61,6 +62,9 @@ class TestLowLevelInternals(unittest.TestCase): with self.assertRaises(TypeError): tempfile._infer_return_type(b'', None, '') + def test_infer_return_type_pathlib(self): + self.assertIs(str, tempfile._infer_return_type(pathlib.Path('/'))) + # Common functionality. @@ -84,8 +88,13 @@ class BaseTestCase(unittest.TestCase): nsuf = nbase[len(nbase)-len(suf):] if dir is not None: - self.assertIs(type(name), str if type(dir) is str else bytes, - "unexpected return type") + self.assertIs( + type(name), + str + if type(dir) is str or isinstance(dir, os.PathLike) else + bytes, + "unexpected return type", + ) if pre is not None: self.assertIs(type(name), str if type(pre) is str else bytes, "unexpected return type") @@ -430,6 +439,7 @@ class TestMkstempInner(TestBadTempdir, BaseTestCase): dir = tempfile.mkdtemp() try: self.do_create(dir=dir).write(b"blat") + self.do_create(dir=pathlib.Path(dir)).write(b"blat") finally: os.rmdir(dir) @@ -666,6 +676,7 @@ class TestMkstemp(BaseTestCase): dir = tempfile.mkdtemp() try: self.do_create(dir=dir) + self.do_create(dir=pathlib.Path(dir)) finally: os.rmdir(dir) @@ -735,6 +746,7 @@ class TestMkdtemp(TestBadTempdir, BaseTestCase): dir = tempfile.mkdtemp() try: os.rmdir(self.do_create(dir=dir)) + os.rmdir(self.do_create(dir=pathlib.Path(dir))) finally: os.rmdir(dir) @@ -1022,7 +1034,8 @@ class TestSpooledTemporaryFile(BaseTestCase): # Verify writelines with a SpooledTemporaryFile f = self.do_create() f.writelines((b'x', b'y', b'z')) - f.seek(0) + pos = f.seek(0) + self.assertEqual(pos, 0) buf = f.read() self.assertEqual(buf, b'xyz') @@ -1040,7 +1053,8 @@ class TestSpooledTemporaryFile(BaseTestCase): # when that occurs f = self.do_create(max_size=30) self.assertFalse(f._rolled) - f.seek(100, 0) + pos = f.seek(100, 0) + self.assertEqual(pos, 100) self.assertFalse(f._rolled) f.write(b'x') self.assertTrue(f._rolled) @@ -1107,7 +1121,8 @@ class TestSpooledTemporaryFile(BaseTestCase): def test_text_mode(self): # Creating a SpooledTemporaryFile with a text mode should produce # a file object reading and writing (Unicode) text strings. - f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10) + f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10, + encoding="utf-8") f.write("abc\n") f.seek(0) self.assertEqual(f.read(), "abc\n") @@ -1117,8 +1132,8 @@ class TestSpooledTemporaryFile(BaseTestCase): self.assertFalse(f._rolled) self.assertEqual(f.mode, 'w+') self.assertIsNone(f.name) - self.assertIsNone(f.newlines) - self.assertIsNone(f.encoding) + self.assertEqual(f.newlines, os.linesep) + self.assertEqual(f.encoding, "utf-8") f.write("xyzzy\n") f.seek(0) @@ -1131,7 +1146,7 @@ class TestSpooledTemporaryFile(BaseTestCase): self.assertEqual(f.mode, 'w+') self.assertIsNotNone(f.name) self.assertEqual(f.newlines, os.linesep) - self.assertIsNotNone(f.encoding) + self.assertEqual(f.encoding, "utf-8") def test_text_newline_and_encoding(self): f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10, @@ -1142,12 +1157,14 @@ class TestSpooledTemporaryFile(BaseTestCase): self.assertFalse(f._rolled) self.assertEqual(f.mode, 'w+') self.assertIsNone(f.name) - self.assertIsNone(f.newlines) - self.assertIsNone(f.encoding) + self.assertIsNotNone(f.newlines) + self.assertEqual(f.encoding, "utf-8") - f.write("\u039B" * 20 + "\r\n") + f.write("\u039C" * 10 + "\r\n") + f.write("\u039D" * 20) f.seek(0) - self.assertEqual(f.read(), "\u039B\r\n" + ("\u039B" * 20) + "\r\n") + self.assertEqual(f.read(), + "\u039B\r\n" + ("\u039C" * 10) + "\r\n" + ("\u039D" * 20)) self.assertTrue(f._rolled) self.assertEqual(f.mode, 'w+') self.assertIsNotNone(f.name) diff --git a/lib-python/3/test/test_timeout.py b/lib-python/3/test/test_timeout.py index b54fc826ae..b07c07cbfc 100644 --- a/lib-python/3/test/test_timeout.py +++ b/lib-python/3/test/test_timeout.py @@ -150,6 +150,7 @@ class TCPTimeoutTestCase(TimeoutTestCase): def tearDown(self): self.sock.close() + @unittest.skipIf(True, 'need to replace these hosts; see bpo-35518') def testConnectTimeout(self): # Testing connect timeout is tricky: we need to have IP connectivity # to a host that silently drops our packets. We can't simulate this diff --git a/lib-python/3/test/test_tools/test_unparse.py b/lib-python/3/test/test_tools/test_unparse.py index f3386f5e31..12a41c756c 100644 --- a/lib-python/3/test/test_tools/test_unparse.py +++ b/lib-python/3/test/test_tools/test_unparse.py @@ -260,6 +260,20 @@ class UnparseTestCase(ASTTestCase): self.check_roundtrip(r"""{**{'y': 2}, 'x': 1}""") self.check_roundtrip(r"""{**{'y': 2}, **{'x': 1}}""") + def test_subscript(self): + self.check_roundtrip("a[i]") + self.check_roundtrip("a[i,]") + self.check_roundtrip("a[i, j]") + self.check_roundtrip("a[()]") + self.check_roundtrip("a[i:j]") + self.check_roundtrip("a[:j]") + self.check_roundtrip("a[i:]") + self.check_roundtrip("a[i:j:k]") + self.check_roundtrip("a[:j:k]") + self.check_roundtrip("a[i::k]") + self.check_roundtrip("a[i:j,]") + self.check_roundtrip("a[i:j, k]") + class DirectoryTestCase(ASTTestCase): """Test roundtrip behaviour on all files in Lib and Lib/test.""" diff --git a/lib-python/3/test/test_typing.py b/lib-python/3/test/test_typing.py index b396283a02..c141baf1a9 100644 --- a/lib-python/3/test/test_typing.py +++ b/lib-python/3/test/test_typing.py @@ -218,6 +218,13 @@ class TypeVarTests(BaseTestCase): with self.assertRaises(TypeError): TypeVar('X', str, float, bound=Employee) + def test_missing__name__(self): + # See bpo-39942 + code = ("import typing\n" + "T = typing.TypeVar('T')\n" + ) + exec(code, {}) + def test_no_bivariant(self): with self.assertRaises(ValueError): TypeVar('T', covariant=True, contravariant=True) @@ -1505,6 +1512,65 @@ class ForwardRefTests(BaseTestCase): self.assertEqual(fr, typing.ForwardRef('int')) self.assertNotEqual(List['int'], List[int]) + def test_forward_equality_gth(self): + c1 = typing.ForwardRef('C') + c1_gth = typing.ForwardRef('C') + c2 = typing.ForwardRef('C') + c2_gth = typing.ForwardRef('C') + + class C: + pass + def foo(a: c1_gth, b: c2_gth): + pass + + self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': C, 'b': C}) + self.assertEqual(c1, c2) + self.assertEqual(c1, c1_gth) + self.assertEqual(c1_gth, c2_gth) + self.assertEqual(List[c1], List[c1_gth]) + self.assertNotEqual(List[c1], List[C]) + self.assertNotEqual(List[c1_gth], List[C]) + self.assertEquals(Union[c1, c1_gth], Union[c1]) + self.assertEquals(Union[c1, c1_gth, int], Union[c1, int]) + + def test_forward_equality_hash(self): + c1 = typing.ForwardRef('int') + c1_gth = typing.ForwardRef('int') + c2 = typing.ForwardRef('int') + c2_gth = typing.ForwardRef('int') + + def foo(a: c1_gth, b: c2_gth): + pass + get_type_hints(foo, globals(), locals()) + + self.assertEqual(hash(c1), hash(c2)) + self.assertEqual(hash(c1_gth), hash(c2_gth)) + self.assertEqual(hash(c1), hash(c1_gth)) + + def test_forward_equality_namespace(self): + class A: + pass + def namespace1(): + a = typing.ForwardRef('A') + def fun(x: a): + pass + get_type_hints(fun, globals(), locals()) + return a + + def namespace2(): + a = typing.ForwardRef('A') + + class A: + pass + def fun(x: a): + pass + + get_type_hints(fun, globals(), locals()) + return a + + self.assertEqual(namespace1(), namespace1()) + self.assertNotEqual(namespace1(), namespace2()) + def test_forward_repr(self): self.assertEqual(repr(List['int']), "typing.List[ForwardRef('int')]") @@ -1524,6 +1590,63 @@ class ForwardRefTests(BaseTestCase): self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': Tuple[T]}) + def test_forward_recursion_actually(self): + def namespace1(): + a = typing.ForwardRef('A') + A = a + def fun(x: a): pass + + ret = get_type_hints(fun, globals(), locals()) + return a + + def namespace2(): + a = typing.ForwardRef('A') + A = a + def fun(x: a): pass + + ret = get_type_hints(fun, globals(), locals()) + return a + + def cmp(o1, o2): + return o1 == o2 + + r1 = namespace1() + r2 = namespace2() + self.assertIsNot(r1, r2) + self.assertRaises(RecursionError, cmp, r1, r2) + + def test_union_forward_recursion(self): + ValueList = List['Value'] + Value = Union[str, ValueList] + + class C: + foo: List[Value] + class D: + foo: Union[Value, ValueList] + class E: + foo: Union[List[Value], ValueList] + class F: + foo: Union[Value, List[Value], ValueList] + + self.assertEqual(get_type_hints(C, globals(), locals()), get_type_hints(C, globals(), locals())) + self.assertEqual(get_type_hints(C, globals(), locals()), + {'foo': List[Union[str, List[Union[str, List['Value']]]]]}) + self.assertEqual(get_type_hints(D, globals(), locals()), + {'foo': Union[str, List[Union[str, List['Value']]]]}) + self.assertEqual(get_type_hints(E, globals(), locals()), + {'foo': Union[ + List[Union[str, List[Union[str, List['Value']]]]], + List[Union[str, List['Value']]] + ] + }) + self.assertEqual(get_type_hints(F, globals(), locals()), + {'foo': Union[ + str, + List[Union[str, List['Value']]], + List[Union[str, List[Union[str, List['Value']]]]] + ] + }) + def test_callable_forward(self): def foo(a: Callable[['T'], 'T']): @@ -1787,6 +1910,16 @@ except StopIteration as e: gth = get_type_hints +class ForRefExample: + @ann_module.dec + def func(self: 'ForRefExample'): + pass + + @ann_module.dec + @ann_module.dec + def nested(self: 'ForRefExample'): + pass + class GetTypeHintTests(BaseTestCase): def test_get_type_hints_from_various_objects(self): @@ -1885,6 +2018,11 @@ class GetTypeHintTests(BaseTestCase): 'x': ClassVar[Optional[B]]}) self.assertEqual(gth(G), {'lst': ClassVar[List[T]]}) + def test_get_type_hints_wrapped_decoratored_func(self): + expects = {'self': ForRefExample} + self.assertEqual(gth(ForRefExample.func), expects) + self.assertEqual(gth(ForRefExample.nested), expects) + class CollectionsAbcTests(BaseTestCase): @@ -2441,6 +2579,9 @@ class NewTypeTests(BaseTestCase): class NamedTupleTests(BaseTestCase): + class NestedEmployee(NamedTuple): + name: str + cool: int def test_basics(self): Emp = NamedTuple('Emp', [('name', str), ('id', int)]) @@ -2536,14 +2677,53 @@ class XMethBad2(NamedTuple): with self.assertRaises(TypeError): NamedTuple('Name', x=1, y='a') - def test_pickle(self): + def test_namedtuple_special_keyword_names(self): + NT = NamedTuple("NT", cls=type, self=object, typename=str, fields=list) + self.assertEqual(NT.__name__, 'NT') + self.assertEqual(NT._fields, ('cls', 'self', 'typename', 'fields')) + a = NT(cls=str, self=42, typename='foo', fields=[('bar', tuple)]) + self.assertEqual(a.cls, str) + self.assertEqual(a.self, 42) + self.assertEqual(a.typename, 'foo') + self.assertEqual(a.fields, [('bar', tuple)]) + + def test_namedtuple_errors(self): + with self.assertRaises(TypeError): + NamedTuple.__new__() + with self.assertRaises(TypeError): + NamedTuple() + with self.assertRaises(TypeError): + NamedTuple('Emp', [('name', str)], None) + with self.assertRaises(ValueError): + NamedTuple('Emp', [('_name', str)]) + + Emp = NamedTuple(typename='Emp', name=str, id=int) + self.assertEqual(Emp.__name__, 'Emp') + self.assertEqual(Emp._fields, ('name', 'id')) + + Emp = NamedTuple('Emp', fields=[('name', str), ('id', int)]) + self.assertEqual(Emp.__name__, 'Emp') + self.assertEqual(Emp._fields, ('name', 'id')) + + def test_copy_and_pickle(self): global Emp # pickle wants to reference the class by name - Emp = NamedTuple('Emp', [('name', str), ('id', int)]) - jane = Emp('jane', 37) - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - z = pickle.dumps(jane, proto) - jane2 = pickle.loads(z) - self.assertEqual(jane2, jane) + Emp = NamedTuple('Emp', [('name', str), ('cool', int)]) + for cls in Emp, CoolEmployee, self.NestedEmployee: + with self.subTest(cls=cls): + jane = cls('jane', 37) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + z = pickle.dumps(jane, proto) + jane2 = pickle.loads(z) + self.assertEqual(jane2, jane) + self.assertIsInstance(jane2, cls) + + jane2 = copy(jane) + self.assertEqual(jane2, jane) + self.assertIsInstance(jane2, cls) + + jane2 = deepcopy(jane) + self.assertEqual(jane2, jane) + self.assertIsInstance(jane2, cls) class IOTests(BaseTestCase): diff --git a/lib-python/3/test/test_unicode.py b/lib-python/3/test/test_unicode.py index 1aad933407..4ebd82d3e0 100644 --- a/lib-python/3/test/test_unicode.py +++ b/lib-python/3/test/test_unicode.py @@ -11,6 +11,7 @@ import itertools import operator import struct import sys +import unicodedata import unittest import warnings from test import support, string_tests @@ -615,11 +616,21 @@ class UnicodeTest(string_tests.CommonTest, self.checkequalnofix(True, '\u2000', 'isspace') self.checkequalnofix(True, '\u200a', 'isspace') self.checkequalnofix(False, '\u2014', 'isspace') - # apparently there are no non-BMP spaces chars in Unicode 6 + # There are no non-BMP whitespace chars as of Unicode 12. for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', '\U0001F40D', '\U0001F46F']: self.assertFalse(ch.isspace(), '{!a} is not space.'.format(ch)) + @support.requires_resource('cpu') + def test_isspace_invariant(self): + for codepoint in range(sys.maxunicode + 1): + char = chr(codepoint) + bidirectional = unicodedata.bidirectional(char) + category = unicodedata.category(char) + self.assertEqual(char.isspace(), + (bidirectional in ('WS', 'B', 'S') + or category == 'Zs')) + def test_isalnum(self): super().test_isalnum() for ch in ['\U00010401', '\U00010427', '\U00010429', '\U0001044E', diff --git a/lib-python/3/test/test_urllib.py b/lib-python/3/test/test_urllib.py index 7ec365b928..0305e7aa18 100644 --- a/lib-python/3/test/test_urllib.py +++ b/lib-python/3/test/test_urllib.py @@ -253,14 +253,36 @@ class ProxyTests(unittest.TestCase): self.assertTrue(bypass('localhost')) self.assertTrue(bypass('LocalHost')) # MixedCase self.assertTrue(bypass('LOCALHOST')) # UPPERCASE + self.assertTrue(bypass('.localhost')) self.assertTrue(bypass('newdomain.com:1234')) + self.assertTrue(bypass('.newdomain.com:1234')) self.assertTrue(bypass('foo.d.o.t')) # issue 29142 + self.assertTrue(bypass('d.o.t')) self.assertTrue(bypass('anotherdomain.com:8888')) + self.assertTrue(bypass('.anotherdomain.com:8888')) self.assertTrue(bypass('www.newdomain.com:1234')) self.assertFalse(bypass('prelocalhost')) self.assertFalse(bypass('newdomain.com')) # no port self.assertFalse(bypass('newdomain.com:1235')) # wrong port + def test_proxy_bypass_environment_always_match(self): + bypass = urllib.request.proxy_bypass_environment + self.env.set('NO_PROXY', '*') + self.assertTrue(bypass('newdomain.com')) + self.assertTrue(bypass('newdomain.com:1234')) + self.env.set('NO_PROXY', '*, anotherdomain.com') + self.assertTrue(bypass('anotherdomain.com')) + self.assertFalse(bypass('newdomain.com')) + self.assertFalse(bypass('newdomain.com:1234')) + + def test_proxy_bypass_environment_newline(self): + bypass = urllib.request.proxy_bypass_environment + self.env.set('NO_PROXY', + 'localhost, anotherdomain.com, newdomain.com:1234') + self.assertFalse(bypass('localhost\n')) + self.assertFalse(bypass('anotherdomain.com:8888\n')) + self.assertFalse(bypass('newdomain.com:1234\n')) + class ProxyTests_withOrderedEnv(unittest.TestCase): @@ -331,7 +353,7 @@ class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): self.unfakehttp() @unittest.skipUnless(ssl, "ssl module required") - def test_url_with_control_char_rejected(self): + def test_url_path_with_control_char_rejected(self): for char_no in list(range(0, 0x21)) + [0x7f]: char = chr(char_no) schemeless_url = f"//localhost:7777/test{char}/" @@ -358,7 +380,7 @@ class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): self.unfakehttp() @unittest.skipUnless(ssl, "ssl module required") - def test_url_with_newline_header_injection_rejected(self): + def test_url_path_with_newline_header_injection_rejected(self): self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.") host = "localhost:7777?a=1 HTTP/1.1\r\nX-injected: header\r\nTEST: 123" schemeless_url = "//" + host + ":8080/test/?test=a" @@ -383,6 +405,38 @@ class urlopen_HttpTests(unittest.TestCase, FakeHTTPMixin, FakeFTPMixin): finally: self.unfakehttp() + @unittest.skipUnless(ssl, "ssl module required") + def test_url_host_with_control_char_rejected(self): + for char_no in list(range(0, 0x21)) + [0x7f]: + char = chr(char_no) + schemeless_url = f"//localhost{char}/test/" + self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.") + try: + escaped_char_repr = repr(char).replace('\\', r'\\') + InvalidURL = http.client.InvalidURL + with self.assertRaisesRegex( + InvalidURL, f"contain control.*{escaped_char_repr}"): + urlopen(f"http:{schemeless_url}") + with self.assertRaisesRegex(InvalidURL, f"contain control.*{escaped_char_repr}"): + urlopen(f"https:{schemeless_url}") + finally: + self.unfakehttp() + + @unittest.skipUnless(ssl, "ssl module required") + def test_url_host_with_newline_header_injection_rejected(self): + self.fakehttp(b"HTTP/1.1 200 OK\r\n\r\nHello.") + host = "localhost\r\nX-injected: header\r\n" + schemeless_url = "//" + host + ":8080/test/?test=a" + try: + InvalidURL = http.client.InvalidURL + with self.assertRaisesRegex( + InvalidURL, r"contain control.*\\r"): + urlopen(f"http:{schemeless_url}") + with self.assertRaisesRegex(InvalidURL, r"contain control.*\\n"): + urlopen(f"https:{schemeless_url}") + finally: + self.unfakehttp() + def test_read_0_9(self): # "0.9" response accepted (but not "simple responses" without # a status line) diff --git a/lib-python/3/test/test_urllib2.py b/lib-python/3/test/test_urllib2.py index 876fcd4199..fe9a32bfda 100644 --- a/lib-python/3/test/test_urllib2.py +++ b/lib-python/3/test/test_urllib2.py @@ -1445,40 +1445,64 @@ class HandlerTests(unittest.TestCase): bypass = {'exclude_simple': True, 'exceptions': []} self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass)) - def test_basic_auth(self, quote_char='"'): - opener = OpenerDirector() - password_manager = MockPasswordManager() - auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) - realm = "ACME Widget Store" - http_handler = MockHTTPHandler( - 401, 'WWW-Authenticate: Basic realm=%s%s%s\r\n\r\n' % - (quote_char, realm, quote_char)) - opener.add_handler(auth_handler) - opener.add_handler(http_handler) - self._test_basic_auth(opener, auth_handler, "Authorization", - realm, http_handler, password_manager, - "http://acme.example.com/protected", - "http://acme.example.com/protected", - ) - - def test_basic_auth_with_single_quoted_realm(self): - self.test_basic_auth(quote_char="'") - - def test_basic_auth_with_unquoted_realm(self): - opener = OpenerDirector() - password_manager = MockPasswordManager() - auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) - realm = "ACME Widget Store" - http_handler = MockHTTPHandler( - 401, 'WWW-Authenticate: Basic realm=%s\r\n\r\n' % realm) - opener.add_handler(auth_handler) - opener.add_handler(http_handler) - with self.assertWarns(UserWarning): + def check_basic_auth(self, headers, realm): + with self.subTest(realm=realm, headers=headers): + opener = OpenerDirector() + password_manager = MockPasswordManager() + auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) + body = '\r\n'.join(headers) + '\r\n\r\n' + http_handler = MockHTTPHandler(401, body) + opener.add_handler(auth_handler) + opener.add_handler(http_handler) self._test_basic_auth(opener, auth_handler, "Authorization", - realm, http_handler, password_manager, - "http://acme.example.com/protected", - "http://acme.example.com/protected", - ) + realm, http_handler, password_manager, + "http://acme.example.com/protected", + "http://acme.example.com/protected") + + def test_basic_auth(self): + realm = "realm2@example.com" + realm2 = "realm2@example.com" + basic = f'Basic realm="{realm}"' + basic2 = f'Basic realm="{realm2}"' + other_no_realm = 'Otherscheme xxx' + digest = (f'Digest realm="{realm2}", ' + f'qop="auth, auth-int", ' + f'nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", ' + f'opaque="5ccc069c403ebaf9f0171e9517f40e41"') + for realm_str in ( + # test "quote" and 'quote' + f'Basic realm="{realm}"', + f"Basic realm='{realm}'", + + # charset is ignored + f'Basic realm="{realm}", charset="UTF-8"', + + # Multiple challenges per header + f'{basic}, {basic2}', + f'{basic}, {other_no_realm}', + f'{other_no_realm}, {basic}', + f'{basic}, {digest}', + f'{digest}, {basic}', + ): + headers = [f'WWW-Authenticate: {realm_str}'] + self.check_basic_auth(headers, realm) + + # no quote: expect a warning + with support.check_warnings(("Basic Auth Realm was unquoted", + UserWarning)): + headers = [f'WWW-Authenticate: Basic realm={realm}'] + self.check_basic_auth(headers, realm) + + # Multiple headers: one challenge per header. + # Use the first Basic realm. + for challenges in ( + [basic, basic2], + [basic, digest], + [digest, basic], + ): + headers = [f'WWW-Authenticate: {challenge}' + for challenge in challenges] + self.check_basic_auth(headers, realm) def test_proxy_basic_auth(self): opener = OpenerDirector() diff --git a/lib-python/3/test/test_utf8_mode.py b/lib-python/3/test/test_utf8_mode.py index 554abfab31..06fe1979dd 100644 --- a/lib-python/3/test/test_utf8_mode.py +++ b/lib-python/3/test/test_utf8_mode.py @@ -228,6 +228,8 @@ class UTF8ModeTests(unittest.TestCase): if sys.platform == 'darwin' or support.is_android: c_arg = arg_utf8 + elif sys.platform.startswith("aix"): + c_arg = arg.decode('iso-8859-1') else: c_arg = arg_ascii for loc in POSIX_LOCALES: diff --git a/lib-python/3/test/test_uu.py b/lib-python/3/test/test_uu.py index c9f05e5b76..c8709f7a0d 100644 --- a/lib-python/3/test/test_uu.py +++ b/lib-python/3/test/test_uu.py @@ -136,6 +136,15 @@ class UUTest(unittest.TestCase): decoded = codecs.decode(encodedtext, "uu_codec") self.assertEqual(decoded, plaintext) + def test_newlines_escaped(self): + # Test newlines are escaped with uu.encode + inp = io.BytesIO(plaintext) + out = io.BytesIO() + filename = "test.txt\n\roverflow.txt" + safefilename = b"test.txt\\n\\roverflow.txt" + uu.encode(inp, out, filename) + self.assertIn(safefilename, out.getvalue()) + class UUStdIOTest(unittest.TestCase): def setUp(self): diff --git a/lib-python/3/test/test_venv.py b/lib-python/3/test/test_venv.py index 67f9f46e65..a1fc6759d8 100644 --- a/lib-python/3/test/test_venv.py +++ b/lib-python/3/test/test_venv.py @@ -9,6 +9,7 @@ import ensurepip import os import os.path import re +import shutil import struct import subprocess import sys @@ -325,6 +326,37 @@ class BasicTest(BaseTest): 'pool.terminate()']) self.assertEqual(out.strip(), "python".encode()) + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + def test_deactivate_with_strict_bash_opts(self): + bash = shutil.which("bash") + if bash is None: + self.skipTest("bash required for this test") + rmtree(self.env_dir) + builder = venv.EnvBuilder(clear=True) + builder.create(self.env_dir) + activate = os.path.join(self.env_dir, self.bindir, "activate") + test_script = os.path.join(self.env_dir, "test_strict.sh") + with open(test_script, "w") as f: + f.write("set -euo pipefail\n" + f"source {activate}\n" + "deactivate\n") + out, err = check_output([bash, test_script]) + self.assertEqual(out, "".encode()) + self.assertEqual(err, "".encode()) + + + @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS') + def test_macos_env(self): + rmtree(self.env_dir) + builder = venv.EnvBuilder() + builder.create(self.env_dir) + + envpy = os.path.join(os.path.realpath(self.env_dir), + self.bindir, self.exe) + out, err = check_output([envpy, '-c', + 'import os; print("__PYVENV_LAUNCHER__" in os.environ)']) + self.assertEqual(out.strip(), 'False'.encode()) + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" @@ -438,8 +470,9 @@ class EnsurePipTest(BaseTest): # Please check the permissions and owner of that directory. If # executing pip with sudo, you may want sudo's -H flag." # where $HOME is replaced by the HOME environment variable. - err = re.sub("^The directory .* or its parent directory is not owned " - "by the current user .*$", "", err, flags=re.MULTILINE) + err = re.sub("^(WARNING: )?The directory .* or its parent directory " + "is not owned or is not writable by the current user.*$", "", + err, flags=re.MULTILINE) self.assertEqual(err.rstrip(), "") # Being fairly specific regarding the expected behaviour for the # initial bundling phase in Python 3.4. If the output changes in diff --git a/lib-python/3/test/test_warnings/__init__.py b/lib-python/3/test/test_warnings/__init__.py index 87cc3a7e36..d41f3dff02 100644 --- a/lib-python/3/test/test_warnings/__init__.py +++ b/lib-python/3/test/test_warnings/__init__.py @@ -43,6 +43,10 @@ def warnings_state(module): module.filters = original_filters +class TestWarning(Warning): + pass + + class BaseTest: """Basic bookkeeping required for testing.""" @@ -629,9 +633,28 @@ class WCmdLineTests(BaseTest): self.module._setoption, 'bogus::Warning') self.assertRaises(self.module._OptionError, self.module._setoption, 'ignore:2::4:-5') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::123') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::123abc') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::===') + with self.assertRaisesRegex(self.module._OptionError, 'Wärning'): + self.module._setoption('ignore::Wärning') self.module._setoption('error::Warning::0') self.assertRaises(UserWarning, self.module.warn, 'convert to error') + def test_import_from_module(self): + with original_warnings.catch_warnings(module=self.module): + self.module._setoption('ignore::Warning') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::TestWarning') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::test.test_warnings.bogus') + self.module._setoption('error::test.test_warnings.TestWarning') + with self.assertRaises(TestWarning): + self.module.warn('test warning', TestWarning) + class CWCmdLineTests(WCmdLineTests, unittest.TestCase): module = c_warnings diff --git a/lib-python/3/test/test_weakref.py b/lib-python/3/test/test_weakref.py index ad7a6acfcc..14ec8efdc6 100644 --- a/lib-python/3/test/test_weakref.py +++ b/lib-python/3/test/test_weakref.py @@ -376,6 +376,26 @@ class ReferencesTestCase(TestBase): lyst = List() self.assertEqual(bool(weakref.proxy(lyst)), bool(lyst)) + def test_proxy_iter(self): + # Test fails with a debug build of the interpreter + # (see bpo-38395). + + obj = None + + class MyObj: + def __iter__(self): + nonlocal obj + del obj + return NotImplemented + + obj = MyObj() + p = weakref.proxy(obj) + with self.assertRaises(TypeError): + # "blech" in p calls MyObj.__iter__ through the proxy, + # without keeping a reference to the real object, so it + # can be killed in the middle of the call + "blech" in p + def test_getweakrefcount(self): o = C() ref1 = weakref.ref(o) @@ -1770,6 +1790,11 @@ class MappingTestCase(TestBase): # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, True) + @support.cpython_only + def test_remove_closure(self): + d = weakref.WeakValueDictionary() + self.assertIsNone(d._remove.__closure__) + from test import mapping_tests diff --git a/lib-python/3/test/test_winreg.py b/lib-python/3/test/test_winreg.py index 11d054e16c..be9313db78 100644 --- a/lib-python/3/test/test_winreg.py +++ b/lib-python/3/test/test_winreg.py @@ -41,6 +41,7 @@ test_data = [ ("String Val", "A string value", REG_SZ), ("StringExpand", "The path is %path%", REG_EXPAND_SZ), ("Multi-string", ["Lots", "of", "string", "values"], REG_MULTI_SZ), + ("Multi-nul", ["", "", "", ""], REG_MULTI_SZ), ("Raw Data", b"binary\x00data", REG_BINARY), ("Big String", "x"*(2**14-1), REG_SZ), ("Big Binary", b"x"*(2**14), REG_BINARY), diff --git a/lib-python/3/test/test_wsgiref.py b/lib-python/3/test/test_wsgiref.py index 7650e1392c..da9481f839 100644 --- a/lib-python/3/test/test_wsgiref.py +++ b/lib-python/3/test/test_wsgiref.py @@ -540,32 +540,62 @@ class TestHandler(ErrorHandler): class HandlerTests(TestCase): - - def checkEnvironAttrs(self, handler): - env = handler.environ - for attr in [ - 'version','multithread','multiprocess','run_once','file_wrapper' - ]: - if attr=='file_wrapper' and handler.wsgi_file_wrapper is None: - continue - self.assertEqual(getattr(handler,'wsgi_'+attr),env['wsgi.'+attr]) - - def checkOSEnviron(self,handler): - empty = {}; setup_testing_defaults(empty) - env = handler.environ - from os import environ - for k,v in environ.items(): - if k not in empty: - self.assertEqual(env[k],v) - for k,v in empty.items(): - self.assertIn(k, env) + # testEnviron() can produce long error message + maxDiff = 80 * 50 def testEnviron(self): - h = TestHandler(X="Y") - h.setup_environ() - self.checkEnvironAttrs(h) - self.checkOSEnviron(h) - self.assertEqual(h.environ["X"],"Y") + os_environ = { + # very basic environment + 'HOME': '/my/home', + 'PATH': '/my/path', + 'LANG': 'fr_FR.UTF-8', + + # set some WSGI variables + 'SCRIPT_NAME': 'test_script_name', + 'SERVER_NAME': 'test_server_name', + } + + with support.swap_attr(TestHandler, 'os_environ', os_environ): + # override X and HOME variables + handler = TestHandler(X="Y", HOME="/override/home") + handler.setup_environ() + + # Check that wsgi_xxx attributes are copied to wsgi.xxx variables + # of handler.environ + for attr in ('version', 'multithread', 'multiprocess', 'run_once', + 'file_wrapper'): + self.assertEqual(getattr(handler, 'wsgi_' + attr), + handler.environ['wsgi.' + attr]) + + # Test handler.environ as a dict + expected = {} + setup_testing_defaults(expected) + # Handler inherits os_environ variables which are not overriden + # by SimpleHandler.add_cgi_vars() (SimpleHandler.base_env) + for key, value in os_environ.items(): + if key not in expected: + expected[key] = value + expected.update({ + # X doesn't exist in os_environ + "X": "Y", + # HOME is overriden by TestHandler + 'HOME': "/override/home", + + # overriden by setup_testing_defaults() + "SCRIPT_NAME": "", + "SERVER_NAME": "127.0.0.1", + + # set by BaseHandler.setup_environ() + 'wsgi.input': handler.get_stdin(), + 'wsgi.errors': handler.get_stderr(), + 'wsgi.version': (1, 0), + 'wsgi.run_once': False, + 'wsgi.url_scheme': 'http', + 'wsgi.multithread': True, + 'wsgi.multiprocess': True, + 'wsgi.file_wrapper': util.FileWrapper, + }) + self.assertDictEqual(handler.environ, expected) def testCGIEnviron(self): h = BaseCGIHandler(None,None,None,{}) diff --git a/lib-python/3/test/test_xml_etree_c.py b/lib-python/3/test/test_xml_etree_c.py index 2144d203e1..24bf7f3d2c 100644 --- a/lib-python/3/test/test_xml_etree_c.py +++ b/lib-python/3/test/test_xml_etree_c.py @@ -118,6 +118,23 @@ class MiscTests(unittest.TestCase): elem.tail = X() elem.__setstate__({'tag': 42}) # shouldn't cause an assertion failure + @support.cpython_only + def test_uninitialized_parser(self): + # The interpreter shouldn't crash in case of calling methods or + # accessing attributes of uninitialized XMLParser objects. + parser = cET.XMLParser.__new__(cET.XMLParser) + self.assertRaises(ValueError, parser.close) + self.assertRaises(ValueError, parser.feed, 'foo') + class MockFile: + def read(*args): + return '' + self.assertRaises(ValueError, parser._parse_whole, MockFile()) + self.assertRaises(ValueError, parser._setevents, None) + with self.assertRaises(ValueError): + parser.entity + with self.assertRaises(ValueError): + parser.target + def test_setstate_leaks(self): # Test reference leaks elem = cET.Element.__new__(cET.Element) diff --git a/lib-python/3/test/test_zipfile.py b/lib-python/3/test/test_zipfile.py index ac8f64ce22..7e8e8d2c89 100644 --- a/lib-python/3/test/test_zipfile.py +++ b/lib-python/3/test/test_zipfile.py @@ -1,5 +1,6 @@ import contextlib import io +import itertools import os import importlib.util import pathlib @@ -801,6 +802,227 @@ class StoredTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, zinfo = zipfp.getinfo("strfile") self.assertEqual(zinfo.extra, extra) + def make_zip64_file( + self, file_size_64_set=False, file_size_extra=False, + compress_size_64_set=False, compress_size_extra=False, + header_offset_64_set=False, header_offset_extra=False, + ): + """Generate bytes sequence for a zip with (incomplete) zip64 data. + + The actual values (not the zip 64 0xffffffff values) stored in the file + are: + file_size: 8 + compress_size: 8 + header_offset: 0 + """ + actual_size = 8 + actual_header_offset = 0 + local_zip64_fields = [] + central_zip64_fields = [] + + file_size = actual_size + if file_size_64_set: + file_size = 0xffffffff + if file_size_extra: + local_zip64_fields.append(actual_size) + central_zip64_fields.append(actual_size) + file_size = struct.pack("<L", file_size) + + compress_size = actual_size + if compress_size_64_set: + compress_size = 0xffffffff + if compress_size_extra: + local_zip64_fields.append(actual_size) + central_zip64_fields.append(actual_size) + compress_size = struct.pack("<L", compress_size) + + header_offset = actual_header_offset + if header_offset_64_set: + header_offset = 0xffffffff + if header_offset_extra: + central_zip64_fields.append(actual_header_offset) + header_offset = struct.pack("<L", header_offset) + + local_extra = struct.pack( + '<HH' + 'Q'*len(local_zip64_fields), + 0x0001, + 8*len(local_zip64_fields), + *local_zip64_fields + ) + + central_extra = struct.pack( + '<HH' + 'Q'*len(central_zip64_fields), + 0x0001, + 8*len(central_zip64_fields), + *central_zip64_fields + ) + + central_dir_size = struct.pack('<Q', 58 + 8 * len(central_zip64_fields)) + offset_to_central_dir = struct.pack('<Q', 50 + 8 * len(local_zip64_fields)) + + local_extra_length = struct.pack("<H", 4 + 8 * len(local_zip64_fields)) + central_extra_length = struct.pack("<H", 4 + 8 * len(central_zip64_fields)) + + filename = b"test.txt" + content = b"test1234" + filename_length = struct.pack("<H", len(filename)) + zip64_contents = ( + # Local file header + b"PK\x03\x04\x14\x00\x00\x00\x00\x00\x00\x00!\x00\x9e%\xf5\xaf" + + compress_size + + file_size + + filename_length + + local_extra_length + + filename + + local_extra + + content + # Central directory: + + b"PK\x01\x02-\x03-\x00\x00\x00\x00\x00\x00\x00!\x00\x9e%\xf5\xaf" + + compress_size + + file_size + + filename_length + + central_extra_length + + b"\x00\x00\x00\x00\x00\x00\x00\x00\x80\x01" + + header_offset + + filename + + central_extra + # Zip64 end of central directory + + b"PK\x06\x06,\x00\x00\x00\x00\x00\x00\x00-\x00-" + + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00" + + b"\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00" + + central_dir_size + + offset_to_central_dir + # Zip64 end of central directory locator + + b"PK\x06\x07\x00\x00\x00\x00l\x00\x00\x00\x00\x00\x00\x00\x01" + + b"\x00\x00\x00" + # end of central directory + + b"PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00:\x00\x00\x002\x00" + + b"\x00\x00\x00\x00" + ) + return zip64_contents + + def test_bad_zip64_extra(self): + """Missing zip64 extra records raises an exception. + + There are 4 fields that the zip64 format handles (the disk number is + not used in this module and so is ignored here). According to the zip + spec: + The order of the fields in the zip64 extended + information record is fixed, but the fields MUST + only appear if the corresponding Local or Central + directory record field is set to 0xFFFF or 0xFFFFFFFF. + + If the zip64 extra content doesn't contain enough entries for the + number of fields marked with 0xFFFF or 0xFFFFFFFF, we raise an error. + This test mismatches the length of the zip64 extra field and the number + of fields set to indicate the presence of zip64 data. + """ + # zip64 file size present, no fields in extra, expecting one, equals + # missing file size. + missing_file_size_extra = self.make_zip64_file( + file_size_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_file_size_extra)) + self.assertIn('file size', str(e.exception).lower()) + + # zip64 file size present, zip64 compress size present, one field in + # extra, expecting two, equals missing compress size. + missing_compress_size_extra = self.make_zip64_file( + file_size_64_set=True, + file_size_extra=True, + compress_size_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) + self.assertIn('compress size', str(e.exception).lower()) + + # zip64 compress size present, no fields in extra, expecting one, + # equals missing compress size. + missing_compress_size_extra = self.make_zip64_file( + compress_size_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) + self.assertIn('compress size', str(e.exception).lower()) + + # zip64 file size present, zip64 compress size present, zip64 header + # offset present, two fields in extra, expecting three, equals missing + # header offset + missing_header_offset_extra = self.make_zip64_file( + file_size_64_set=True, + file_size_extra=True, + compress_size_64_set=True, + compress_size_extra=True, + header_offset_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) + self.assertIn('header offset', str(e.exception).lower()) + + # zip64 compress size present, zip64 header offset present, one field + # in extra, expecting two, equals missing header offset + missing_header_offset_extra = self.make_zip64_file( + file_size_64_set=False, + compress_size_64_set=True, + compress_size_extra=True, + header_offset_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) + self.assertIn('header offset', str(e.exception).lower()) + + # zip64 file size present, zip64 header offset present, one field in + # extra, expecting two, equals missing header offset + missing_header_offset_extra = self.make_zip64_file( + file_size_64_set=True, + file_size_extra=True, + compress_size_64_set=False, + header_offset_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) + self.assertIn('header offset', str(e.exception).lower()) + + # zip64 header offset present, no fields in extra, expecting one, + # equals missing header offset + missing_header_offset_extra = self.make_zip64_file( + file_size_64_set=False, + compress_size_64_set=False, + header_offset_64_set=True, + ) + with self.assertRaises(zipfile.BadZipFile) as e: + zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) + self.assertIn('header offset', str(e.exception).lower()) + + def test_generated_valid_zip64_extra(self): + # These values are what is set in the make_zip64_file method. + expected_file_size = 8 + expected_compress_size = 8 + expected_header_offset = 0 + expected_content = b"test1234" + + # Loop through the various valid combinations of zip64 masks + # present and extra fields present. + params = ( + {"file_size_64_set": True, "file_size_extra": True}, + {"compress_size_64_set": True, "compress_size_extra": True}, + {"header_offset_64_set": True, "header_offset_extra": True}, + ) + + for r in range(1, len(params) + 1): + for combo in itertools.combinations(params, r): + kwargs = {} + for c in combo: + kwargs.update(c) + with zipfile.ZipFile(io.BytesIO(self.make_zip64_file(**kwargs))) as zf: + zinfo = zf.infolist()[0] + self.assertEqual(zinfo.file_size, expected_file_size) + self.assertEqual(zinfo.compress_size, expected_compress_size) + self.assertEqual(zinfo.header_offset, expected_header_offset) + self.assertEqual(zf.read(zinfo), expected_content) + + @requires_zlib class DeflateTestZip64InSmallFiles(AbstractTestZip64InSmallFiles, unittest.TestCase): @@ -1874,6 +2096,44 @@ class DecryptionTests(unittest.TestCase): self.assertRaises(TypeError, self.zip.open, "test.txt", pwd="python") self.assertRaises(TypeError, self.zip.extract, "test.txt", pwd="python") + def test_seek_tell(self): + self.zip.setpassword(b"python") + txt = self.plain + test_word = b'encryption' + bloc = txt.find(test_word) + bloc_len = len(test_word) + with self.zip.open("test.txt", "r") as fp: + fp.seek(bloc, os.SEEK_SET) + self.assertEqual(fp.tell(), bloc) + fp.seek(-bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), 0) + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.tell(), bloc) + self.assertEqual(fp.read(bloc_len), txt[bloc:bloc+bloc_len]) + + # Make sure that the second read after seeking back beyond + # _readbuffer returns the same content (ie. rewind to the start of + # the file to read forward to the required position). + old_read_size = fp.MIN_READ_SIZE + fp.MIN_READ_SIZE = 1 + fp._readbuffer = b'' + fp._offset = 0 + fp.seek(0, os.SEEK_SET) + self.assertEqual(fp.tell(), 0) + fp.seek(bloc, os.SEEK_CUR) + self.assertEqual(fp.read(bloc_len), txt[bloc:bloc+bloc_len]) + fp.MIN_READ_SIZE = old_read_size + + fp.seek(0, os.SEEK_END) + self.assertEqual(fp.tell(), len(txt)) + fp.seek(0, os.SEEK_SET) + self.assertEqual(fp.tell(), 0) + + # Read the file completely to definitely call any eof integrity + # checks (crc) and make sure they still pass. + fp.read() + + class AbstractTestsWithRandomBinaryFiles: @classmethod def setUpClass(cls): diff --git a/lib-python/3/textwrap.py b/lib-python/3/textwrap.py index 8103f34745..30e693c8de 100644 --- a/lib-python/3/textwrap.py +++ b/lib-python/3/textwrap.py @@ -420,9 +420,9 @@ def dedent(text): Note that tabs and spaces are both treated as whitespace, but they are not equal: the lines " hello" and "\\thello" are - considered to have no common leading whitespace. (This behaviour is - new in Python 2.5; older versions of this module incorrectly - expanded tabs before searching for common leading whitespace.) + considered to have no common leading whitespace. + + Entirely blank lines are normalized to a newline character. """ # Look for the longest leading string of spaces and tabs common to # all lines. diff --git a/lib-python/3/threading.py b/lib-python/3/threading.py index 0fb3bdd55c..b961456fb7 100644 --- a/lib-python/3/threading.py +++ b/lib-python/3/threading.py @@ -1122,8 +1122,7 @@ class Thread: main thread is not a daemon thread and therefore all threads created in the main thread default to daemon = False. - The entire Python program exits when no alive non-daemon threads are - left. + The entire Python program exits when only daemon threads are left. """ assert self._initialized, "Thread.__init__() not called" @@ -1341,7 +1340,15 @@ def _after_fork(): # fork() only copied the current thread; clear references to others. new_active = {} - current = current_thread() + + try: + current = _active[get_ident()] + except KeyError: + # fork() was called in a thread which was not spawned + # by threading.Thread. For example, a thread spawned + # by thread.start_new_thread(). + current = _MainThread() + _main_thread = current # reset _shutdown() locks: threads re-register their _tstate_lock below diff --git a/lib-python/3/timeit.py b/lib-python/3/timeit.py index c0362bcc5f..6c3ec01067 100755 --- a/lib-python/3/timeit.py +++ b/lib-python/3/timeit.py @@ -29,7 +29,8 @@ argument in quotes and using leading spaces. Multiple -s options are treated similarly. If -n is not given, a suitable number of loops is calculated by trying -successive powers of 10 until the total time is at least 0.2 seconds. +increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the +total time is at least 0.2 seconds. Note: there is a certain baseline overhead associated with executing a pass statement. It differs between versions. The code here doesn't try diff --git a/lib-python/3/tkinter/__init__.py b/lib-python/3/tkinter/__init__.py index a6e8fb2e61..92f64ac225 100644 --- a/lib-python/3/tkinter/__init__.py +++ b/lib-python/3/tkinter/__init__.py @@ -969,7 +969,7 @@ class Misc: """Return window class name of this widget.""" return self.tk.call('winfo', 'class', self._w) def winfo_colormapfull(self): - """Return true if at the last color request the colormap was full.""" + """Return True if at the last color request the colormap was full.""" return self.tk.getboolean( self.tk.call('winfo', 'colormapfull', self._w)) def winfo_containing(self, rootX, rootY, displayof=0): @@ -2829,7 +2829,7 @@ class Listbox(Widget, XView, YView): 'selection', 'clear', first, last) select_clear = selection_clear def selection_includes(self, index): - """Return 1 if INDEX is part of the selection.""" + """Return True if INDEX is part of the selection.""" return self.tk.getboolean(self.tk.call( self._w, 'selection', 'includes', index)) select_includes = selection_includes diff --git a/lib-python/3/tkinter/test/test_tkinter/test_misc.py b/lib-python/3/tkinter/test/test_tkinter/test_misc.py index 1d1a3c29f6..236cae0e9f 100644 --- a/lib-python/3/tkinter/test/test_tkinter/test_misc.py +++ b/lib-python/3/tkinter/test/test_tkinter/test_misc.py @@ -156,6 +156,28 @@ class MiscTest(AbstractTkTest, unittest.TestCase): with self.assertRaises(tkinter.TclError): root.tk.call('after', 'info', idle1) + def test_clipboard(self): + root = self.root + root.clipboard_clear() + root.clipboard_append('Ùñî') + self.assertEqual(root.clipboard_get(), 'Ùñî') + root.clipboard_append('çōđě') + self.assertEqual(root.clipboard_get(), 'Ùñîçōđě') + root.clipboard_clear() + with self.assertRaises(tkinter.TclError): + root.clipboard_get() + + def test_clipboard_astral(self): + root = self.root + root.clipboard_clear() + root.clipboard_append('𝔘𝔫𝔦') + self.assertEqual(root.clipboard_get(), '𝔘𝔫𝔦') + root.clipboard_append('𝔠𝔬𝔡𝔢') + self.assertEqual(root.clipboard_get(), '𝔘𝔫𝔦𝔠𝔬𝔡𝔢') + root.clipboard_clear() + with self.assertRaises(tkinter.TclError): + root.clipboard_get() + tests_gui = (MiscTest, ) diff --git a/lib-python/3/tkinter/test/test_ttk/test_widgets.py b/lib-python/3/tkinter/test/test_ttk/test_widgets.py index ba9e3b54f7..9667e05cab 100644 --- a/lib-python/3/tkinter/test/test_ttk/test_widgets.py +++ b/lib-python/3/tkinter/test/test_ttk/test_widgets.py @@ -489,8 +489,7 @@ class ComboboxTest(EntryTest, unittest.TestCase): expected=('mon', 'tue', 'wed', 'thur')) self.checkParam(self.combo, 'values', ('mon', 'tue', 'wed', 'thur')) self.checkParam(self.combo, 'values', (42, 3.14, '', 'any string')) - self.checkParam(self.combo, 'values', '', - expected='' if get_tk_patchlevel() < (8, 5, 10) else ()) + self.checkParam(self.combo, 'values', '') self.combo['values'] = ['a', 1, 'c'] @@ -1245,12 +1244,7 @@ class SpinboxTest(EntryTest, unittest.TestCase): expected=('mon', 'tue', 'wed', 'thur')) self.checkParam(self.spin, 'values', ('mon', 'tue', 'wed', 'thur')) self.checkParam(self.spin, 'values', (42, 3.14, '', 'any string')) - self.checkParam( - self.spin, - 'values', - '', - expected='' if get_tk_patchlevel() < (8, 5, 10) else () - ) + self.checkParam(self.spin, 'values', '') self.spin['values'] = ['a', 1, 'c'] @@ -1308,8 +1302,7 @@ class TreeviewTest(AbstractWidgetTest, unittest.TestCase): self.checkParam(widget, 'columns', 'a b c', expected=('a', 'b', 'c')) self.checkParam(widget, 'columns', ('a', 'b', 'c')) - self.checkParam(widget, 'columns', (), - expected='' if get_tk_patchlevel() < (8, 5, 10) else ()) + self.checkParam(widget, 'columns', '') def test_displaycolumns(self): widget = self.create() diff --git a/lib-python/3/tkinter/test/widget_tests.py b/lib-python/3/tkinter/test/widget_tests.py index 75a068fbbf..b42ff52178 100644 --- a/lib-python/3/tkinter/test/widget_tests.py +++ b/lib-python/3/tkinter/test/widget_tests.py @@ -3,7 +3,6 @@ import unittest import sys import tkinter -from tkinter.ttk import Scale from tkinter.test.support import (AbstractTkTest, tcl_version, requires_tcl, get_tk_patchlevel, pixels_conv, tcl_obj_eq) import test.support @@ -63,11 +62,9 @@ class AbstractWidgetTest(AbstractTkTest): eq = tcl_obj_eq self.assertEqual2(widget[name], expected, eq=eq) self.assertEqual2(widget.cget(name), expected, eq=eq) - # XXX - if not isinstance(widget, Scale): - t = widget.configure(name) - self.assertEqual(len(t), 5) - self.assertEqual2(t[4], expected, eq=eq) + t = widget.configure(name) + self.assertEqual(len(t), 5) + self.assertEqual2(t[4], expected, eq=eq) def checkInvalidParam(self, widget, name, value, errmsg=None, *, keep_orig=True): @@ -209,9 +206,7 @@ class AbstractWidgetTest(AbstractTkTest): def test_keys(self): widget = self.create() keys = widget.keys() - # XXX - if not isinstance(widget, Scale): - self.assertEqual(sorted(keys), sorted(widget.configure())) + self.assertEqual(sorted(keys), sorted(widget.configure())) for k in keys: widget[k] # Test if OPTIONS contains all keys diff --git a/lib-python/3/tkinter/tix.py b/lib-python/3/tkinter/tix.py index d9c097a77c..ac545502e4 100644 --- a/lib-python/3/tkinter/tix.py +++ b/lib-python/3/tkinter/tix.py @@ -1890,7 +1890,7 @@ class Grid(TixWidget, XView, YView): containing the current size setting of the given column. When option-value pairs are given, the corresponding options of the size setting of the given column are changed. Options may be one - of the follwing: + of the following: pad0 pixels Specifies the paddings to the left of a column. pad1 pixels @@ -1915,7 +1915,7 @@ class Grid(TixWidget, XView, YView): When no option-value pair is given, this command returns a list con- taining the current size setting of the given row . When option-value pairs are given, the corresponding options of the size setting of the - given row are changed. Options may be one of the follwing: + given row are changed. Options may be one of the following: pad0 pixels Specifies the paddings to the top of a row. pad1 pixels diff --git a/lib-python/3/tkinter/ttk.py b/lib-python/3/tkinter/ttk.py index 12f4ac0bd9..52b1a30928 100644 --- a/lib-python/3/tkinter/ttk.py +++ b/lib-python/3/tkinter/ttk.py @@ -1086,11 +1086,12 @@ class Scale(Widget, tkinter.Scale): Setting a value for any of the "from", "from_" or "to" options generates a <<RangeChanged>> event.""" - if cnf: + retval = Widget.configure(self, cnf, **kw) + if not isinstance(cnf, (type(None), str)): kw.update(cnf) - Widget.configure(self, **kw) if any(['from' in kw, 'from_' in kw, 'to' in kw]): self.event_generate('<<RangeChanged>>') + return retval def get(self, x=None, y=None): diff --git a/lib-python/3/trace.py b/lib-python/3/trace.py index 206bd2b689..c804a0d756 100755 --- a/lib-python/3/trace.py +++ b/lib-python/3/trace.py @@ -53,6 +53,7 @@ import linecache import os import re import sys +import sysconfig import token import tokenize import inspect @@ -671,9 +672,8 @@ def main(): opts = parser.parse_args() if opts.ignore_dir: - rel_path = 'lib', 'python{0.major}.{0.minor}'.format(sys.version_info) - _prefix = os.path.join(sys.base_prefix, *rel_path) - _exec_prefix = os.path.join(sys.base_exec_prefix, *rel_path) + _prefix = sysconfig.get_path("stdlib") + _exec_prefix = sysconfig.get_path("platstdlib") def parse_ignore_dir(s): s = os.path.expanduser(os.path.expandvars(s)) diff --git a/lib-python/3/traceback.py b/lib-python/3/traceback.py index 4e7605d15f..59d6d1d069 100644 --- a/lib-python/3/traceback.py +++ b/lib-python/3/traceback.py @@ -546,7 +546,7 @@ class TracebackException: The return value is a generator of strings, each ending in a newline. Normally, the generator emits a single string; however, for - SyntaxError exceptions, it emites several lines that (when + SyntaxError exceptions, it emits several lines that (when printed) display detailed information about where the syntax error occurred. diff --git a/lib-python/3/turtledemo/__main__.py b/lib-python/3/turtledemo/__main__.py index 17fe9a75e1..12be5098da 100644 --- a/lib-python/3/turtledemo/__main__.py +++ b/lib-python/3/turtledemo/__main__.py @@ -272,7 +272,7 @@ class DemoWindow(object): self.stop_btn.config(state=stop, bg="#d00" if stop == NORMAL else "#fca") self.clear_btn.config(state=clear, - bg="#d00" if clear == NORMAL else"#fca") + bg="#d00" if clear == NORMAL else "#fca") self.output_lbl.config(text=txt, fg=color) def makeLoadDemoMenu(self, master): diff --git a/lib-python/3/typing.py b/lib-python/3/typing.py index 9851cb4c7e..3deb1eac3b 100644 --- a/lib-python/3/typing.py +++ b/lib-python/3/typing.py @@ -473,11 +473,13 @@ class ForwardRef(_Final, _root=True): def __eq__(self, other): if not isinstance(other, ForwardRef): return NotImplemented - return (self.__forward_arg__ == other.__forward_arg__ and - self.__forward_value__ == other.__forward_value__) + if self.__forward_evaluated__ and other.__forward_evaluated__: + return (self.__forward_arg__ == other.__forward_arg__ and + self.__forward_value__ == other.__forward_value__) + return self.__forward_arg__ == other.__forward_arg__ def __hash__(self): - return hash((self.__forward_arg__, self.__forward_value__)) + return hash(self.__forward_arg__) def __repr__(self): return f'ForwardRef({self.__forward_arg__!r})' @@ -547,7 +549,10 @@ class TypeVar(_Final, _Immutable, _root=True): self.__bound__ = _type_check(bound, "Bound must be a type.") else: self.__bound__ = None - def_mod = sys._getframe(1).f_globals['__name__'] # for pickling + try: + def_mod = sys._getframe(1).f_globals.get('__name__', '__main__') # for pickling + except (AttributeError, ValueError): + def_mod = None if def_mod != 'typing': self.__module__ = def_mod @@ -981,7 +986,11 @@ def get_type_hints(obj, globalns=None, localns=None): if isinstance(obj, types.ModuleType): globalns = obj.__dict__ else: - globalns = getattr(obj, '__globals__', {}) + nsobj = obj + # Find globalns for the unwrapped object. + while hasattr(nsobj, '__wrapped__'): + nsobj = nsobj.__wrapped__ + globalns = getattr(nsobj, '__globals__', {}) if localns is None: localns = globalns elif localns is None: @@ -1277,6 +1286,7 @@ Type.__doc__ = \ class SupportsInt(_Protocol): + """An ABC with one abstract method __int__.""" __slots__ = () @abstractmethod @@ -1285,6 +1295,7 @@ class SupportsInt(_Protocol): class SupportsFloat(_Protocol): + """An ABC with one abstract method __float__.""" __slots__ = () @abstractmethod @@ -1293,6 +1304,7 @@ class SupportsFloat(_Protocol): class SupportsComplex(_Protocol): + """An ABC with one abstract method __complex__.""" __slots__ = () @abstractmethod @@ -1301,6 +1313,7 @@ class SupportsComplex(_Protocol): class SupportsBytes(_Protocol): + """An ABC with one abstract method __bytes__.""" __slots__ = () @abstractmethod @@ -1309,6 +1322,7 @@ class SupportsBytes(_Protocol): class SupportsAbs(_Protocol[T_co]): + """An ABC with one abstract method __abs__ that is covariant in its return type.""" __slots__ = () @abstractmethod @@ -1317,6 +1331,7 @@ class SupportsAbs(_Protocol[T_co]): class SupportsRound(_Protocol[T_co]): + """An ABC with one abstract method __round__ that is covariant in its return type.""" __slots__ = () @abstractmethod @@ -1343,7 +1358,7 @@ _prohibited = ('__new__', '__init__', '__slots__', '__getnewargs__', '_fields', '_field_defaults', '_field_types', '_make', '_replace', '_asdict', '_source') -_special = ('__module__', '__name__', '__qualname__', '__annotations__') +_special = ('__module__', '__name__', '__annotations__') class NamedTupleMeta(type): @@ -1405,13 +1420,36 @@ class NamedTuple(metaclass=NamedTupleMeta): """ _root = True - def __new__(self, typename, fields=None, **kwargs): + def __new__(*args, **kwargs): + if not args: + raise TypeError('NamedTuple.__new__(): not enough arguments') + cls, *args = args # allow the "cls" keyword be passed + if args: + typename, *args = args # allow the "typename" keyword be passed + elif 'typename' in kwargs: + typename = kwargs.pop('typename') + else: + raise TypeError("NamedTuple.__new__() missing 1 required positional " + "argument: 'typename'") + if args: + try: + fields, = args # allow the "fields" keyword be passed + except ValueError: + raise TypeError(f'NamedTuple.__new__() takes from 2 to 3 ' + f'positional arguments but {len(args) + 2} ' + f'were given') from None + elif 'fields' in kwargs and len(kwargs) == 1: + fields = kwargs.pop('fields') + else: + fields = None + if fields is None: fields = kwargs.items() elif kwargs: raise TypeError("Either list of fields or keywords" " can be provided to NamedTuple, not both") return _make_nmtuple(typename, fields) + __new__.__text_signature__ = '($cls, typename, fields=None, /, **kwargs)' def NewType(name, tp): @@ -1476,7 +1514,7 @@ class IO(Generic[AnyStr]): def close(self) -> None: pass - @abstractmethod + @abstractproperty def closed(self) -> bool: pass diff --git a/lib-python/3/unittest/case.py b/lib-python/3/unittest/case.py index 811f5df23d..d65ff1c5fb 100644 --- a/lib-python/3/unittest/case.py +++ b/lib-python/3/unittest/case.py @@ -227,7 +227,7 @@ class _AssertWarnsContext(_AssertRaisesBaseContext): def __enter__(self): # The __warningregistry__'s need to be in a pristine state for tests # to work properly. - for v in sys.modules.values(): + for v in list(sys.modules.values()): if getattr(v, '__warningregistry__', None): v.__warningregistry__ = {} self.warnings_manager = warnings.catch_warnings(record=True) @@ -493,7 +493,7 @@ class TestCase(object): the specified test method's docstring. """ doc = self._testMethodDoc - return doc and doc.split("\n")[0].strip() or None + return doc.strip().split("\n")[0].strip() if doc else None def id(self): diff --git a/lib-python/3/unittest/mock.py b/lib-python/3/unittest/mock.py index 569a5146c8..43fb00feac 100644 --- a/lib-python/3/unittest/mock.py +++ b/lib-python/3/unittest/mock.py @@ -30,6 +30,7 @@ import inspect import pprint import sys import builtins +import contextlib from types import ModuleType, MethodType from functools import wraps, partial @@ -62,6 +63,15 @@ def _is_exception(obj): ) +def _extract_mock(obj): + # Autospecced functions will return a FunctionType with "mock" attribute + # which is the actual mock object that needs to be used. + if isinstance(obj, FunctionTypes) and hasattr(obj, 'mock'): + return obj.mock + else: + return obj + + def _get_signature_object(func, as_instance, eat_self): """ Given an arbitrary, possibly callable object, try to create a suitable @@ -323,13 +333,7 @@ class _CallList(list): def _check_and_set_parent(parent, value, name, new_name): - # function passed to create_autospec will have mock - # attribute attached to which parent must be set - if isinstance(value, FunctionTypes): - try: - value = value.mock - except AttributeError: - pass + value = _extract_mock(value) if not _is_instance_mock(value): return False @@ -433,10 +437,12 @@ class NonCallableMock(Base): Attach a mock as an attribute of this one, replacing its name and parent. Calls to the attached mock will be recorded in the `method_calls` and `mock_calls` attributes of this one.""" - mock._mock_parent = None - mock._mock_new_parent = None - mock._mock_name = '' - mock._mock_new_name = None + inner_mock = _extract_mock(mock) + + inner_mock._mock_parent = None + inner_mock._mock_new_parent = None + inner_mock._mock_name = '' + inner_mock._mock_new_name = None setattr(self, attribute, mock) @@ -766,6 +772,39 @@ class NonCallableMock(Base): return message % (expected_string, actual_string) + def _get_call_signature_from_name(self, name): + """ + * If call objects are asserted against a method/function like obj.meth1 + then there could be no name for the call object to lookup. Hence just + return the spec_signature of the method/function being asserted against. + * If the name is not empty then remove () and split by '.' to get + list of names to iterate through the children until a potential + match is found. A child mock is created only during attribute access + so if we get a _SpecState then no attributes of the spec were accessed + and can be safely exited. + """ + if not name: + return self._spec_signature + + sig = None + names = name.replace('()', '').split('.') + children = self._mock_children + + for name in names: + child = children.get(name) + if child is None or isinstance(child, _SpecState): + break + else: + # If an autospecced object is attached using attach_mock the + # child would be a function with mock object as attribute from + # which signature has to be derived. + child = _extract_mock(child) + children = child._mock_children + sig = child._spec_signature + + return sig + + def _call_matcher(self, _call): """ Given a call (or simply an (args, kwargs) tuple), return a @@ -773,7 +812,12 @@ class NonCallableMock(Base): This is a best effort method which relies on the spec's signature, if available, or falls back on the arguments themselves. """ - sig = self._spec_signature + + if isinstance(_call, tuple) and len(_call) > 2: + sig = self._get_call_signature_from_name(_call[0]) + else: + sig = self._spec_signature + if sig is not None: if len(_call) == 2: name = '' @@ -856,13 +900,20 @@ class NonCallableMock(Base): If `any_order` is True then the calls can be in any order, but they must all appear in `mock_calls`.""" expected = [self._call_matcher(c) for c in calls] - cause = expected if isinstance(expected, Exception) else None + cause = next((e for e in expected if isinstance(e, Exception)), None) all_calls = _CallList(self._call_matcher(c) for c in self.mock_calls) if not any_order: if expected not in all_calls: + if cause is None: + problem = 'Calls not found.' + else: + problem = ('Error processing expected calls.\n' + 'Errors: {}').format( + [e if isinstance(e, Exception) else None + for e in expected]) raise AssertionError( - 'Calls not found.\nExpected: %r\n' - 'Actual: %r' % (_CallList(calls), self.mock_calls) + '%s\nExpected: %r\nActual: %r' % ( + problem, _CallList(calls), self.mock_calls) ) from cause return @@ -1193,13 +1244,9 @@ class _patch(object): @wraps(func) def patched(*args, **keywargs): extra_args = [] - entered_patchers = [] - - exc_info = tuple() - try: + with contextlib.ExitStack() as exit_stack: for patching in patched.patchings: - arg = patching.__enter__() - entered_patchers.append(patching) + arg = exit_stack.enter_context(patching) if patching.attribute_name is not None: keywargs.update(arg) elif patching.new is DEFAULT: @@ -1207,19 +1254,6 @@ class _patch(object): args += tuple(extra_args) return func(*args, **keywargs) - except: - if (patching not in entered_patchers and - _is_started(patching)): - # the patcher may have been started, but an exception - # raised whilst entering one of its additional_patchers - entered_patchers.append(patching) - # Pass the exception to __exit__ - exc_info = sys.exc_info() - # re-raise the exception - raise - finally: - for patching in reversed(entered_patchers): - patching.__exit__(*exc_info) patched.patchings = [self] return patched @@ -1361,19 +1395,23 @@ class _patch(object): self.temp_original = original self.is_local = local - setattr(self.target, self.attribute, new_attr) - if self.attribute_name is not None: - extra_args = {} - if self.new is DEFAULT: - extra_args[self.attribute_name] = new - for patching in self.additional_patchers: - arg = patching.__enter__() - if patching.new is DEFAULT: - extra_args.update(arg) - return extra_args - - return new - + self._exit_stack = contextlib.ExitStack() + try: + setattr(self.target, self.attribute, new_attr) + if self.attribute_name is not None: + extra_args = {} + if self.new is DEFAULT: + extra_args[self.attribute_name] = new + for patching in self.additional_patchers: + arg = self._exit_stack.enter_context(patching) + if patching.new is DEFAULT: + extra_args.update(arg) + return extra_args + + return new + except: + if not self.__exit__(*sys.exc_info()): + raise def __exit__(self, *exc_info): """Undo the patch.""" @@ -1394,9 +1432,9 @@ class _patch(object): del self.temp_original del self.is_local del self.target - for patcher in reversed(self.additional_patchers): - if _is_started(patcher): - patcher.__exit__(*exc_info) + exit_stack = self._exit_stack + del self._exit_stack + return exit_stack.__exit__(*exc_info) def start(self): @@ -1414,7 +1452,7 @@ class _patch(object): # If the patch hasn't been started this will fail pass - return self.__exit__() + return self.__exit__(None, None, None) @@ -1446,6 +1484,10 @@ def _patch_object( When used as a class decorator `patch.object` honours `patch.TEST_PREFIX` for choosing which methods to wrap. """ + if type(target) is str: + raise TypeError( + f"{target!r} must be the actual object to be patched, not a str" + ) getter = lambda: target return _patch( getter, attribute, new, spec, create, @@ -1837,10 +1879,10 @@ def _set_return_value(mock, method, name): method.return_value = fixed return - return_calulator = _calculate_return_value.get(name) - if return_calulator is not None: + return_calculator = _calculate_return_value.get(name) + if return_calculator is not None: try: - return_value = return_calulator(mock) + return_value = return_calculator(mock) except AttributeError: # XXXX why do we return AttributeError here? # set it as a side_effect instead? @@ -2288,7 +2330,7 @@ def _must_skip(spec, entry, is_type): continue if isinstance(result, (staticmethod, classmethod)): return False - elif isinstance(getattr(result, '__get__', None), MethodWrapperTypes): + elif isinstance(result, FunctionTypes): # Normal method => skip if looked up on type # (if looked up on instance, self is already skipped) return is_type @@ -2327,10 +2369,6 @@ FunctionTypes = ( type(ANY.__eq__), ) -MethodWrapperTypes = ( - type(ANY.__eq__.__get__), -) - file_spec = None diff --git a/lib-python/3/unittest/test/test_case.py b/lib-python/3/unittest/test/test_case.py index 6d58201ea8..ff541f8cf0 100644 --- a/lib-python/3/unittest/test/test_case.py +++ b/lib-python/3/unittest/test/test_case.py @@ -8,6 +8,7 @@ import logging import warnings import weakref import inspect +import types from copy import deepcopy from test import support @@ -610,6 +611,15 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing): 'Tests shortDescription() for a method with a longer ' 'docstring.') + def testShortDescriptionWhitespaceTrimming(self): + """ + Tests shortDescription() whitespace is trimmed, so that the first + line of nonwhite-space text becomes the docstring. + """ + self.assertEqual( + self.shortDescription(), + 'Tests shortDescription() whitespace is trimmed, so that the first') + def testAddTypeEqualityFunc(self): class SadSnake(object): """Dummy class for test_addTypeEqualityFunc.""" @@ -1343,6 +1353,20 @@ test case pass self.assertRaises(TypeError, self.assertWarnsRegex, MyWarn, lambda: True) + def testAssertWarnsModifySysModules(self): + # bpo-29620: handle modified sys.modules during iteration + class Foo(types.ModuleType): + @property + def __warningregistry__(self): + sys.modules['@bar@'] = 'bar' + + sys.modules['@foo@'] = Foo('foo') + try: + self.assertWarns(UserWarning, warnings.warn, 'expected') + finally: + del sys.modules['@foo@'] + del sys.modules['@bar@'] + def testAssertRaisesRegexMismatch(self): def Stub(): raise Exception('Unexpected') diff --git a/lib-python/3/unittest/test/testmock/testmock.py b/lib-python/3/unittest/test/testmock/testmock.py index f92b921fe6..12772d39fc 100644 --- a/lib-python/3/unittest/test/testmock/testmock.py +++ b/lib-python/3/unittest/test/testmock/testmock.py @@ -1,4 +1,5 @@ import copy +import re import sys import tempfile @@ -39,6 +40,9 @@ class Something(object): pass +def something(a): pass + + class MockTest(unittest.TestCase): def test_all(self): @@ -1311,6 +1315,54 @@ class MockTest(unittest.TestCase): ) + def test_assert_has_calls_nested_spec(self): + class Something: + + def __init__(self): pass + def meth(self, a, b, c, d=None): pass + + class Foo: + + def __init__(self, a): pass + def meth1(self, a, b): pass + + mock_class = create_autospec(Something) + + for m in [mock_class, mock_class()]: + m.meth(1, 2, 3, d=1) + m.assert_has_calls([call.meth(1, 2, 3, d=1)]) + m.assert_has_calls([call.meth(1, 2, 3, 1)]) + + mock_class.reset_mock() + + for m in [mock_class, mock_class()]: + self.assertRaises(AssertionError, m.assert_has_calls, [call.Foo()]) + m.Foo(1).meth1(1, 2) + m.assert_has_calls([call.Foo(1), call.Foo(1).meth1(1, 2)]) + m.Foo.assert_has_calls([call(1), call().meth1(1, 2)]) + + mock_class.reset_mock() + + invalid_calls = [call.meth(1), + call.non_existent(1), + call.Foo().non_existent(1), + call.Foo().meth(1, 2, 3, 4)] + + for kall in invalid_calls: + self.assertRaises(AssertionError, + mock_class.assert_has_calls, + [kall] + ) + + + def test_assert_has_calls_nested_without_spec(self): + m = MagicMock() + m().foo().bar().baz() + m.one().two().three() + calls = call.one().two().three().call_list() + m.assert_has_calls(calls) + + def test_assert_has_calls_with_function_spec(self): def f(a, b, c, d=None): pass @@ -1343,6 +1395,32 @@ class MockTest(unittest.TestCase): mock.assert_has_calls(calls[:-1]) mock.assert_has_calls(calls[:-1], any_order=True) + def test_assert_has_calls_not_matching_spec_error(self): + def f(x=None): pass + + mock = Mock(spec=f) + mock(1) + + with self.assertRaisesRegex( + AssertionError, + '^{}$'.format( + re.escape('Calls not found.\n' + 'Expected: [call()]\n' + 'Actual: [call(1)]'))) as cm: + mock.assert_has_calls([call()]) + self.assertIsNone(cm.exception.__cause__) + + + with self.assertRaisesRegex( + AssertionError, + '^{}$'.format( + re.escape( + 'Error processing expected calls.\n' + "Errors: [None, TypeError('too many positional arguments')]\n" + "Expected: [call(), call(1, 2)]\n" + 'Actual: [call(1)]'))) as cm: + mock.assert_has_calls([call(), call(1, 2)]) + self.assertIsInstance(cm.exception.__cause__, TypeError) def test_assert_any_call(self): mock = Mock() @@ -1765,6 +1843,55 @@ class MockTest(unittest.TestCase): self.assertEqual(m.mock_calls, call().foo().call_list()) + def test_attach_mock_patch_autospec(self): + parent = Mock() + + with mock.patch(f'{__name__}.something', autospec=True) as mock_func: + self.assertEqual(mock_func.mock._extract_mock_name(), 'something') + parent.attach_mock(mock_func, 'child') + parent.child(1) + something(2) + mock_func(3) + + parent_calls = [call.child(1), call.child(2), call.child(3)] + child_calls = [call(1), call(2), call(3)] + self.assertEqual(parent.mock_calls, parent_calls) + self.assertEqual(parent.child.mock_calls, child_calls) + self.assertEqual(something.mock_calls, child_calls) + self.assertEqual(mock_func.mock_calls, child_calls) + self.assertIn('mock.child', repr(parent.child.mock)) + self.assertEqual(mock_func.mock._extract_mock_name(), 'mock.child') + + + def test_attach_mock_patch_autospec_signature(self): + with mock.patch(f'{__name__}.Something.meth', autospec=True) as mocked: + manager = Mock() + manager.attach_mock(mocked, 'attach_meth') + obj = Something() + obj.meth(1, 2, 3, d=4) + manager.assert_has_calls([call.attach_meth(mock.ANY, 1, 2, 3, d=4)]) + obj.meth.assert_has_calls([call(mock.ANY, 1, 2, 3, d=4)]) + mocked.assert_has_calls([call(mock.ANY, 1, 2, 3, d=4)]) + + with mock.patch(f'{__name__}.something', autospec=True) as mocked: + manager = Mock() + manager.attach_mock(mocked, 'attach_func') + something(1) + manager.assert_has_calls([call.attach_func(1)]) + something.assert_has_calls([call(1)]) + mocked.assert_has_calls([call(1)]) + + with mock.patch(f'{__name__}.Something', autospec=True) as mocked: + manager = Mock() + manager.attach_mock(mocked, 'attach_obj') + obj = Something() + obj.meth(1, 2, 3, d=4) + manager.assert_has_calls([call.attach_obj(), + call.attach_obj().meth(1, 2, 3, d=4)]) + obj.meth.assert_has_calls([call(1, 2, 3, d=4)]) + mocked.assert_has_calls([call(), call().meth(1, 2, 3, d=4)]) + + def test_attribute_deletion(self): for mock in (Mock(), MagicMock(), NonCallableMagicMock(), NonCallableMock()): @@ -1849,6 +1976,20 @@ class MockTest(unittest.TestCase): self.assertRaises(TypeError, mock.child, 1) self.assertEqual(mock.mock_calls, [call.child(1, 2)]) + self.assertIn('mock.child', repr(mock.child.mock)) + + def test_parent_propagation_with_autospec_attach_mock(self): + + def foo(a, b): pass + + parent = Mock() + parent.attach_mock(create_autospec(foo, name='bar'), 'child') + parent.child(1, 2) + + self.assertRaises(TypeError, parent.child, 1) + self.assertEqual(parent.child.mock_calls, [call.child(1, 2)]) + self.assertIn('mock.child', repr(parent.child.mock)) + def test_isinstance_under_settrace(self): # bpo-36593 : __class__ is not set for a class that has __class__ diff --git a/lib-python/3/unittest/test/testmock/testpatch.py b/lib-python/3/unittest/test/testmock/testpatch.py index 6358154b3e..7453c4902a 100644 --- a/lib-python/3/unittest/test/testmock/testpatch.py +++ b/lib-python/3/unittest/test/testmock/testpatch.py @@ -112,6 +112,10 @@ class PatchTest(unittest.TestCase): self.assertEqual(Something.attribute, sentinel.Original, "patch not restored") + def test_patchobject_with_string_as_target(self): + msg = "'Something' must be the actual object to be patched, not a str" + with self.assertRaisesRegex(TypeError, msg): + patch.object('Something', 'do_something') def test_patchobject_with_none(self): class Something(object): diff --git a/lib-python/3/urllib/parse.py b/lib-python/3/urllib/parse.py index 4c8e77fe39..94df275c46 100644 --- a/lib-python/3/urllib/parse.py +++ b/lib-python/3/urllib/parse.py @@ -1017,9 +1017,9 @@ def splitport(host): """splitport('host:port') --> 'host', 'port'.""" global _portprog if _portprog is None: - _portprog = re.compile('(.*):([0-9]*)$', re.DOTALL) + _portprog = re.compile('(.*):([0-9]*)', re.DOTALL) - match = _portprog.match(host) + match = _portprog.fullmatch(host) if match: host, port = match.groups() if port: diff --git a/lib-python/3/urllib/request.py b/lib-python/3/urllib/request.py index 37b2548628..4f42919b09 100644 --- a/lib-python/3/urllib/request.py +++ b/lib-python/3/urllib/request.py @@ -944,8 +944,15 @@ class AbstractBasicAuthHandler: # allow for double- and single-quoted realm values # (single quotes are a violation of the RFC, but appear in the wild) - rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' - 'realm=(["\']?)([^"\']*)\\2', re.I) + rx = re.compile('(?:^|,)' # start of the string or ',' + '[ \t]*' # optional whitespaces + '([^ \t]+)' # scheme like "Basic" + '[ \t]+' # mandatory whitespaces + # realm=xxx + # realm='xxx' + # realm="xxx" + 'realm=(["\']?)([^"\']*)\\2', + re.I) # XXX could pre-emptively send auth info already accepted (RFC 2617, # end of section 2, and section 1.2 immediately after "credentials" @@ -957,27 +964,51 @@ class AbstractBasicAuthHandler: self.passwd = password_mgr self.add_password = self.passwd.add_password + def _parse_realm(self, header): + # parse WWW-Authenticate header: accept multiple challenges per header + found_challenge = False + for mo in AbstractBasicAuthHandler.rx.finditer(header): + scheme, quote, realm = mo.groups() + if quote not in ['"', "'"]: + warnings.warn("Basic Auth Realm was unquoted", + UserWarning, 3) + + yield (scheme, realm) + + found_challenge = True + + if not found_challenge: + if header: + scheme = header.split()[0] + else: + scheme = '' + yield (scheme, None) + def http_error_auth_reqed(self, authreq, host, req, headers): # host may be an authority (without userinfo) or a URL with an # authority - # XXX could be multiple headers - authreq = headers.get(authreq, None) + headers = headers.get_all(authreq) + if not headers: + # no header found + return - if authreq: - scheme = authreq.split()[0] - if scheme.lower() != 'basic': - raise ValueError("AbstractBasicAuthHandler does not" - " support the following scheme: '%s'" % - scheme) - else: - mo = AbstractBasicAuthHandler.rx.search(authreq) - if mo: - scheme, quote, realm = mo.groups() - if quote not in ['"',"'"]: - warnings.warn("Basic Auth Realm was unquoted", - UserWarning, 2) - if scheme.lower() == 'basic': - return self.retry_http_basic_auth(host, req, realm) + unsupported = None + for header in headers: + for scheme, realm in self._parse_realm(header): + if scheme.lower() != 'basic': + unsupported = scheme + continue + + if realm is not None: + # Use the first matching Basic challenge. + # Ignore following challenges even if they use the Basic + # scheme. + return self.retry_http_basic_auth(host, req, realm) + + if unsupported is not None: + raise ValueError("AbstractBasicAuthHandler does not " + "support the following scheme: %r" + % (scheme,)) def retry_http_basic_auth(self, host, req, realm): user, pw = self.passwd.find_user_password(realm, host) @@ -1143,7 +1174,11 @@ class AbstractDigestAuthHandler: A2 = "%s:%s" % (req.get_method(), # XXX selector: what about proxies and full urls req.selector) - if qop == 'auth': + # NOTE: As per RFC 2617, when server sends "auth,auth-int", the client could use either `auth` + # or `auth-int` to the response back. we use `auth` to send the response back. + if qop is None: + respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) + elif 'auth' in qop.split(','): if nonce == self.last_nonce: self.nonce_count += 1 else: @@ -1151,10 +1186,8 @@ class AbstractDigestAuthHandler: self.last_nonce = nonce ncvalue = '%08x' % self.nonce_count cnonce = self.get_cnonce(nonce) - noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, qop, H(A2)) + noncebit = "%s:%s:%s:%s:%s" % (nonce, ncvalue, cnonce, 'auth', H(A2)) respdig = KD(H(A1), noncebit) - elif qop is None: - respdig = KD(H(A1), "%s:%s" % (nonce, H(A2))) else: # XXX handle auth-int. raise URLError("qop '%s' is not supported." % qop) @@ -2497,24 +2530,26 @@ def proxy_bypass_environment(host, proxies=None): try: no_proxy = proxies['no'] except KeyError: - return 0 + return False # '*' is special case for always bypass if no_proxy == '*': - return 1 + return True + host = host.lower() # strip port off host hostonly, port = splitport(host) # check if the host ends with any of the DNS suffixes - no_proxy_list = [proxy.strip() for proxy in no_proxy.split(',')] - for name in no_proxy_list: + for name in no_proxy.split(','): + name = name.strip() if name: name = name.lstrip('.') # ignore leading dots - name = re.escape(name) - pattern = r'(.+\.)?%s$' % name - if (re.match(pattern, hostonly, re.I) - or re.match(pattern, host, re.I)): - return 1 + name = name.lower() + if hostonly == name or host == name: + return True + name = '.' + name + if hostonly.endswith(name) or host.endswith(name): + return True # otherwise, don't bypass - return 0 + return False # This code tests an OSX specific data structure but is testable on all @@ -2640,7 +2675,7 @@ elif os.name == 'nt': for p in proxyServer.split(';'): protocol, address = p.split('=', 1) # See if address has a type:// prefix - if not re.match('^([^/:]+)://', address): + if not re.match('(?:[^/:]+)://', address): address = '%s://%s' % (protocol, address) proxies[protocol] = address else: diff --git a/lib-python/3/uu.py b/lib-python/3/uu.py index 9b1e5e6072..9f1f37f1a6 100755 --- a/lib-python/3/uu.py +++ b/lib-python/3/uu.py @@ -73,6 +73,13 @@ def encode(in_file, out_file, name=None, mode=None, *, backtick=False): name = '-' if mode is None: mode = 0o666 + + # + # Remove newline chars from name + # + name = name.replace('\n','\\n') + name = name.replace('\r','\\r') + # # Write the data # diff --git a/lib-python/3/uuid.py b/lib-python/3/uuid.py index 26faa1accd..b1abfe315d 100644 --- a/lib-python/3/uuid.py +++ b/lib-python/3/uuid.py @@ -205,12 +205,14 @@ class UUID: self.__dict__['is_safe'] = is_safe def __getstate__(self): - state = self.__dict__ + state = self.__dict__.copy() if self.is_safe != SafeUUID.unknown: # is_safe is a SafeUUID instance. Return just its value, so that # it can be un-pickled in older Python versions without SafeUUID. - state = state.copy() state['is_safe'] = self.is_safe.value + else: + # omit is_safe when it is "unknown" + del state['is_safe'] return state def __setstate__(self, state): diff --git a/lib-python/3/venv/scripts/common/activate b/lib-python/3/venv/scripts/common/activate index fff0765af5..b9d498fb2e 100644 --- a/lib-python/3/venv/scripts/common/activate +++ b/lib-python/3/venv/scripts/common/activate @@ -28,7 +28,7 @@ deactivate () { fi unset VIRTUAL_ENV - if [ ! "$1" = "nondestructive" ] ; then + if [ ! "${1:-}" = "nondestructive" ] ; then # Self destruct! unset -f deactivate fi diff --git a/lib-python/3/venv/scripts/nt/activate.bat b/lib-python/3/venv/scripts/nt/activate.bat index 14ffb2cf78..f61413e232 100644 --- a/lib-python/3/venv/scripts/nt/activate.bat +++ b/lib-python/3/venv/scripts/nt/activate.bat @@ -2,44 +2,32 @@ rem This file is UTF-8 encoded, so we need to update the current code page while executing it
for /f "tokens=2 delims=:." %%a in ('"%SystemRoot%\System32\chcp.com"') do (
- set "_OLD_CODEPAGE=%%a"
+ set _OLD_CODEPAGE=%%a
)
if defined _OLD_CODEPAGE (
"%SystemRoot%\System32\chcp.com" 65001 > nul
)
-set "VIRTUAL_ENV=__VENV_DIR__"
+set VIRTUAL_ENV=__VENV_DIR__
-if not defined PROMPT (
- set "PROMPT=$P$G"
-)
-
-if defined _OLD_VIRTUAL_PROMPT (
- set "PROMPT=%_OLD_VIRTUAL_PROMPT%"
-)
+if not defined PROMPT set PROMPT=$P$G
-if defined _OLD_VIRTUAL_PYTHONHOME (
- set "PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%"
-)
+if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT%
+if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME%
-set "_OLD_VIRTUAL_PROMPT=%PROMPT%"
-set "PROMPT=__VENV_PROMPT__%PROMPT%"
+set _OLD_VIRTUAL_PROMPT=%PROMPT%
+set PROMPT=__VENV_PROMPT__%PROMPT%
-if defined PYTHONHOME (
- set "_OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%"
- set PYTHONHOME=
-)
+if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME%
+set PYTHONHOME=
-if defined _OLD_VIRTUAL_PATH (
- set "PATH=%_OLD_VIRTUAL_PATH%"
-) else (
- set "_OLD_VIRTUAL_PATH=%PATH%"
-)
+if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH%
+if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH%
-set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%"
+set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%
:END
if defined _OLD_CODEPAGE (
"%SystemRoot%\System32\chcp.com" %_OLD_CODEPAGE% > nul
- set "_OLD_CODEPAGE="
+ set _OLD_CODEPAGE=
)
diff --git a/lib-python/3/warnings.py b/lib-python/3/warnings.py index 9064f56827..3e1715c4b4 100644 --- a/lib-python/3/warnings.py +++ b/lib-python/3/warnings.py @@ -211,7 +211,6 @@ def _processoptions(args): # Helper for _processoptions() def _setoption(arg): - import re parts = arg.split(':') if len(parts) > 5: raise _OptionError("too many fields (max 5): %r" % (arg,)) @@ -220,11 +219,13 @@ def _setoption(arg): action, message, category, module, lineno = [s.strip() for s in parts] action = _getaction(action) - message = re.escape(message) category = _getcategory(category) - module = re.escape(module) + if message or module: + import re + if message: + message = re.escape(message) if module: - module = module + '$' + module = re.escape(module) + r'\Z' if lineno: try: lineno = int(lineno) @@ -248,26 +249,21 @@ def _getaction(action): # Helper for _setoption() def _getcategory(category): - import re if not category: return Warning - if re.match("^[a-zA-Z0-9_]+$", category): - try: - cat = eval(category) - except NameError: - raise _OptionError("unknown warning category: %r" % (category,)) from None + if '.' not in category: + import builtins as m + klass = category else: - i = category.rfind(".") - module = category[:i] - klass = category[i+1:] + module, _, klass = category.rpartition('.') try: m = __import__(module, None, None, [klass]) except ImportError: raise _OptionError("invalid module name: %r" % (module,)) from None - try: - cat = getattr(m, klass) - except AttributeError: - raise _OptionError("unknown warning category: %r" % (category,)) from None + try: + cat = getattr(m, klass) + except AttributeError: + raise _OptionError("unknown warning category: %r" % (category,)) from None if not issubclass(cat, Warning): raise _OptionError("invalid warning category: %r" % (category,)) return cat diff --git a/lib-python/3/wave.py b/lib-python/3/wave.py index f155879a9a..823f091dea 100644 --- a/lib-python/3/wave.py +++ b/lib-python/3/wave.py @@ -53,7 +53,7 @@ This returns an instance of a class with the following public methods: -- set all parameters at once tell() -- return current position in output file writeframesraw(data) - -- write audio frames without pathing up the + -- write audio frames without patching up the file header writeframes(data) -- write audio frames and patch up the file header diff --git a/lib-python/3/weakref.py b/lib-python/3/weakref.py index 59b3aa5621..461c997a12 100644 --- a/lib-python/3/weakref.py +++ b/lib-python/3/weakref.py @@ -114,12 +114,12 @@ class WeakValueDictionary(_collections_abc.MutableMapping): else: # Atomic removal is necessary since this function # can be called asynchronously by the GC - _atomic_removal(d, wr.key) + _atomic_removal(self.data, wr.key) self._remove = remove # A list of keys to be removed self._pending_removals = [] self._iterating = set() - self.data = d = {} + self.data = {} self.update(*args, **kw) def _commit_removals(self): diff --git a/lib-python/3/webbrowser.py b/lib-python/3/webbrowser.py index 82bff835fd..b04ec7b65a 100755 --- a/lib-python/3/webbrowser.py +++ b/lib-python/3/webbrowser.py @@ -69,6 +69,14 @@ def get(using=None): # instead of "from webbrowser import *". def open(url, new=0, autoraise=True): + """Display url using the default browser. + + If possible, open url in a location determined by new. + - 0: the same browser window (the default). + - 1: a new browser window. + - 2: a new browser page ("tab"). + If possible, autoraise raises the window (the default) or not. + """ if _tryorder is None: with _lock: if _tryorder is None: @@ -80,14 +88,22 @@ def open(url, new=0, autoraise=True): return False def open_new(url): + """Open url in a new window of the default browser. + + If not possible, then open url in the only browser window. + """ return open(url, 1) def open_new_tab(url): + """Open url in a new page ("tab") of the default browser. + + If not possible, then the behavior becomes equivalent to open_new(). + """ return open(url, 2) def _synthesize(browser, *, preferred=False): - """Attempt to synthesize a controller base on existing controllers. + """Attempt to synthesize a controller based on existing controllers. This is useful to create a controller when a user specifies a path to an entry in the BROWSER environment variable -- we can copy a general @@ -524,7 +540,7 @@ def register_standard_browsers(): register(browser, None, BackgroundBrowser(browser)) else: # Prefer X browsers if present - if os.environ.get("DISPLAY"): + if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): try: cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) diff --git a/lib-python/3/xmlrpc/server.py b/lib-python/3/xmlrpc/server.py index f1c467eb1b..32aba4df4c 100644 --- a/lib-python/3/xmlrpc/server.py +++ b/lib-python/3/xmlrpc/server.py @@ -108,6 +108,7 @@ from xmlrpc.client import Fault, dumps, loads, gzip_encode, gzip_decode from http.server import BaseHTTPRequestHandler from functools import partial from inspect import signature +import html import http.server import socketserver import sys @@ -894,7 +895,7 @@ class XMLRPCDocGenerator: methods ) - return documenter.page(self.server_title, documentation) + return documenter.page(html.escape(self.server_title), documentation) class DocXMLRPCRequestHandler(SimpleXMLRPCRequestHandler): """XML-RPC and documentation request handler class. diff --git a/lib-python/3/zipfile.py b/lib-python/3/zipfile.py index 955d91fd0a..63748d371f 100644 --- a/lib-python/3/zipfile.py +++ b/lib-python/3/zipfile.py @@ -477,14 +477,26 @@ class ZipInfo (object): # ZIP64 extension (large files and/or large archives) if self.file_size in (0xffffffffffffffff, 0xffffffff): + if len(counts) <= idx: + raise BadZipFile( + "Corrupt zip64 extra field. File size not found." + ) self.file_size = counts[idx] idx += 1 if self.compress_size == 0xFFFFFFFF: + if len(counts) <= idx: + raise BadZipFile( + "Corrupt zip64 extra field. Compress size not found." + ) self.compress_size = counts[idx] idx += 1 if self.header_offset == 0xffffffff: + if len(counts) <= idx: + raise BadZipFile( + "Corrupt zip64 extra field. Header offset not found." + ) old = self.header_offset self.header_offset = counts[idx] idx+=1 @@ -784,10 +796,10 @@ class ZipExtFile(io.BufferedIOBase): # Chunk size to read during seek MAX_SEEK_READ = 1 << 24 - def __init__(self, fileobj, mode, zipinfo, decrypter=None, + def __init__(self, fileobj, mode, zipinfo, pwd=None, close_fileobj=False): self._fileobj = fileobj - self._decrypter = decrypter + self._pwd = pwd self._close_fileobj = close_fileobj self._compress_type = zipinfo.compress_type @@ -802,11 +814,6 @@ class ZipExtFile(io.BufferedIOBase): self.newlines = None - # Adjust read size for encrypted files since the first 12 bytes - # are for the encryption/password information. - if self._decrypter is not None: - self._compress_left -= 12 - self.mode = mode self.name = zipinfo.filename @@ -827,6 +834,30 @@ class ZipExtFile(io.BufferedIOBase): except AttributeError: pass + self._decrypter = None + if pwd: + if zipinfo.flag_bits & 0x8: + # compare against the file type from extended local headers + check_byte = (zipinfo._raw_time >> 8) & 0xff + else: + # compare against the CRC otherwise + check_byte = (zipinfo.CRC >> 24) & 0xff + h = self._init_decrypter() + if h != check_byte: + raise RuntimeError("Bad password for file %r" % zipinfo.orig_filename) + + + def _init_decrypter(self): + self._decrypter = _ZipDecrypter(self._pwd) + # The first 12 bytes in the cypher stream is an encryption header + # used to strengthen the algorithm. The first 11 bytes are + # completely random, while the 12th contains the MSB of the CRC, + # or the MSB of the file time depending on the header type + # and is used to check the correctness of the password. + header = self._fileobj.read(12) + self._compress_left -= 12 + return self._decrypter(header)[11] + def __repr__(self): result = ['<%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)] @@ -1053,6 +1084,8 @@ class ZipExtFile(io.BufferedIOBase): self._decompressor = _get_decompressor(self._compress_type) self._eof = False read_offset = new_pos + if self._decrypter is not None: + self._init_decrypter() while read_offset > 0: read_len = min(self.MAX_SEEK_READ, read_offset) @@ -1515,32 +1548,16 @@ class ZipFile: # check for encrypted flag & handle password is_encrypted = zinfo.flag_bits & 0x1 - zd = None if is_encrypted: if not pwd: pwd = self.pwd if not pwd: raise RuntimeError("File %r is encrypted, password " "required for extraction" % name) + else: + pwd = None - zd = _ZipDecrypter(pwd) - # The first 12 bytes in the cypher stream is an encryption header - # used to strengthen the algorithm. The first 11 bytes are - # completely random, while the 12th contains the MSB of the CRC, - # or the MSB of the file time depending on the header type - # and is used to check the correctness of the password. - header = zef_file.read(12) - h = zd(header[0:12]) - if zinfo.flag_bits & 0x8: - # compare against the file type from extended local headers - check_byte = (zinfo._raw_time >> 8) & 0xff - else: - # compare against the CRC otherwise - check_byte = (zinfo.CRC >> 24) & 0xff - if h[11] != check_byte: - raise RuntimeError("Bad password for file %r" % name) - - return ZipExtFile(zef_file, mode, zinfo, zd, True) + return ZipExtFile(zef_file, mode, zinfo, pwd, True) except: zef_file.close() raise diff --git a/lib_pypy/_ctypes_test.c b/lib_pypy/_ctypes_test.c index 0152945ca1..d08a011e94 100644 --- a/lib_pypy/_ctypes_test.c +++ b/lib_pypy/_ctypes_test.c @@ -74,6 +74,179 @@ _testfunc_reg_struct_update_value(TestReg in) ((volatile TestReg *)&in)->second = 0x0badf00d; } +/* + * See bpo-22273. Structs containing arrays should work on Linux 64-bit. + */ + +typedef struct { + unsigned char data[16]; +} Test2; + +EXPORT(int) +_testfunc_array_in_struct1(Test2 in) +{ + int result = 0; + + for (unsigned i = 0; i < 16; i++) + result += in.data[i]; + /* As the structure is passed by value, changes to it shouldn't be + * reflected in the caller. + */ + memset(in.data, 0, sizeof(in.data)); + return result; +} + +typedef struct { + double data[2]; +} Test3; + +typedef struct { + float data[2]; + float more_data[2]; +} Test3B; + +EXPORT(double) +_testfunc_array_in_struct2(Test3 in) +{ + double result = 0; + + for (unsigned i = 0; i < 2; i++) + result += in.data[i]; + /* As the structure is passed by value, changes to it shouldn't be + * reflected in the caller. + */ + memset(in.data, 0, sizeof(in.data)); + return result; +} + +EXPORT(double) +_testfunc_array_in_struct2a(Test3B in) +{ + double result = 0; + + for (unsigned i = 0; i < 2; i++) + result += in.data[i]; + for (unsigned i = 0; i < 2; i++) + result += in.more_data[i]; + /* As the structure is passed by value, changes to it shouldn't be + * reflected in the caller. + */ + memset(in.data, 0, sizeof(in.data)); + return result; +} + +typedef union { + long a_long; + struct { + int an_int; + int another_int; + } a_struct; +} Test4; + +typedef struct { + int an_int; + struct { + int an_int; + Test4 a_union; + } nested; + int another_int; +} Test5; + +EXPORT(long) +_testfunc_union_by_value1(Test4 in) { + long result = in.a_long + in.a_struct.an_int + in.a_struct.another_int; + + /* As the union/struct are passed by value, changes to them shouldn't be + * reflected in the caller. + */ + memset(&in, 0, sizeof(in)); + return result; +} + +EXPORT(long) +_testfunc_union_by_value2(Test5 in) { + long result = in.an_int + in.nested.an_int; + + /* As the union/struct are passed by value, changes to them shouldn't be + * reflected in the caller. + */ + memset(&in, 0, sizeof(in)); + return result; +} + +EXPORT(long) +_testfunc_union_by_reference1(Test4 *in) { + long result = in->a_long; + + memset(in, 0, sizeof(Test4)); + return result; +} + +EXPORT(long) +_testfunc_union_by_reference2(Test4 *in) { + long result = in->a_struct.an_int + in->a_struct.another_int; + + memset(in, 0, sizeof(Test4)); + return result; +} + +EXPORT(long) +_testfunc_union_by_reference3(Test5 *in) { + long result = in->an_int + in->nested.an_int + in->another_int; + + memset(in, 0, sizeof(Test5)); + return result; +} + +typedef struct { + signed int A: 1, B:2, C:3, D:2; +} Test6; + +EXPORT(long) +_testfunc_bitfield_by_value1(Test6 in) { + long result = in.A + in.B + in.C + in.D; + + /* As the struct is passed by value, changes to it shouldn't be + * reflected in the caller. + */ + memset(&in, 0, sizeof(in)); + return result; +} + +EXPORT(long) +_testfunc_bitfield_by_reference1(Test6 *in) { + long result = in->A + in->B + in->C + in->D; + + memset(in, 0, sizeof(Test6)); + return result; +} + +typedef struct { + unsigned int A: 1, B:2, C:3, D:2; +} Test7; + +EXPORT(long) +_testfunc_bitfield_by_reference2(Test7 *in) { + long result = in->A + in->B + in->C + in->D; + + memset(in, 0, sizeof(Test7)); + return result; +} + +typedef union { + signed int A: 1, B:2, C:3, D:2; +} Test8; + +EXPORT(long) +_testfunc_bitfield_by_value2(Test8 in) { + long result = in.A + in.B + in.C + in.D; + + /* As the struct is passed by value, changes to it shouldn't be + * reflected in the caller. + */ + memset(&in, 0, sizeof(in)); + return result; +} EXPORT(void)testfunc_array(int values[4]) { diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c index 64b7b69049..6996d22684 100644 --- a/lib_pypy/_testcapimodule.c +++ b/lib_pypy/_testcapimodule.c @@ -4653,6 +4653,19 @@ get_main_config(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject* +pynumber_tobase(PyObject *module, PyObject *args) +{ + PyObject *obj; + int base; + if (!PyArg_ParseTuple(args, "Oi:pynumber_tobase", + &obj, &base)) { + return NULL; + } + return PyNumber_ToBase(obj, base); +} + + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, {"raise_memoryerror", (PyCFunction)raise_memoryerror, METH_NOARGS}, @@ -4888,6 +4901,7 @@ static PyMethodDef TestMethods[] = { {"get_global_config", get_global_config, METH_NOARGS}, {"get_core_config", get_core_config, METH_NOARGS}, {"get_main_config", get_main_config, METH_NOARGS}, + {"pynumber_tobase", pynumber_tobase, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; @@ -5444,11 +5458,14 @@ PyInit__testcapi(void) PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type); PyModule_AddIntConstant(m, "the_number_three", 3); + PyObject *v; #ifdef WITH_PYMALLOC - PyModule_AddObject(m, "WITH_PYMALLOC", Py_True); + v = Py_True; #else - PyModule_AddObject(m, "WITH_PYMALLOC", Py_False); + v = Py_False; #endif + Py_INCREF(v); + PyModule_AddObject(m, "WITH_PYMALLOC", v); TestError = PyErr_NewException("_testcapi.error", NULL, NULL); Py_INCREF(TestError); |
