diff --git a/CHANGES.txt b/CHANGES.txt index b9f044a5d8..486a795c26 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -172,6 +172,9 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER builds. Also add a simple filesystem-based locking protocol to try to avoid the problem occuring. - Update the first two chapters on building with SCons in the User Guide. + - Some cleanup to the Util package, including renaming SCons.Util.types + to SCons.Util.sctypes to avoid any possible confusion with the + Python stdlib types module. - TeX tests: skip tests that use makeindex or epstopdf not installed, or if `kpsewhich glossaries.sty` fails. diff --git a/RELEASE.txt b/RELEASE.txt index 258d1c5573..73a0b3c9c8 100644 --- a/RELEASE.txt +++ b/RELEASE.txt @@ -72,6 +72,12 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY The "--warn=missing-sconscript" commandline option is no longer available as the warning was part of the transitional phase. - Add missing directories to searched paths for mingw installs +- SCons.Util.types renamed to to SCons.Util.sctypes to avoid any possible + confusion with the Python stdlib "types" module. Note that it was briefly + (for 4.5.x only) possible to import directly from SCons.Util.types, + although the preferred usage remains to import from SCons.Util only. + Any code that did the direct import will have to change to import from + SCons.Util.sctypes. FIXES ----- diff --git a/SCons/Errors.py b/SCons/Errors.py index b40ba0e744..012d1c6779 100644 --- a/SCons/Errors.py +++ b/SCons/Errors.py @@ -27,7 +27,8 @@ """ import shutil -import SCons.Util + +from SCons.Util.sctypes import to_String, is_String # Note that not all Errors are defined here, some are at the point of use @@ -77,7 +78,7 @@ def __init__(self, # py3: errstr should be string and not bytes. - self.errstr = SCons.Util.to_String(errstr) + self.errstr = to_String(errstr) self.status = status self.exitstatus = exitstatus self.filename = filename @@ -189,7 +190,7 @@ def convert_to_BuildError(status, exc_info=None): status=2, exitstatus=2, exc_info=exc_info) - elif SCons.Util.is_String(status): + elif is_String(status): buildError = BuildError( errstr=status, status=2, diff --git a/SCons/Node/FS.py b/SCons/Node/FS.py index 3162b21b35..a5282e6aa5 100644 --- a/SCons/Node/FS.py +++ b/SCons/Node/FS.py @@ -1204,7 +1204,7 @@ def islink(self, path) -> bool: if hasattr(os, 'readlink'): - def readlink(self, file): + def readlink(self, file) -> str: return os.readlink(file) else: diff --git a/SCons/UtilTests.py b/SCons/Util/UtilTests.py similarity index 94% rename from SCons/UtilTests.py rename to SCons/Util/UtilTests.py index 54cb658c3f..ff32bab8d2 100644 --- a/SCons/UtilTests.py +++ b/SCons/Util/UtilTests.py @@ -22,14 +22,15 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import functools +import hashlib import io import os import sys import unittest import unittest.mock -import hashlib import warnings from collections import UserDict, UserList, UserString, namedtuple +from typing import Union import TestCmd @@ -87,8 +88,8 @@ class OutBuffer: def __init__(self) -> None: self.buffer = "" - def write(self, str) -> None: - self.buffer = self.buffer + str + def write(self, text: str) -> None: + self.buffer = self.buffer + text class dictifyTestCase(unittest.TestCase): @@ -155,10 +156,10 @@ def tree_case_1(self): bar_o = self.Node("bar.o", [bar_c]) foo_c = self.Node("foo.c", [stdio_h]) foo_o = self.Node("foo.o", [foo_c]) - foo = self.Node("foo", [foo_o, bar_o]) + prog = self.Node("prog", [foo_o, bar_o]) expect = """\ -+-foo ++-prog +-foo.o | +-foo.c | +-stdio.h @@ -172,7 +173,7 @@ def tree_case_1(self): lines = ['[E BSPACN ]' + l for l in lines] withtags = '\n'.join(lines) + '\n' - return foo, expect, withtags + return prog, expect, withtags def tree_case_2(self, prune: int=1): """Fixture for the render_tree() and print_tree() tests.""" @@ -471,29 +472,29 @@ def test_Proxy(self) -> None: """Test generic Proxy class.""" class Subject: - def foo(self) -> int: + def meth(self) -> int: return 1 - def bar(self) -> int: + def other(self) -> int: return 2 s = Subject() - s.baz = 3 + s.attr = 3 class ProxyTest(Proxy): - def bar(self) -> int: + def other(self) -> int: return 4 p = ProxyTest(s) - assert p.foo() == 1, p.foo() - assert p.bar() == 4, p.bar() - assert p.baz == 3, p.baz + assert p.meth() == 1, p.meth() + assert p.other() == 4, p.other() + assert p.attr == 3, p.attr - p.baz = 5 - s.baz = 6 + p.attr = 5 + s.attr = 6 - assert p.baz == 5, p.baz + assert p.attr == 5, p.attr assert p.get() == s, p.get() def test_display(self) -> None: @@ -519,10 +520,10 @@ def test_get_native_path(self) -> None: os.close(f) data = '1234567890 ' + filename try: - with open(filename, 'w') as f: - f.write(data) - with open(get_native_path(filename), 'r') as f: - assert f.read() == data + with open(filename, 'w') as file: + file.write(data) + with open(get_native_path(filename), 'r') as native: + assert native.read() == data finally: try: os.unlink(filename) @@ -531,8 +532,8 @@ def test_get_native_path(self) -> None: def test_PrependPath(self) -> None: """Test prepending to a path""" - p1 = r'C:\dir\num\one;C:\dir\num\two' - p2 = r'C:\mydir\num\one;C:\mydir\num\two' + p1: Union[list, str] = r'C:\dir\num\one;C:\dir\num\two' + p2: Union[list, str] = r'C:\mydir\num\one;C:\mydir\num\two' # have to include the pathsep here so that the test will work on UNIX too. p1 = PrependPath(p1, r'C:\dir\num\two', sep=';') p1 = PrependPath(p1, r'C:\dir\num\three', sep=';') @@ -543,14 +544,14 @@ def test_PrependPath(self) -> None: assert p2 == r'C:\mydir\num\one;C:\mydir\num\three;C:\mydir\num\two', p2 # check (only) first one is kept if there are dupes in new - p3 = r'C:\dir\num\one' + p3: Union[list, str] = r'C:\dir\num\one' p3 = PrependPath(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\two', sep=';') assert p3 == r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\one', p3 def test_AppendPath(self) -> None: """Test appending to a path.""" - p1 = r'C:\dir\num\one;C:\dir\num\two' - p2 = r'C:\mydir\num\one;C:\mydir\num\two' + p1: Union[list, str] = r'C:\dir\num\one;C:\dir\num\two' + p2: Union[list, str] = r'C:\mydir\num\one;C:\mydir\num\two' # have to include the pathsep here so that the test will work on UNIX too. p1 = AppendPath(p1, r'C:\dir\num\two', sep=';') p1 = AppendPath(p1, r'C:\dir\num\three', sep=';') @@ -561,23 +562,23 @@ def test_AppendPath(self) -> None: assert p2 == r'C:\mydir\num\two;C:\mydir\num\three;C:\mydir\num\one', p2 # check (only) last one is kept if there are dupes in new - p3 = r'C:\dir\num\one' + p3: Union[list, str] = r'C:\dir\num\one' p3 = AppendPath(p3, r'C:\dir\num\two;C:\dir\num\three;C:\dir\num\two', sep=';') assert p3 == r'C:\dir\num\one;C:\dir\num\three;C:\dir\num\two', p3 def test_PrependPathPreserveOld(self) -> None: """Test prepending to a path while preserving old paths""" - p1 = r'C:\dir\num\one;C:\dir\num\two' + p1: Union[list, str] = r'C:\dir\num\one;C:\dir\num\two' # have to include the pathsep here so that the test will work on UNIX too. - p1 = PrependPath(p1, r'C:\dir\num\two', sep=';', delete_existing=0) + p1 = PrependPath(p1, r'C:\dir\num\two', sep=';', delete_existing=False) p1 = PrependPath(p1, r'C:\dir\num\three', sep=';') assert p1 == r'C:\dir\num\three;C:\dir\num\one;C:\dir\num\two', p1 def test_AppendPathPreserveOld(self) -> None: """Test appending to a path while preserving old paths""" - p1 = r'C:\dir\num\one;C:\dir\num\two' + p1: Union[list, str] = r'C:\dir\num\one;C:\dir\num\two' # have to include the pathsep here so that the test will work on UNIX too. - p1 = AppendPath(p1, r'C:\dir\num\one', sep=';', delete_existing=0) + p1 = AppendPath(p1, r'C:\dir\num\one', sep=';', delete_existing=False) p1 = AppendPath(p1, r'C:\dir\num\three', sep=';') assert p1 == r'C:\dir\num\one;C:\dir\num\two;C:\dir\num\three', p1 @@ -930,14 +931,14 @@ def __init__(self, *args, **kwargs) -> None: all_throw = unittest.mock.Mock(**{'md5.side_effect': ValueError, 'sha1.side_effect': ValueError, 'sha256.side_effect': ValueError}) self.all_throw=all_throw - + no_algorithms = unittest.mock.Mock() del no_algorithms.md5 del no_algorithms.sha1 del no_algorithms.sha256 del no_algorithms.nonexist self.no_algorithms=no_algorithms - + unsupported_algorithm = unittest.mock.Mock(unsupported=self.fake_sha256) del unsupported_algorithm.md5 del unsupported_algorithm.sha1 @@ -976,7 +977,7 @@ def test_usedforsecurity_flag_behavior(self) -> None: self.sys_v4_8: (False, 'md5'), }.items(): assert _attempt_init_of_python_3_9_hash_object(self.fake_md5, version) == expected - + def test_automatic_default_to_md5(self) -> None: """Test automatic default to md5 even if sha1 available""" for version, expected in { @@ -1009,7 +1010,7 @@ def test_automatic_default_to_sha1(self) -> None: _set_allowed_viable_default_hashes(self.sha1Default, version) set_hash_format(None, self.sha1Default, version) assert _get_hash_object(None, self.sha1Default, version) == expected - + def test_no_available_algorithms(self) -> None: """expect exceptions on no available algorithms or when all algorithms throw""" self.assertRaises(SCons.Errors.SConsEnvironmentError, _set_allowed_viable_default_hashes, self.no_algorithms) @@ -1022,7 +1023,7 @@ def test_bad_algorithm_set_attempt(self) -> None: # nonexistant hash algorithm, not supported by SCons _set_allowed_viable_default_hashes(self.md5Available) self.assertRaises(SCons.Errors.UserError, set_hash_format, 'blah blah blah', hashlib_used=self.no_algorithms) - + # md5 is default-allowed, but in this case throws when we attempt to use it _set_allowed_viable_default_hashes(self.md5Available) self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) @@ -1030,7 +1031,7 @@ def test_bad_algorithm_set_attempt(self) -> None: # user attempts to use an algorithm that isn't supported by their current system but is supported by SCons _set_allowed_viable_default_hashes(self.sha1Default) self.assertRaises(SCons.Errors.UserError, set_hash_format, 'md5', hashlib_used=self.all_throw) - + # user attempts to use an algorithm that is supported by their current system but isn't supported by SCons _set_allowed_viable_default_hashes(self.sha1Default) self.assertRaises(SCons.Errors.UserError, set_hash_format, 'unsupported', hashlib_used=self.unsupported_algorithm) @@ -1048,16 +1049,16 @@ def test_simple_attributes(self) -> None: class TestClass: def __init__(self, name, child=None) -> None: self.child = child - self.bar = name + self.name = name t1 = TestClass('t1', TestClass('t1child')) t2 = TestClass('t2', TestClass('t2child')) t3 = TestClass('t3') nl = NodeList([t1, t2, t3]) - assert nl.bar == ['t1', 't2', 't3'], nl.bar - assert nl[0:2].child.bar == ['t1child', 't2child'], \ - nl[0:2].child.bar + assert nl.name == ['t1', 't2', 't3'], nl.name + assert nl[0:2].child.name == ['t1child', 't2child'], \ + nl[0:2].child.name def test_callable_attributes(self) -> None: """Test callable attributes of a NodeList class""" @@ -1065,10 +1066,10 @@ def test_callable_attributes(self) -> None: class TestClass: def __init__(self, name, child=None) -> None: self.child = child - self.bar = name + self.name = name - def foo(self): - return self.bar + "foo" + def meth(self): + return self.name + "foo" def getself(self): return self @@ -1078,13 +1079,13 @@ def getself(self): t3 = TestClass('t3') nl = NodeList([t1, t2, t3]) - assert nl.foo() == ['t1foo', 't2foo', 't3foo'], nl.foo() - assert nl.bar == ['t1', 't2', 't3'], nl.bar - assert nl.getself().bar == ['t1', 't2', 't3'], nl.getself().bar - assert nl[0:2].child.foo() == ['t1childfoo', 't2childfoo'], \ - nl[0:2].child.foo() - assert nl[0:2].child.bar == ['t1child', 't2child'], \ - nl[0:2].child.bar + assert nl.meth() == ['t1foo', 't2foo', 't3foo'], nl.meth() + assert nl.name == ['t1', 't2', 't3'], nl.name + assert nl.getself().name == ['t1', 't2', 't3'], nl.getself().name + assert nl[0:2].child.meth() == ['t1childfoo', 't2childfoo'], \ + nl[0:2].child.meth() + assert nl[0:2].child.name == ['t1child', 't2child'], \ + nl[0:2].child.name def test_null(self): """Test a null NodeList""" @@ -1133,7 +1134,7 @@ def __exit__(self, *args) -> None: class get_env_boolTestCase(unittest.TestCase): def test_missing(self) -> None: - env = dict() + env = {} var = get_env_bool(env, 'FOO') assert var is False, "var should be False, not %s" % repr(var) env = {'FOO': '1'} @@ -1141,22 +1142,22 @@ def test_missing(self) -> None: assert var is False, "var should be False, not %s" % repr(var) def test_true(self) -> None: - for foo in ['TRUE', 'True', 'true', + for arg in ['TRUE', 'True', 'true', 'YES', 'Yes', 'yes', 'Y', 'y', 'ON', 'On', 'on', '1', '20', '-1']: - env = {'FOO': foo} + env = {'FOO': arg} var = get_env_bool(env, 'FOO') assert var is True, 'var should be True, not %s' % repr(var) def test_false(self) -> None: - for foo in ['FALSE', 'False', 'false', + for arg in ['FALSE', 'False', 'false', 'NO', 'No', 'no', 'N', 'n', 'OFF', 'Off', 'off', '0']: - env = {'FOO': foo} + env = {'FOO': arg} var = get_env_bool(env, 'FOO', True) assert var is False, 'var should be True, not %s' % repr(var) @@ -1170,7 +1171,7 @@ def test_default(self) -> None: class get_os_env_boolTestCase(unittest.TestCase): def test_missing(self) -> None: - with OsEnviron(dict()): + with OsEnviron({}): var = get_os_env_bool('FOO') assert var is False, "var should be False, not %s" % repr(var) with OsEnviron({'FOO': '1'}): @@ -1178,22 +1179,22 @@ def test_missing(self) -> None: assert var is False, "var should be False, not %s" % repr(var) def test_true(self) -> None: - for foo in ['TRUE', 'True', 'true', + for arg in ['TRUE', 'True', 'true', 'YES', 'Yes', 'yes', 'Y', 'y', 'ON', 'On', 'on', '1', '20', '-1']: - with OsEnviron({'FOO': foo}): + with OsEnviron({'FOO': arg}): var = get_os_env_bool('FOO') assert var is True, 'var should be True, not %s' % repr(var) def test_false(self) -> None: - for foo in ['FALSE', 'False', 'false', + for arg in ['FALSE', 'False', 'false', 'NO', 'No', 'no', 'N', 'n', 'OFF', 'Off', 'off', '0']: - with OsEnviron({'FOO': foo}): + with OsEnviron({'FOO': arg}): var = get_os_env_bool('FOO', True) assert var is False, 'var should be True, not %s' % repr(var) diff --git a/SCons/Util/__init__.py b/SCons/Util/__init__.py index 078414ff9c..be2142f034 100644 --- a/SCons/Util/__init__.py +++ b/SCons/Util/__init__.py @@ -21,24 +21,50 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -"""SCons utility functions +""" +SCons utility functions This package contains routines for use by other parts of SCons. +Candidates for inclusion here are routines that do not need other parts +of SCons (other than Util), and have a reasonable chance of being useful +in multiple places, rather then being topical only to one module/package. """ +# Warning: SCons.Util may not be able to import other parts of SCons +# globally without hitting import loops, as various modules import +# SCons.Util themselves. If a top-level import fails, try a local import. +# If local imports work, please annotate them for pylint (and for human +# readers) to know why, with: +# importstuff # pylint: disable=import-outside-toplevel +# +# Be aware that Black will break this if the annotated line is too long - +# which it almost certainly will be. It will split it like this: +# from SCons.Errors import ( +# SConsEnvironmentError, +# ) # pylint: disable=import-outside-toplevel +# That's syntactically valid as far as Python goes, but pylint will not +# recorgnize the annotation comment unless it's on the first line, like: +# from SCons.Errors import ( # pylint: disable=import-outside-toplevel +# SConsEnvironmentError, +# ) +# (issue filed on this upstream, for now just be aware) + import copy import hashlib +import logging import os import re import sys import time -from collections import UserDict, UserList, OrderedDict, deque +from collections import UserDict, UserList, deque from contextlib import suppress from types import MethodType, FunctionType -from typing import Optional, Union +from typing import Optional, Union, Any, List from logging import Formatter -from .types import ( +# Util split into a package. Make sure things that used to work +# when importing just Util itself still work: +from .sctypes import ( DictTypes, ListTypes, SequenceTypes, @@ -83,24 +109,6 @@ ) from .filelock import FileLock, SConsLockFailure - -# Note: the Util package cannot import other parts of SCons globally without -# hitting import loops. Both of these modules import SCons.Util early on, -# and are imported in many other modules: -# --> SCons.Warnings -# --> SCons.Errors -# If you run into places that have to do local imports for this reason, -# annotate them for pylint and for human readers to know why: -# pylint: disable=import-outside-toplevel -# Be aware that Black can break this if the annotated line is too -# long and it wants to split: -# from SCons.Errors import ( -# SConsEnvironmentError, -# ) # pylint: disable=import-outside-toplevel -# That's syntactically valid, but pylint won't recorgnize it with the -# annotation at the end, it would have to be on the first line -# (issues filed upstream, for now just be aware) - PYPY = hasattr(sys, 'pypy_translation_info') # this string will be hashed if a Node refers to a file that doesn't exist @@ -238,20 +246,27 @@ def set_mode(self, mode) -> None: display = DisplayEngine() -# TODO: W0102: Dangerous default value [] as argument (dangerous-default-value) -def render_tree(root, child_func, prune: int=0, margin=[0], visited=None) -> str: +# TODO: check if this could cause problems +# pylint: disable=dangerous-default-value +def render_tree( + root, + child_func, + prune: bool = False, + margin: List[bool] = [False], + visited: Optional[dict] = None, +) -> str: """Render a tree of nodes into an ASCII tree view. Args: root: the root node of the tree child_func: the function called to get the children of a node prune: don't visit the same node twice - margin: the format of the left margin to use for children of `root`. - 1 results in a pipe, and 0 results in no pipe. + margin: the format of the left margin to use for children of *root*. + Each entry represents a column where a true value will display + a vertical bar and a false one a blank. visited: a dictionary of visited nodes in the current branch if - `prune` is 0, or in the whole tree if `prune` is 1. + *prune* is false, or in the whole tree if *prune* is true. """ - rname = str(root) # Initialize 'visited' dict, if required @@ -275,7 +290,7 @@ def render_tree(root, child_func, prune: int=0, margin=[0], visited=None) -> str visited[rname] = True for i, child in enumerate(children): - margin.append(i < len(children)-1) + margin.append(i < len(children) - 1) retval = retval + render_tree(child, child_func, prune, margin, visited) margin.pop() @@ -299,14 +314,15 @@ def IDX(n) -> bool: BOX_HORIZ_DOWN = chr(0x252c) # '┬' -# TODO: W0102: Dangerous default value [] as argument (dangerous-default-value) +# TODO: check if this could cause problems +# pylint: disable=dangerous-default-value def print_tree( root, child_func, - prune: int=0, - showtags: bool=False, - margin=[0], - visited=None, + prune: bool = False, + showtags: int = 0, + margin: List[bool] = [False], + visited: Optional[dict] = None, lastChild: bool = False, singleLineDraw: bool = False, ) -> None: @@ -321,10 +337,13 @@ def print_tree( child_func: the function called to get the children of a node prune: don't visit the same node twice showtags: print status information to the left of each node line + The default is false (value 0). A value of 2 will also print + a legend for the margin tags. margin: the format of the left margin to use for children of *root*. - 1 results in a pipe, and 0 results in no pipe. + Each entry represents a column, where a true value will display + a vertical bar and a false one a blank. visited: a dictionary of visited nodes in the current branch if - *prune* is 0, or in the whole tree if *prune* is 1. + *prune* is false, or in the whole tree if *prune* is true. lastChild: this is the last leaf of a branch singleLineDraw: use line-drawing characters rather than ASCII. """ @@ -405,7 +424,7 @@ def MMM(m): # if this item has children: if children: - margin.append(1) # Initialize margin with 1 for vertical bar. + margin.append(True) # Initialize margin for vertical bar. idx = IDX(showtags) _child = 0 # Initialize this for the first child. for C in children[:-1]: @@ -421,19 +440,19 @@ def MMM(m): singleLineDraw, ) # margins are with space (index 0) because we arrived to the last child. - margin[-1] = 0 + margin[-1] = False # for this call child and nr of children needs to be set 0, to signal the second phase. print_tree(children[-1], child_func, prune, idx, margin, visited, True, singleLineDraw) margin.pop() # destroy the last margin added -def do_flatten( +def do_flatten( # pylint: disable=redefined-outer-name,redefined-builtin sequence, result, isinstance=isinstance, StringTypes=StringTypes, SequenceTypes=SequenceTypes, -) -> None: # pylint: disable=redefined-outer-name,redefined-builtin +) -> None: for item in sequence: if isinstance(item, StringTypes) or not isinstance(item, SequenceTypes): result.append(item) @@ -538,7 +557,7 @@ def semi_deepcopy(obj): class Proxy: """A simple generic Proxy class, forwarding all calls to subject. - This means you can take an object, let's call it `'obj_a`, + This means you can take an object, let's call it `'obj_a``, and wrap it in this Proxy class, with a statement like this:: proxy_obj = Proxy(obj_a) @@ -548,14 +567,15 @@ class Proxy: x = proxy_obj.var1 since the :class:`Proxy` class does not have a :attr:`var1` attribute - (but presumably `objA` does), the request actually is equivalent to saying:: + (but presumably ``obj_a`` does), the request actually is equivalent + to saying:: x = obj_a.var1 Inherit from this class to create a Proxy. With Python 3.5+ this does *not* work transparently - for :class:`Proxy` subclasses that use special .__*__() method names, + for :class:`Proxy` subclasses that use special dunder method names, because those names are now bound to the class, not the individual instances. You now need to know in advance which special method names you want to pass on to the underlying Proxy object, and specifically delegate @@ -759,14 +779,14 @@ def WhereIs(file, path=None, pathext=None, reject=None) -> Optional[str]: f = os.path.join(p, file) if os.path.isfile(f): try: - st = os.stat(f) + mode = os.stat(f).st_mode except OSError: # os.stat() raises OSError, not IOError if the file # doesn't exist, so in this case we let IOError get # raised so as to not mask possibly serious disk or # network issues. continue - if stat.S_IMODE(st[stat.ST_MODE]) & 0o111: + if stat.S_IXUSR & mode: try: reject.index(f) except ValueError: @@ -775,46 +795,61 @@ def WhereIs(file, path=None, pathext=None, reject=None) -> Optional[str]: return None WhereIs.__doc__ = """\ -Return the path to an executable that matches `file`. - -Searches the given `path` for `file`, respecting any filename -extensions `pathext` (on the Windows platform only), and -returns the full path to the matching command. If no -command is found, return ``None``. - -If `path` is not specified, :attr:`os.environ[PATH]` is used. -If `pathext` is not specified, :attr:`os.environ[PATHEXT]` -is used. Will not select any path name or names in the optional -`reject` list. +Return the path to an executable that matches *file*. + +Searches the given *path* for *file*, considering any filename +extensions in *pathext* (on the Windows platform only), and +returns the full path to the matching command of the first match, +or ``None`` if there are no matches. +Will not select any path name or names in the optional +*reject* list. + +If *path* is ``None`` (the default), :attr:`os.environ[PATH]` is used. +On Windows, If *pathext* is ``None`` (the default), +:attr:`os.environ[PATHEXT]` is used. + +The construction environment method of the same name wraps a +call to this function by filling in *path* from the execution +environment if it is ``None`` (and for *pathext* on Windows, +if necessary), so if called from there, this function +will not backfill from :attr:`os.environ`. + +Note: + Finding things in :attr:`os.environ` may answer the question + "does *file* exist on the system", but not the question + "can SCons use that executable", unless the path element that + yields the match is also in the the Execution Environment + (e.g. ``env['ENV']['PATH']``). Since this utility function has no + environment reference, it cannot make that determination. """ if sys.platform == 'cygwin': import subprocess # pylint: disable=import-outside-toplevel - def get_native_path(path) -> str: + def get_native_path(path: str) -> str: cp = subprocess.run(('cygpath', '-w', path), check=False, stdout=subprocess.PIPE) return cp.stdout.decode().replace('\n', '') else: - def get_native_path(path) -> str: + def get_native_path(path: str) -> str: return path get_native_path.__doc__ = """\ Transform an absolute path into a native path for the system. In Cygwin, this converts from a Cygwin path to a Windows path, -without regard to whether `path` refers to an existing file -system object. For other platforms, `path` is unchanged. +without regard to whether *path* refers to an existing file +system object. For other platforms, *path* is unchanged. """ def Split(arg) -> list: """Returns a list of file names or other objects. - If `arg` is a string, it will be split on strings of white-space - characters within the string. If `arg` is already a list, the list - will be returned untouched. If `arg` is any other type of object, - it will be returned as a list containing just the object. + If *arg* is a string, it will be split on whitespace + within the string. If *arg* is already a list, the list + will be returned untouched. If *arg* is any other type of object, + it will be returned in a single-item list. >>> print(Split(" this is a string ")) ['this', 'is', 'a', 'string'] @@ -854,9 +889,11 @@ class CLVar(UserList): >>> c = CLVar("--some --opts and args") >>> print(len(c), repr(c)) 4 ['--some', '--opts', 'and', 'args'] - >>> c += " strips spaces " + >>> c += " strips spaces " >>> print(len(c), repr(c)) 6 ['--some', '--opts', 'and', 'args', 'strips', 'spaces'] + >>> c += [" does not split or strip "] + 7 ['--some', '--opts', 'and', 'args', 'strips', 'spaces', ' does not split or strip '] """ def __init__(self, initlist=None) -> None: @@ -877,10 +914,13 @@ def __str__(self) -> str: return ' '.join([str(d) for d in self.data]) -class Selector(OrderedDict): - """A callable ordered dictionary that maps file suffixes to - dictionary values. We preserve the order in which items are added - so that :func:`get_suffix` calls always return the first suffix added. +class Selector(dict): + """A callable dict for file suffix lookup. + + Often used to associate actions or emitters with file types. + + Depends on insertion order being preserved so that :meth:`get_suffix` + calls always return the first suffix added. """ def __call__(self, env, source, ext=None): if ext is None: @@ -890,7 +930,7 @@ def __call__(self, env, source, ext=None): ext = "" try: return self[ext] - except KeyError: + except KeyError as exc: # Try to perform Environment substitution on the keys of # the dictionary before giving up. s_dict = {} @@ -902,7 +942,7 @@ def __call__(self, env, source, ext=None): # to the same suffix. If one suffix is literal # and a variable suffix contains this literal, # the literal wins and we don't raise an error. - raise KeyError(s_dict[s_k][0], k, s_k) + raise KeyError(s_dict[s_k][0], k, s_k) from exc s_dict[s_k] = (k, v) try: return s_dict[ext][1] @@ -916,13 +956,16 @@ def __call__(self, env, source, ext=None): if sys.platform == 'cygwin': # On Cygwin, os.path.normcase() lies, so just report back the # fact that the underlying Windows OS is case-insensitive. - def case_sensitive_suffixes(s1, s2) -> bool: # pylint: disable=unused-argument + def case_sensitive_suffixes(s1: str, s2: str) -> bool: # pylint: disable=unused-argument return False else: - def case_sensitive_suffixes(s1, s2) -> bool: + def case_sensitive_suffixes(s1: str, s2: str) -> bool: return os.path.normcase(s1) != os.path.normcase(s2) +case_sensitive_suffixes.__doc__ = """\ +Returns whether platform distinguishes case in file suffixes.""" + def adjustixes(fname, pre, suf, ensure_suffix: bool=False) -> str: """Adjust filename prefixes and suffixes as needed. @@ -958,6 +1001,17 @@ def adjustixes(fname, pre, suf, ensure_suffix: bool=False) -> str: def unique(seq): """Return a list of the elements in seq without duplicates, ignoring order. + For best speed, all sequence elements should be hashable. Then + :func:`unique` will usually work in linear time. + + If not possible, the sequence elements should enjoy a total + ordering, and if ``list(s).sort()`` doesn't raise ``TypeError`` + it is assumed that they do enjoy a total ordering. Then + :func:`unique` will usually work in O(N*log2(N)) time. + + If that's not possible either, the sequence elements must support + equality-testing. Then :func:`unique` will usually work in quadratic time. + >>> mylist = unique([1, 2, 3, 1, 2, 3]) >>> print(sorted(mylist)) [1, 2, 3] @@ -967,17 +1021,6 @@ def unique(seq): >>> mylist = unique(([1, 2], [2, 3], [1, 2])) >>> print(sorted(mylist)) [[1, 2], [2, 3]] - - For best speed, all sequence elements should be hashable. Then - unique() will usually work in linear time. - - If not possible, the sequence elements should enjoy a total - ordering, and if list(s).sort() doesn't raise TypeError it's - assumed that they do enjoy a total ordering. Then unique() will - usually work in O(N*log2(N)) time. - - If that's not possible either, the sequence elements must support - equality-testing. Then unique() will usually work in quadratic time. """ if not seq: @@ -1046,7 +1089,7 @@ def logical_lines(physical_lines, joiner=''.join): class LogicalLines: - """ Wrapper class for the logical_lines method. + """Wrapper class for the :func:`logical_lines` function. Allows us to read all "logical" lines at once from a given file object. """ @@ -1061,9 +1104,10 @@ def readlines(self): class UniqueList(UserList): """A list which maintains uniqueness. - Uniquing is lazy: rather than being assured on list changes, it is fixed + Uniquing is lazy: rather than being enforced on list changes, it is fixed up on access by those methods which need to act on a unique list to be - correct. That means things like "in" don't have to eat the uniquing time. + correct. That means things like membership tests don't have to eat the + uniquing time. """ def __init__(self, initlist=None) -> None: super().__init__(initlist) @@ -1221,25 +1265,29 @@ def make_path_relative(path) -> str: return path -def silent_intern(x): - """ +def silent_intern(__string: Any) -> str: + """Intern a string without failing. + Perform :mod:`sys.intern` on the passed argument and return the result. If the input is ineligible for interning the original argument is returned and no exception is thrown. """ try: - return sys.intern(x) + return sys.intern(__string) except TypeError: - return x + return __string def cmp(a, b) -> bool: - """A cmp function because one is no longer available in python3.""" + """A cmp function because one is no longer available in Python3.""" return (a > b) - (a < b) def print_time(): """Hack to return a value from Main if can't import Main.""" + # this specifically violates the rule of Util not depending on other + # parts of SCons in order to work around other import-loop issues. + # # pylint: disable=redefined-outer-name,import-outside-toplevel from SCons.Script.Main import print_time return print_time @@ -1278,20 +1326,31 @@ def wait_for_process_to_die(pid) -> None: # From: https://stackoverflow.com/questions/1741972/how-to-use-different-formatters-with-the-same-logging-handler-in-python class DispatchingFormatter(Formatter): + """Logging formatter which dispatches to various formatters.""" def __init__(self, formatters, default_formatter) -> None: self._formatters = formatters self._default_formatter = default_formatter def format(self, record): - formatter = self._formatters.get(record.name, self._default_formatter) + # Search from record's logger up to its parents: + logger = logging.getLogger(record.name) + while logger: + # Check if suitable formatter for current logger exists: + if logger.name in self._formatters: + formatter = self._formatters[logger.name] + break + logger = logger.parent + else: + # If no formatter found, just use default: + formatter = self._default_formatter return formatter.format(record) def sanitize_shell_env(execution_env: dict) -> dict: """Sanitize all values in *execution_env* - The execution environment (typically comes from (env['ENV']) is + The execution environment (typically comes from ``env['ENV']``) is propagated to the shell, and may need to be cleaned first. Args: diff --git a/SCons/Util/envs.py b/SCons/Util/envs.py index 64e728a8ca..db4d65ab94 100644 --- a/SCons/Util/envs.py +++ b/SCons/Util/envs.py @@ -2,21 +2,26 @@ # # Copyright The SCons Foundation -"""Various SCons utility functions +""" +SCons environment utility functions. Routines for working with environments and construction variables -that don't need the specifics of Environment. +that don't need the specifics of the Environment class. """ import os from types import MethodType, FunctionType -from typing import Union +from typing import Union, Callable, Optional, Any -from .types import is_List, is_Tuple, is_String +from .sctypes import is_List, is_Tuple, is_String def PrependPath( - oldpath, newpath, sep=os.pathsep, delete_existing: bool=True, canonicalize=None + oldpath, + newpath, + sep=os.pathsep, + delete_existing: bool = True, + canonicalize: Optional[Callable] = None, ) -> Union[list, str]: """Prepend *newpath* path elements to *oldpath*. @@ -50,10 +55,10 @@ def PrependPath( if is_String(newpath): newpaths = newpath.split(sep) - elif not is_List(newpath) and not is_Tuple(newpath): - newpaths = [newpath] # might be a Dir - else: + elif is_List(newpath) or is_Tuple(newpath): newpaths = newpath + else: + newpaths = [newpath] # might be a Dir if canonicalize: newpaths = list(map(canonicalize, newpaths)) @@ -102,7 +107,11 @@ def PrependPath( def AppendPath( - oldpath, newpath, sep=os.pathsep, delete_existing: bool=True, canonicalize=None + oldpath, + newpath, + sep=os.pathsep, + delete_existing: bool = True, + canonicalize: Optional[Callable] = None, ) -> Union[list, str]: """Append *newpath* path elements to *oldpath*. @@ -136,10 +145,10 @@ def AppendPath( if is_String(newpath): newpaths = newpath.split(sep) - elif not is_List(newpath) and not is_Tuple(newpath): - newpaths = [newpath] # might be a Dir - else: + elif is_List(newpath) or is_Tuple(newpath): newpaths = newpath + else: + newpaths = [newpath] # might be a Dir if canonicalize: newpaths = list(map(canonicalize, newpaths)) @@ -187,7 +196,7 @@ def AppendPath( return sep.join(paths) -def AddPathIfNotExists(env_dict, key, path, sep=os.pathsep) -> None: +def AddPathIfNotExists(env_dict, key, path, sep: str = os.pathsep) -> None: """Add a path element to a construction variable. `key` is looked up in `env_dict`, and `path` is added to it if it @@ -229,12 +238,12 @@ class MethodWrapper: a new underlying object being copied (without which we wouldn't need to save that info). """ - def __init__(self, obj, method, name=None) -> None: + def __init__(self, obj: Any, method: Callable, name: Optional[str] = None) -> None: if name is None: name = method.__name__ self.object = obj self.method = method - self.name = name + self.name: str = name setattr(self.object, name, self) def __call__(self, *args, **kwargs): @@ -265,7 +274,7 @@ def clone(self, new_object): # is not needed, the remaining bit is now used inline in AddMethod. -def AddMethod(obj, function, name=None) -> None: +def AddMethod(obj, function: Callable, name: Optional[str] = None) -> None: """Add a method to an object. Adds *function* to *obj* if *obj* is a class object. @@ -304,6 +313,8 @@ def AddMethod(obj, function, name=None) -> None: function.__code__, function.__globals__, name, function.__defaults__ ) + method: Union[MethodType, MethodWrapper, Callable] + if hasattr(obj, '__class__') and obj.__class__ is not type: # obj is an instance, so it gets a bound method. if hasattr(obj, "added_methods"): diff --git a/SCons/Util/filelock.py b/SCons/Util/filelock.py index 4b825a6e10..48a2a39db4 100644 --- a/SCons/Util/filelock.py +++ b/SCons/Util/filelock.py @@ -141,7 +141,7 @@ def __exit__(self, exc_type, exc_value, exc_tb) -> None: def __repr__(self) -> str: """Nicer display if someone repr's the lock class.""" return ( - f"FileLock(" + f"{self.__class__.__name__}(" f"file={self.file!r}, " f"timeout={self.timeout!r}, " f"delay={self.delay!r}, " diff --git a/SCons/Util/hashes.py b/SCons/Util/hashes.py index e14da012aa..566897abbe 100644 --- a/SCons/Util/hashes.py +++ b/SCons/Util/hashes.py @@ -2,15 +2,18 @@ # # Copyright The SCons Foundation -"""SCons utility functions +""" +SCons hash utility routines. -Routines for working with hash formats. +Routines for working with content and signature hashes. """ +import functools import hashlib import sys +from typing import Optional, Union -from .types import to_bytes +from .sctypes import to_bytes # Default hash function and format. SCons-internal. @@ -97,9 +100,9 @@ def _set_allowed_viable_default_hashes(hashlib_used, sys_used=sys) -> None: continue if len(ALLOWED_HASH_FORMATS) == 0: - from SCons.Errors import ( + from SCons.Errors import ( # pylint: disable=import-outside-toplevel SConsEnvironmentError, - ) # pylint: disable=import-outside-toplevel + ) # chain the exception thrown with the most recent error from hashlib. raise SConsEnvironmentError( @@ -159,9 +162,9 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): if hash_format: hash_format_lower = hash_format.lower() if hash_format_lower not in ALLOWED_HASH_FORMATS: - from SCons.Errors import ( + from SCons.Errors import ( # pylint: disable=import-outside-toplevel UserError, - ) # pylint: disable=import-outside-toplevel + ) # User can select something not supported by their OS but # normally supported by SCons, example, selecting MD5 in an @@ -207,9 +210,9 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): ) if _HASH_FUNCTION is None: - from SCons.Errors import ( + from SCons.Errors import ( # pylint: disable=import-outside-toplevel UserError, - ) # pylint: disable=import-outside-toplevel + ) raise UserError( f'Hash format "{hash_format_lower}" is not available in your ' @@ -228,9 +231,9 @@ def set_hash_format(hash_format, hashlib_used=hashlib, sys_used=sys): break else: # This is not expected to happen in practice. - from SCons.Errors import ( + from SCons.Errors import ( # pylint: disable=import-outside-toplevel UserError, - ) # pylint: disable=import-outside-toplevel + ) raise UserError( 'Your Python interpreter does not have MD5, SHA1, or SHA256. ' @@ -270,9 +273,9 @@ def _get_hash_object(hash_format, hashlib_used=hashlib, sys_used=sys): """ if hash_format is None: if _HASH_FUNCTION is None: - from SCons.Errors import ( + from SCons.Errors import ( # pylint: disable=import-outside-toplevel UserError, - ) # pylint: disable=import-outside-toplevel + ) raise UserError( 'There is no default hash function. Did you call ' @@ -334,6 +337,10 @@ def hash_file_signature(fname, chunksize: int=65536, hash_format=None): if not blck: break m.update(to_bytes(blck)) + # TODO: can use this when base is Python 3.8+ + # while (blk := f.read(chunksize)) != b'': + # m.update(to_bytes(blk)) + return m.hexdigest() diff --git a/SCons/Util/types.py b/SCons/Util/sctypes.py similarity index 90% rename from SCons/Util/types.py rename to SCons/Util/sctypes.py index b2bc040ac2..53fcc562a5 100644 --- a/SCons/Util/types.py +++ b/SCons/Util/sctypes.py @@ -241,27 +241,24 @@ def to_String_for_signature( # pylint: disable=redefined-outer-name,redefined-b # TODO: Change code when floor is raised to PY36 return pprint.pformat(obj, width=1000000) return to_String_for_subst(obj) - else: - return f() + return f() -def get_env_bool(env, name, default: bool=False) -> bool: +def get_env_bool(env, name: str, default: bool=False) -> bool: """Convert a construction variable to bool. - If the value of *name* in *env* is 'true', 'yes', 'y', 'on' (case - insensitive) or anything convertible to int that yields non-zero then - return ``True``; if 'false', 'no', 'n', 'off' (case insensitive) - or a number that converts to integer zero return ``False``. - Otherwise, return `default`. + If the value of *name* in dict-like object *env* is 'true', 'yes', + 'y', 'on' (case insensitive) or anything convertible to int that + yields non-zero, return ``True``; if 'false', 'no', 'n', 'off' + (case insensitive) or a number that converts to integer zero return + ``False``. Otherwise, or if *name* is not found, return the value + of *default*. Args: - env: construction environment, or any dict-like object - name: name of the variable + env: construction environment, or any dict-like object. + name: name of the variable. default: value to return if *name* not in *env* or cannot - be converted (default: False) - - Returns: - the "truthiness" of `name` + be converted (default: False). """ try: var = env[name] @@ -279,10 +276,10 @@ def get_env_bool(env, name, default: bool=False) -> bool: return default -def get_os_env_bool(name, default: bool=False) -> bool: - """Convert an environment variable to bool. +def get_os_env_bool(name: str, default: bool=False) -> bool: + """Convert an external environment variable to boolean. - Conversion is the same as for :func:`get_env_bool`. + Like :func:`get_env_bool`, but uses :attr:`os.environ` as the lookup dict. """ return get_env_bool(os.environ, name, default) @@ -293,10 +290,10 @@ def get_os_env_bool(name, default: bool=False) -> bool: def get_environment_var(varstr) -> Optional[str]: """Return undecorated construction variable string. - Determine if `varstr` looks like a reference - to a single environment variable, like `"$FOO"` or `"${FOO}"`. - If so, return that variable with no decorations, like `"FOO"`. - If not, return `None`. + Determine if *varstr* looks like a reference + to a single environment variable, like ``"$FOO"`` or ``"${FOO}"``. + If so, return that variable with no decorations, like ``"FOO"``. + If not, return ``None``. """ mo = _get_env_var.match(to_String(varstr)) if mo: diff --git a/SCons/Util/stats.py b/SCons/Util/stats.py index 22135e3ef6..ce820e66c6 100644 --- a/SCons/Util/stats.py +++ b/SCons/Util/stats.py @@ -22,17 +22,26 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -This package provides a way to gather various statistics during a SCons run and dump that info in several formats +SCons statistics routines. -Additionally, it probably makes sense to do stderr/stdout output of those statistics here as well +This package provides a way to gather various statistics during an SCons +run and dump that info in several formats -There are basically two types of stats. -1. Timer (start/stop/time) for specific event. These events can be hierarchical. So you can record the children events of some parent. - Think program compile could contain the total Program builder time, which could include linking, and stripping the executable -2. Counter. Counting the number of events and/or objects created. This would likely only be reported at the end of a given SCons run, - though it might be useful to query during a run. +Additionally, it probably makes sense to do stderr/stdout output of +those statistics here as well + +There are basically two types of stats: +1. Timer (start/stop/time) for specific event. These events can be + hierarchical. So you can record the children events of some parent. + Think program compile could contain the total Program builder time, + which could include linking, and stripping the executable + +2. Counter. Counting the number of events and/or objects created. This + would likely only be reported at the end of a given SCons run, + though it might be useful to query during a run. """ + from abc import ABC import platform @@ -47,13 +56,10 @@ JSON_OUTPUT_FILE = 'scons_stats.json' def add_stat_type(name, stat_object): - """ - Add a statistic type to the global collection - """ + """Add a statistic type to the global collection""" if name in all_stats: raise UserWarning(f'Stat type {name} already exists') - else: - all_stats[name] = stat_object + all_stats[name] = stat_object class Stats(ABC): @@ -108,8 +114,8 @@ def do_print(self): fmt2 = ''.join(pre + [' %7d'] * l + post) labels = self.labels[:l] labels.append(("", "Class")) - self.outfp.write(fmt1 % tuple([x[0] for x in labels])) - self.outfp.write(fmt1 % tuple([x[1] for x in labels])) + self.outfp.write(fmt1 % tuple(x[0] for x in labels)) + self.outfp.write(fmt1 % tuple(x[1] for x in labels)) for k in sorted(self.stats_table.keys()): r = self.stats_table[k][:l] + [k] self.outfp.write(fmt2 % tuple(r)) @@ -160,9 +166,12 @@ def write_scons_stats_file(): """ # Have to import where used to avoid import loop - from SCons.Script import BUILD_TARGETS, COMMAND_LINE_TARGETS, ARGUMENTS, \ - ARGLIST # [import-outside-toplevel] - + from SCons.Script import ( # pylint: disable=import-outside-toplevel + BUILD_TARGETS, + COMMAND_LINE_TARGETS, + ARGUMENTS, + ARGLIST, + ) # print(f"DUMPING JSON FILE: {JSON_OUTPUT_FILE}") json_structure = {} if count_stats.enabled: diff --git a/doc/sphinx/SCons.Util.rst b/doc/sphinx/SCons.Util.rst new file mode 100644 index 0000000000..553db7a1b1 --- /dev/null +++ b/doc/sphinx/SCons.Util.rst @@ -0,0 +1,36 @@ +SCons.Util package +================== + +Submodules +---------- + +.. automodule:: SCons.Util + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: SCons.Util.envs + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: SCons.Util.filelock + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: SCons.Util.hashes + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: SCons.Util.sctypes + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: SCons.Util.stats + :members: + :undoc-members: + :show-inheritance: + diff --git a/doc/sphinx/SCons.rst b/doc/sphinx/SCons.rst index 45e20eeacb..85f58788eb 100644 --- a/doc/sphinx/SCons.rst +++ b/doc/sphinx/SCons.rst @@ -20,6 +20,7 @@ Subpackages SCons.Script SCons.Taskmaster SCons.Tool + SCons.Util SCons.Variables SCons.compat @@ -140,14 +141,6 @@ SCons.Subst module :undoc-members: :show-inheritance: -SCons.Util module ------------------ - -.. automodule:: SCons.Util - :members: - :undoc-members: - :show-inheritance: - SCons.Warnings module --------------------- diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index f8d5f471c7..04bfdc2a55 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -17,7 +17,7 @@ SCons API Documentation The target audience is developers working on SCons itself: what is "Public API" is not clearly deliniated here. The interfaces available for use in SCons configuration scripts, - which have a consistency guarantee, are those documented in the + which have a consistency guarantee, are those documented in the `SCons Reference Manual `_. @@ -33,6 +33,7 @@ SCons API Documentation SCons.Script SCons.Taskmaster SCons.Tool + SCons.Util SCons.Variables