From f2046a4aa3d2c6bef08554a37a43b065a4e73769 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Mon, 7 Oct 2013 18:12:26 -0700 Subject: Start to make a bigger package out of utils.py - moved none_compare functions to util.none_high and util.none_low packages - renamed utils.py to util package --- lib/spack/spack/__init__.py | 2 +- lib/spack/spack/arch.py | 2 +- lib/spack/spack/cmd/test.py | 2 +- lib/spack/spack/compilers/__init__.py | 2 +- lib/spack/spack/directory_layout.py | 2 +- lib/spack/spack/globals.py | 2 +- lib/spack/spack/none_compare.py | 60 ---------- lib/spack/spack/packages/__init__.py | 2 +- lib/spack/spack/url.py | 6 +- lib/spack/spack/util/__init__.py | 209 ++++++++++++++++++++++++++++++++++ lib/spack/spack/util/none_high.py | 46 ++++++++ lib/spack/spack/util/none_low.py | 46 ++++++++ lib/spack/spack/utils.py | 209 ---------------------------------- lib/spack/spack/validate.py | 2 +- lib/spack/spack/version.py | 16 +-- 15 files changed, 320 insertions(+), 288 deletions(-) delete mode 100644 lib/spack/spack/none_compare.py create mode 100644 lib/spack/spack/util/__init__.py create mode 100644 lib/spack/spack/util/none_high.py create mode 100644 lib/spack/spack/util/none_low.py delete mode 100644 lib/spack/spack/utils.py (limited to 'lib') diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 54a3e92a57..3ec0cac441 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -1,5 +1,5 @@ from globals import * -from utils import * +from util import * from error import * from package import Package diff --git a/lib/spack/spack/arch.py b/lib/spack/spack/arch.py index 6610006804..74d2f35c41 100644 --- a/lib/spack/spack/arch.py +++ b/lib/spack/spack/arch.py @@ -4,7 +4,7 @@ import platform as py_platform import spack import error as serr from version import Version -from utils import memoized +from util import memoized class InvalidSysTypeError(serr.SpackError): diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index 93f38bafe8..a246cdf7b6 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -1,7 +1,7 @@ import spack import spack.packages as packages import spack.test -from spack.utils import list_modules +from spack.util import list_modules from spack.colify import colify from pprint import pprint diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 04e084b6ed..3ed2db8423 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -4,7 +4,7 @@ import spack import spack.compilers.gcc -from spack.utils import list_modules, memoized +from spack.util import list_modules, memoized @memoized diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 55abb6d4b5..2fdc55dc3f 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -3,7 +3,7 @@ import re import os import spack.spec as spec -from spack.utils import * +from spack.util import * from spack.error import SpackError diff --git a/lib/spack/spack/globals.py b/lib/spack/spack/globals.py index 2001f4dacd..3dc7310a9d 100644 --- a/lib/spack/spack/globals.py +++ b/lib/spack/spack/globals.py @@ -1,6 +1,6 @@ import os from version import Version -from utils import * +from util import * import arch from directory_layout import DefaultDirectoryLayout diff --git a/lib/spack/spack/none_compare.py b/lib/spack/spack/none_compare.py deleted file mode 100644 index 67a00b8a40..0000000000 --- a/lib/spack/spack/none_compare.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Functions for comparing values that may potentially be None. -Functions prefixed with 'none_low_' treat None as less than all other values. -Functions prefixed with 'none_high_' treat None as greater than all other values. -""" - -def none_low_lt(lhs, rhs): - """Less-than comparison. None is lower than any value.""" - return lhs != rhs and (lhs == None or (rhs != None and lhs < rhs)) - - -def none_low_le(lhs, rhs): - """Less-than-or-equal comparison. None is less than any value.""" - return lhs == rhs or none_low_lt(lhs, rhs) - - -def none_low_gt(lhs, rhs): - """Greater-than comparison. None is less than any value.""" - return lhs != rhs and not none_low_lt(lhs, rhs) - - -def none_low_ge(lhs, rhs): - """Greater-than-or-equal comparison. None is less than any value.""" - return lhs == rhs or none_low_gt(lhs, rhs) - - -def none_low_min(lhs, rhs): - """Minimum function where None is less than any value.""" - if lhs == None or rhs == None: - return None - else: - return min(lhs, rhs) - - -def none_high_lt(lhs, rhs): - """Less-than comparison. None is greater than any value.""" - return lhs != rhs and (rhs == None or (lhs != None and lhs < rhs)) - - -def none_high_le(lhs, rhs): - """Less-than-or-equal comparison. None is greater than any value.""" - return lhs == rhs or none_high_lt(lhs, rhs) - - -def none_high_gt(lhs, rhs): - """Greater-than comparison. None is greater than any value.""" - return lhs != rhs and not none_high_lt(lhs, rhs) - - -def none_high_ge(lhs, rhs): - """Greater-than-or-equal comparison. None is greater than any value.""" - return lhs == rhs or none_high_gt(lhs, rhs) - - -def none_high_max(lhs, rhs): - """Maximum function where None is greater than any value.""" - if lhs == None or rhs == None: - return None - else: - return max(lhs, rhs) diff --git a/lib/spack/spack/packages/__init__.py b/lib/spack/spack/packages/__init__.py index f189e4c3bc..01788984ea 100644 --- a/lib/spack/spack/packages/__init__.py +++ b/lib/spack/spack/packages/__init__.py @@ -8,7 +8,7 @@ import glob import spack import spack.error import spack.spec -from spack.utils import * +from spack.util import * import spack.arch as arch # Valid package names can contain '-' but can't start with it. diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index f1cf3e78f9..1d1fccdfe7 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -24,7 +24,7 @@ import os import re import spack.error -import spack.utils +import spack.util from spack.version import Version # @@ -61,9 +61,9 @@ def parse_version_string_with_indices(path): if os.path.isdir(path): stem = os.path.basename(path) elif re.search(r'((?:sourceforge.net|sf.net)/.*)/download$', path): - stem = spack.utils.stem(os.path.dirname(path)) + stem = spack.util.stem(os.path.dirname(path)) else: - stem = spack.utils.stem(path) + stem = spack.util.stem(path) version_types = [ # GitHub tarballs, e.g. v1.2.3 diff --git a/lib/spack/spack/util/__init__.py b/lib/spack/spack/util/__init__.py new file mode 100644 index 0000000000..2781262b5a --- /dev/null +++ b/lib/spack/spack/util/__init__.py @@ -0,0 +1,209 @@ +import os +import re +import errno +import shutil +import subprocess +import multiprocessing +from itertools import product +import functools +from contextlib import closing, contextmanager + +import tty + +# Supported archvie extensions. +PRE_EXTS = ["tar"] +EXTS = ["gz", "bz2", "xz", "Z", "zip", "tgz"] + +# Add EXTS last so that .tar.gz is matched *before* tar.gz +ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS + + +def memoized(obj): + """Decorator that caches the results of a function, storing them + in an attribute of that function.""" + cache = obj.cache = {} + @functools.wraps(obj) + def memoizer(*args, **kwargs): + if args not in cache: + cache[args] = obj(*args, **kwargs) + return cache[args] + return memoizer + + +def install(src, dest): + tty.info("Installing %s to %s" % (src, dest)) + shutil.copy(src, dest) + + +def list_modules(directory): + """Lists all of the modules, excluding __init__.py, in + a particular directory.""" + for name in os.listdir(directory): + if name == '__init__.py': + continue + + path = new_path(directory, name) + if os.path.isdir(path): + init_py = new_path(path, '__init__.py') + if os.path.isfile(init_py): + yield name + + elif name.endswith('.py'): + yield re.sub('.py$', '', name) + + +@contextmanager +def working_dir(dirname): + orig_dir = os.getcwd() + os.chdir(dirname) + yield + os.chdir(orig_dir) + + +def mkdirp(*paths): + for path in paths: + if not os.path.exists(path): + os.makedirs(path) + elif not os.path.isdir(path): + raise OSError(errno.EEXIST, "File alredy exists", path) + + +def env_flag(name): + if name in os.environ: + return os.environ[name].lower() == "true" + return False + + +def path_set(var_name, directories): + path_str = ":".join(str(dir) for dir in directories) + os.environ[var_name] = path_str + + +def path_put_first(var_name, directories): + """Puts the provided directories first in the path, adding them + if they're not already there. + """ + path = os.environ.get(var_name, "").split(':') + + for dir in directories: + if dir in path: + path.remove(dir) + + new_path = tuple(directories) + tuple(path) + path_set(var_name, new_path) + + +def pop_keys(dictionary, *keys): + for key in keys: + if key in dictionary: + dictionary.pop(key) + + +def remove_items(item_list, *items): + for item in items: + if item in item_list: + item_list.remove(item) + + +def has_whitespace(string): + return re.search(r'\s', string) + + +def new_path(prefix, *args): + path=str(prefix) + for elt in args: + path = os.path.join(path, str(elt)) + + if has_whitespace(path): + tty.die("Invalid path: '%s'. Use a path without whitespace." % path) + + return path + + +def ancestor(dir, n=1): + """Get the nth ancestor of a directory.""" + parent = os.path.abspath(dir) + for i in range(n): + parent = os.path.dirname(parent) + return parent + + +class Executable(object): + """Class representing a program that can be run on the command line.""" + def __init__(self, name): + self.exe = name.split(' ') + + def add_default_arg(self, arg): + self.exe.append(arg) + + def __call__(self, *args, **kwargs): + """Run the executable with subprocess.check_output, return output.""" + return_output = kwargs.get("return_output", False) + fail_on_error = kwargs.get("fail_on_error", True) + + quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] + if quoted_args: + tty.warn("Quotes in package command arguments can confuse shell scripts like configure.", + "The following arguments may cause problems when executed:", + str("\n".join([" "+arg for arg in quoted_args])), + "Quotes aren't needed because spack doesn't use a shell. Consider removing them") + + cmd = self.exe + list(args) + tty.verbose(" ".join(cmd)) + + if return_output: + return subprocess.check_output(cmd) + elif fail_on_error: + return subprocess.check_call(cmd) + else: + return subprocess.call(cmd) + + def __repr__(self): + return "" % self.exe + + +def which(name, **kwargs): + """Finds an executable in the path like command-line which.""" + path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) + required = kwargs.get('required', False) + + if not path: + path = [] + + for dir in path: + exe = os.path.join(dir, name) + if os.access(exe, os.X_OK): + return Executable(exe) + + if required: + tty.die("spack requires %s. Make sure it is in your path." % name) + return None + + +def stem(path): + """Get the part of a path that does not include its compressed + type extension.""" + for type in ALLOWED_ARCHIVE_TYPES: + suffix = r'\.%s$' % type + if re.search(suffix, path): + return re.sub(suffix, "", path) + return path + + +def decompressor_for(path): + """Get the appropriate decompressor for a path.""" + tar = which('tar', required=True) + tar.add_default_arg('-xf') + return tar + + +def md5(filename, block_size=2**20): + import hashlib + md5 = hashlib.md5() + with closing(open(filename)) as file: + while True: + data = file.read(block_size) + if not data: + break + md5.update(data) + return md5.hexdigest() diff --git a/lib/spack/spack/util/none_high.py b/lib/spack/spack/util/none_high.py new file mode 100644 index 0000000000..897cf9ff8d --- /dev/null +++ b/lib/spack/spack/util/none_high.py @@ -0,0 +1,46 @@ +""" +Functions for comparing values that may potentially be None. +These none_high functions consider None as greater than all other values. +""" + +# Preserve builtin min and max functions +_builtin_min = min +_builtin_max = max + + +def lt(lhs, rhs): + """Less-than comparison. None is greater than any value.""" + return lhs != rhs and (rhs == None or (lhs != None and lhs < rhs)) + + +def le(lhs, rhs): + """Less-than-or-equal comparison. None is greater than any value.""" + return lhs == rhs or lt(lhs, rhs) + + +def gt(lhs, rhs): + """Greater-than comparison. None is greater than any value.""" + return lhs != rhs and not lt(lhs, rhs) + + +def ge(lhs, rhs): + """Greater-than-or-equal comparison. None is greater than any value.""" + return lhs == rhs or gt(lhs, rhs) + + +def min(lhs, rhs): + """Minimum function where None is greater than any value.""" + if lhs == None: + return rhs + elif rhs == None: + return lhs + else: + return _builtin_min(lhs, rhs) + + +def max(lhs, rhs): + """Maximum function where None is greater than any value.""" + if lhs == None or rhs == None: + return None + else: + return _builtin_max(lhs, rhs) diff --git a/lib/spack/spack/util/none_low.py b/lib/spack/spack/util/none_low.py new file mode 100644 index 0000000000..c332989248 --- /dev/null +++ b/lib/spack/spack/util/none_low.py @@ -0,0 +1,46 @@ +""" +Functions for comparing values that may potentially be None. +These none_low functions consider None as less than all other values. +""" + +# Preserve builtin min and max functions +_builtin_min = min +_builtin_max = max + + +def lt(lhs, rhs): + """Less-than comparison. None is lower than any value.""" + return lhs != rhs and (lhs == None or (rhs != None and lhs < rhs)) + + +def le(lhs, rhs): + """Less-than-or-equal comparison. None is less than any value.""" + return lhs == rhs or lt(lhs, rhs) + + +def gt(lhs, rhs): + """Greater-than comparison. None is less than any value.""" + return lhs != rhs and not lt(lhs, rhs) + + +def ge(lhs, rhs): + """Greater-than-or-equal comparison. None is less than any value.""" + return lhs == rhs or gt(lhs, rhs) + + +def min(lhs, rhs): + """Minimum function where None is less than any value.""" + if lhs == None or rhs == None: + return None + else: + return _builtin_min(lhs, rhs) + + +def max(lhs, rhs): + """Maximum function where None is less than any value.""" + if lhs == None: + return rhs + elif rhs == None: + return lhs + else: + return _builtin_max(lhs, rhs) diff --git a/lib/spack/spack/utils.py b/lib/spack/spack/utils.py deleted file mode 100644 index 2781262b5a..0000000000 --- a/lib/spack/spack/utils.py +++ /dev/null @@ -1,209 +0,0 @@ -import os -import re -import errno -import shutil -import subprocess -import multiprocessing -from itertools import product -import functools -from contextlib import closing, contextmanager - -import tty - -# Supported archvie extensions. -PRE_EXTS = ["tar"] -EXTS = ["gz", "bz2", "xz", "Z", "zip", "tgz"] - -# Add EXTS last so that .tar.gz is matched *before* tar.gz -ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS - - -def memoized(obj): - """Decorator that caches the results of a function, storing them - in an attribute of that function.""" - cache = obj.cache = {} - @functools.wraps(obj) - def memoizer(*args, **kwargs): - if args not in cache: - cache[args] = obj(*args, **kwargs) - return cache[args] - return memoizer - - -def install(src, dest): - tty.info("Installing %s to %s" % (src, dest)) - shutil.copy(src, dest) - - -def list_modules(directory): - """Lists all of the modules, excluding __init__.py, in - a particular directory.""" - for name in os.listdir(directory): - if name == '__init__.py': - continue - - path = new_path(directory, name) - if os.path.isdir(path): - init_py = new_path(path, '__init__.py') - if os.path.isfile(init_py): - yield name - - elif name.endswith('.py'): - yield re.sub('.py$', '', name) - - -@contextmanager -def working_dir(dirname): - orig_dir = os.getcwd() - os.chdir(dirname) - yield - os.chdir(orig_dir) - - -def mkdirp(*paths): - for path in paths: - if not os.path.exists(path): - os.makedirs(path) - elif not os.path.isdir(path): - raise OSError(errno.EEXIST, "File alredy exists", path) - - -def env_flag(name): - if name in os.environ: - return os.environ[name].lower() == "true" - return False - - -def path_set(var_name, directories): - path_str = ":".join(str(dir) for dir in directories) - os.environ[var_name] = path_str - - -def path_put_first(var_name, directories): - """Puts the provided directories first in the path, adding them - if they're not already there. - """ - path = os.environ.get(var_name, "").split(':') - - for dir in directories: - if dir in path: - path.remove(dir) - - new_path = tuple(directories) + tuple(path) - path_set(var_name, new_path) - - -def pop_keys(dictionary, *keys): - for key in keys: - if key in dictionary: - dictionary.pop(key) - - -def remove_items(item_list, *items): - for item in items: - if item in item_list: - item_list.remove(item) - - -def has_whitespace(string): - return re.search(r'\s', string) - - -def new_path(prefix, *args): - path=str(prefix) - for elt in args: - path = os.path.join(path, str(elt)) - - if has_whitespace(path): - tty.die("Invalid path: '%s'. Use a path without whitespace." % path) - - return path - - -def ancestor(dir, n=1): - """Get the nth ancestor of a directory.""" - parent = os.path.abspath(dir) - for i in range(n): - parent = os.path.dirname(parent) - return parent - - -class Executable(object): - """Class representing a program that can be run on the command line.""" - def __init__(self, name): - self.exe = name.split(' ') - - def add_default_arg(self, arg): - self.exe.append(arg) - - def __call__(self, *args, **kwargs): - """Run the executable with subprocess.check_output, return output.""" - return_output = kwargs.get("return_output", False) - fail_on_error = kwargs.get("fail_on_error", True) - - quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] - if quoted_args: - tty.warn("Quotes in package command arguments can confuse shell scripts like configure.", - "The following arguments may cause problems when executed:", - str("\n".join([" "+arg for arg in quoted_args])), - "Quotes aren't needed because spack doesn't use a shell. Consider removing them") - - cmd = self.exe + list(args) - tty.verbose(" ".join(cmd)) - - if return_output: - return subprocess.check_output(cmd) - elif fail_on_error: - return subprocess.check_call(cmd) - else: - return subprocess.call(cmd) - - def __repr__(self): - return "" % self.exe - - -def which(name, **kwargs): - """Finds an executable in the path like command-line which.""" - path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) - required = kwargs.get('required', False) - - if not path: - path = [] - - for dir in path: - exe = os.path.join(dir, name) - if os.access(exe, os.X_OK): - return Executable(exe) - - if required: - tty.die("spack requires %s. Make sure it is in your path." % name) - return None - - -def stem(path): - """Get the part of a path that does not include its compressed - type extension.""" - for type in ALLOWED_ARCHIVE_TYPES: - suffix = r'\.%s$' % type - if re.search(suffix, path): - return re.sub(suffix, "", path) - return path - - -def decompressor_for(path): - """Get the appropriate decompressor for a path.""" - tar = which('tar', required=True) - tar.add_default_arg('-xf') - return tar - - -def md5(filename, block_size=2**20): - import hashlib - md5 = hashlib.md5() - with closing(open(filename)) as file: - while True: - data = file.read(block_size) - if not data: - break - md5.update(data) - return md5.hexdigest() diff --git a/lib/spack/spack/validate.py b/lib/spack/spack/validate.py index 4b07d75464..c64fea7574 100644 --- a/lib/spack/spack/validate.py +++ b/lib/spack/spack/validate.py @@ -1,5 +1,5 @@ import tty -from utils import ALLOWED_ARCHIVE_TYPES +from util import ALLOWED_ARCHIVE_TYPES from urlparse import urlparse ALLOWED_SCHEMES = ["http", "https", "ftp"] diff --git a/lib/spack/spack/version.py b/lib/spack/spack/version.py index b3ebed3952..9ed7b465bb 100644 --- a/lib/spack/spack/version.py +++ b/lib/spack/spack/version.py @@ -25,8 +25,8 @@ import re from bisect import bisect_left from functools import total_ordering -import utils -from none_compare import * +import spack.util.none_high as none_high +import spack.util.none_low as none_low import spack.error # Valid version characters @@ -258,9 +258,9 @@ class VersionRange(object): if other is None: return False - return (none_low_lt(self.start, other.start) or + return (none_low.lt(self.start, other.start) or (self.start == other.start and - none_high_lt(self.end, other.end))) + none_high.lt(self.end, other.end))) @coerced @@ -281,8 +281,8 @@ class VersionRange(object): @coerced def __contains__(self, other): - return (none_low_ge(other.start, self.start) and - none_high_le(other.end, self.end)) + return (none_low.ge(other.start, self.start) and + none_high.le(other.end, self.end)) @coerced @@ -296,8 +296,8 @@ class VersionRange(object): @coerced def merge(self, other): - return VersionRange(none_low_min(self.start, other.start), - none_high_max(self.end, other.end)) + return VersionRange(none_low.min(self.start, other.start), + none_high.max(self.end, other.end)) def __hash__(self): -- cgit v1.2.3-60-g2f50