From 979b53cab605d250dfe23edaacc6b0d2ff88d978 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 18:33:07 +0200 Subject: qa : modifying 2 packages and a framework file just for the sake of it --- lib/spack/spack/architecture.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 2701fab90c..eec7636d00 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -44,6 +44,7 @@ class NoSysTypeError(serr.SpackError): "Could not determine sys_type for this machine.") + def get_sys_type_from_spack_globals(): """Return the SYS_TYPE from spack globals, or None if it isn't set.""" if not hasattr(spack, "sys_type"): -- cgit v1.2.3-70-g09d2 From 7d74e209f365ba135acd9dfe738737bcf37d4b79 Mon Sep 17 00:00:00 2001 From: alalazo Date: Tue, 10 May 2016 19:20:26 +0200 Subject: qa : this should still fail due to F821 --- lib/spack/spack/architecture.py | 15 +++++++-------- var/spack/repos/builtin/packages/ImageMagick/package.py | 14 +++++++------- var/spack/repos/builtin/packages/Mitos/package.py | 9 +++------ 3 files changed, 17 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index eec7636d00..62c25c8003 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -34,15 +34,14 @@ import spack.error as serr class InvalidSysTypeError(serr.SpackError): def __init__(self, sys_type): - super(InvalidSysTypeError, self).__init__( - "Invalid sys_type value for Spack: " + sys_type) + super(InvalidSysTypeError, + self).__init__("Invalid sys_type value for Spack: " + sys_type) class NoSysTypeError(serr.SpackError): def __init__(self): - super(NoSysTypeError, self).__init__( - "Could not determine sys_type for this machine.") - + super(NoSysTypeError, + self).__init__("Could not determine sys_type for this machine.") def get_sys_type_from_spack_globals(): @@ -70,15 +69,15 @@ def get_sys_type_from_platform(): @memoized def sys_type(): """Returns a SysType for the current machine.""" - methods = [get_sys_type_from_spack_globals, - get_sys_type_from_environment, + methods = [get_sys_type_from_spack_globals, get_sys_type_from_environment, get_sys_type_from_platform] # search for a method that doesn't return None sys_type = None for method in methods: sys_type = method() - if sys_type: break + if sys_type: + break # Couldn't determine the sys_type for this machine. if sys_type is None: diff --git a/var/spack/repos/builtin/packages/ImageMagick/package.py b/var/spack/repos/builtin/packages/ImageMagick/package.py index d0a820b5fc..7a3d12e364 100644 --- a/var/spack/repos/builtin/packages/ImageMagick/package.py +++ b/var/spack/repos/builtin/packages/ImageMagick/package.py @@ -1,10 +1,11 @@ from spack import * + class Imagemagick(Package): """ImageMagick is a image processing library""" homepage = "http://www.imagemagic.org" - #------------------------------------------------------------------------- + # ------------------------------------------------------------------------- # ImageMagick does not keep around anything but *-10 versions, so # this URL may change. If you want the bleeding edge, you can # uncomment it and see if it works but you may need to try to @@ -17,13 +18,13 @@ class Imagemagick(Package): # version('6.9.0-6', 'c1bce7396c22995b8bdb56b7797b4a1b', # url="http://www.imagemagick.org/download/ImageMagick-6.9.0-6.tar.bz2") - #------------------------------------------------------------------------- + # ------------------------------------------------------------------------- # *-10 versions are archived, so these versions should fetch reliably. # ------------------------------------------------------------------------- - version('6.8.9-10', 'aa050bf9785e571c956c111377bbf57c', - url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download") - - + version( + '6.8.9-10', + 'aa050bf9785e571c956c111377bbf57c', + url="http://sourceforge.net/projects/imagemagick/files/old-sources/6.x/6.8/ImageMagick-6.8.9-10.tar.gz/download") # NOQA: ignore=E501 depends_on('jpeg') depends_on('libtool') @@ -34,6 +35,5 @@ class Imagemagick(Package): def install(self, spec, prefix): configure("--prefix=%s" % prefix) - make() make("install") diff --git a/var/spack/repos/builtin/packages/Mitos/package.py b/var/spack/repos/builtin/packages/Mitos/package.py index 073c473b93..ec1d56a5c7 100644 --- a/var/spack/repos/builtin/packages/Mitos/package.py +++ b/var/spack/repos/builtin/packages/Mitos/package.py @@ -1,26 +1,23 @@ from spack import * + class Mitos(Package): """Mitos is a library and a tool for collecting sampled memory performance data to view with MemAxes""" homepage = "https://github.com/llnl/Mitos" - url = "https://github.com/llnl/Mitos" + url = "https://github.com/llnl/Mitos" version('0.9.2', git='https://github.com/llnl/Mitos.git', commit='8cb143a2e8c00353ff531a781a9ca0992b0aaa3d') - version('0.9.1', - git='https://github.com/llnl/Mitos.git', - tag='v0.9.1') + version('0.9.1', git='https://github.com/llnl/Mitos.git', tag='v0.9.1') depends_on('dyninst@8.2.1:') depends_on('hwloc') depends_on('mpi') - - def install(self, spec, prefix): with working_dir('spack-build', create=True): cmake('..', *std_cmake_args) -- cgit v1.2.3-70-g09d2 From 4cb91d6f7b6680140e0011da1f7965c430920067 Mon Sep 17 00:00:00 2001 From: Luca Heltai Date: Tue, 10 May 2016 19:46:24 +0200 Subject: Added documentation on lockf/flock. --- lib/spack/docs/basic_usage.rst | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'lib') diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 29791d98c4..58e67e4550 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1246,6 +1246,38 @@ several variants: spack deactivate -a python +A word of warning +----------------------- + +If you run `spack find` and you get an error similar to the following: + +.. code-block:: sh + + $ ./spack find + Traceback (most recent call last): + File "./spack", line 176, in + main() + File "./spack", line 154, in main + return_val = command(parser, args) + File "./spack/lib/spack/spack/cmd/find.py", line 170, in find + specs = set(spack.installed_db.query(**q_args)) + File "./spack/lib/spack/spack/database.py", line 551, in query + with self.read_transaction(): + File "./spack/lib/spack/spack/database.py", line 598, in __enter__ + if self._enter() and self._acquire_fn: + File "./spack/lib/spack/spack/database.py", line 608, in _enter + return self._db.lock.acquire_read(self._timeout) + File "./spack/lib/spack/llnl/util/lock.py", line 103, in acquire_read + self._lock(fcntl.LOCK_SH, timeout) # can raise LockError. + File "./spack/lib/spack/llnl/util/lock.py", line 64, in _lock + fcntl.lockf(self._fd, op | fcntl.LOCK_NB) + IOError: [Errno 38] Function not implemented + + +it most likely means that you are trying to run spack in a shared +network filesystem without support for lockf/flock. Currently this is not +supported, and you should ask your system administrator to enable lockf/flock. + Getting Help ----------------------- -- cgit v1.2.3-70-g09d2 From 98faee1d5c24ddacaeb1bc630ff8c31ba0d74f9b Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 10 May 2016 11:19:17 -0700 Subject: Minor tweaks to `flock` docs. --- lib/spack/docs/basic_usage.rst | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index 58e67e4550..f08a2bb675 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1246,13 +1246,30 @@ several variants: spack deactivate -a python -A word of warning ------------------------ +Filesystem requirements +-------------------------- -If you run `spack find` and you get an error similar to the following: +Spack currently needs to be run from a filesystem that supports +``flock`` locking semantics. Nearly all local filesystems and recent +versions of NFS support this, but parallel filesystems may be mounted +without ``flock`` support enabled. You can determine how your +filesystems are mounted with ``mount -p``. The output for a Lustre +filesystem might look like this: .. code-block:: sh - + + $ mount -l | grep lscratch + pilsner-mds1-lnet0@o2ib100:/lsd on /p/lscratchd type lustre (rw,nosuid,noauto,_netdev,lazystatfs,flock) + porter-mds1-lnet0@o2ib100:/lse on /p/lscratche type lustre (rw,nosuid,noauto,_netdev,lazystatfs,flock) + +Note the ``flock`` option on both Lustre mounts. If you do not see +this or a similar option for your filesystem, you may need ot ask your +system administrator to enable ``flock``. + +This issue typically manifests with the error below: + +.. code-block:: sh + $ ./spack find Traceback (most recent call last): File "./spack", line 176, in @@ -1273,11 +1290,7 @@ If you run `spack find` and you get an error similar to the following: fcntl.lockf(self._fd, op | fcntl.LOCK_NB) IOError: [Errno 38] Function not implemented - -it most likely means that you are trying to run spack in a shared -network filesystem without support for lockf/flock. Currently this is not -supported, and you should ask your system administrator to enable lockf/flock. - +A nicer error message is TBD in future versions of Spack. Getting Help ----------------------- -- cgit v1.2.3-70-g09d2 From 5d477bc956e0033bd2816d48df1f4096856bc5fe Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Tue, 10 May 2016 23:19:30 +0200 Subject: fix setting config for list parameters --- lib/spack/spack/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 336d47cbb7..06954f535b 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -545,7 +545,10 @@ def update_config(section, update_data, scope=None): # read in the config to ensure we've got current data configuration = get_config(section) - configuration.update(update_data) + if isinstance(update_data, list): + configuration = update_data + else: + configuration.extend(update_data) # read only the requested section's data. scope.sections[section] = {section: configuration} -- cgit v1.2.3-70-g09d2 From 6e462abb4d208584cb5f984f2f4c177bcec37dd4 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Tue, 10 May 2016 23:31:29 +0200 Subject: hmm... test before commit --- lib/spack/spack/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 06954f535b..3bdefd3a6c 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -548,7 +548,7 @@ def update_config(section, update_data, scope=None): if isinstance(update_data, list): configuration = update_data else: - configuration.extend(update_data) + configuration.update(update_data) # read only the requested section's data. scope.sections[section] = {section: configuration} -- cgit v1.2.3-70-g09d2 From 05b6c3f8cfd065dcf4173613a64ed94fa1815d31 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Wed, 11 May 2016 00:14:24 +0200 Subject: add test for list parameters --- lib/spack/spack/test/config.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'lib') diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 3977f0e7d4..ed0797a541 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -72,6 +72,10 @@ b_comps = { } } +# Some Sample repo data +repos_low = [ "/some/path" ] +repos_high = [ "/some/other/path" ] + class ConfigTest(MockPackagesTest): def setUp(self): @@ -95,6 +99,12 @@ class ConfigTest(MockPackagesTest): actual = config[arch][key][c] self.assertEqual(expected, actual) + def test_write_list_in_memory(self): + spack.config.update_config('repos', repos_low, 'test_low_priority') + spack.config.update_config('repos', repos_high, 'test_high_priority') + config = spack.config.get_config('repos') + self.assertEqual(config, repos_high+repos_low) + def test_write_key_in_memory(self): # Write b_comps "on top of" a_comps. spack.config.update_config('compilers', a_comps, 'test_low_priority') -- cgit v1.2.3-70-g09d2 From c1e6b5218438988d7dc2da5ca577fe5a8752d6e1 Mon Sep 17 00:00:00 2001 From: Benedikt Hegner Date: Wed, 11 May 2016 00:28:12 +0200 Subject: first round of coding rules --- lib/spack/spack/config.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 3bdefd3a6c..9da5d7aaf0 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -129,7 +129,6 @@ from ordereddict_backport import OrderedDict import llnl.util.tty as tty from llnl.util.filesystem import mkdirp -import copy import spack from spack.error import SpackError @@ -306,13 +305,14 @@ def extend_with_default(validator_class): yield err return validators.extend(validator_class, { - "properties" : set_defaults, - "patternProperties" : set_pp_defaults + "properties": set_defaults, + "patternProperties": set_pp_defaults }) DefaultSettingValidator = extend_with_default(Draft4Validator) + def validate_section(data, schema): """Validate data read in from a Spack YAML file. @@ -347,16 +347,14 @@ class ConfigScope(object): validate_section_name(section) return os.path.join(self.path, "%s.yaml" % section) - def get_section(self, section): - if not section in self.sections: + if section not in self.sections: path = self.get_section_filename(section) schema = section_schemas[section] data = _read_config_file(path, schema) self.sections[section] = data return self.sections[section] - def write_section(self, section): filename = self.get_section_filename(section) data = self.get_section(section) @@ -370,7 +368,6 @@ class ConfigScope(object): except (yaml.YAMLError, IOError) as e: raise ConfigFileError("Error writing to config file: '%s'" % str(e)) - def clear(self): """Empty cached config information.""" self.sections = {} @@ -476,7 +473,7 @@ def _merge_yaml(dest, source): # Source dict is merged into dest. elif they_are(dict): for sk, sv in source.iteritems(): - if not sk in dest: + if sk not in dest: dest[sk] = copy.copy(sv) else: dest[sk] = _merge_yaml(dest[sk], source[sk]) @@ -590,16 +587,20 @@ def spec_externals(spec): def is_spec_buildable(spec): """Return true if the spec pkgspec is configured as buildable""" allpkgs = get_config('packages') - name = spec.name - if not spec.name in allpkgs: + if spec.name not in allpkgs: return True - if not 'buildable' in allpkgs[spec.name]: + if 'buildable' not in allpkgs[spec.name]: return True return allpkgs[spec.name]['buildable'] -class ConfigError(SpackError): pass -class ConfigFileError(ConfigError): pass +class ConfigError(SpackError): + pass + + +class ConfigFileError(ConfigError): + pass + def get_path(path, data): if path: @@ -607,6 +608,7 @@ def get_path(path, data): else: return data + class ConfigFormatError(ConfigError): """Raised when a configuration format does not match its schema.""" def __init__(self, validation_error, data): @@ -641,5 +643,6 @@ class ConfigFormatError(ConfigError): message = '%s: %s' % (location, validation_error.message) super(ConfigError, self).__init__(message) + class ConfigSanityError(ConfigFormatError): """Same as ConfigFormatError, raised when config is written by Spack.""" -- cgit v1.2.3-70-g09d2 From 6d6eb0f2d1e1f5f4fd340f8e5f3a5159be7e7f40 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 11 May 2016 02:32:13 -0700 Subject: Remove config.py from flake checking for now. --- lib/spack/spack/config.py | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 9da5d7aaf0..21e08ff666 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -1,3 +1,4 @@ +# flake8: noqa ############################################################################## # Copyright (c) 2013-2015, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. -- cgit v1.2.3-70-g09d2