diff options
Diffstat (limited to 'lib')
25 files changed, 756 insertions, 237 deletions
diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index bbedfff3fc..878f0df0dd 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -39,27 +39,18 @@ Spack can install: .. command-output:: spack list -The packages are listed by name in alphabetical order. If you specify a -pattern to match, it will follow this set of rules. A pattern with no -wildcards, ``*`` or ``?``, will be treated as though it started and ended with -``*``, so ``util`` is equivalent to ``*util*``. A pattern with no capital -letters will be treated as case-insensitive. You can also add the ``-i`` flag -to specify a case insensitive search, or ``-d`` to search the description of +The packages are listed by name in alphabetical order. +A pattern to match with no wildcards, ``*`` or ``?``, +will be treated as though it started and ended with +``*``, so ``util`` is equivalent to ``*util*``. All patterns will be treated +as case-insensitive. You can also add the ``-d`` to search the description of the package in addition to the name. Some examples: -All packages whose names contain "sql" case insensitive: +All packages whose names contain "sql": .. command-output:: spack list sql -All packages whose names start with a capital M: - -.. command-output:: spack list 'M*' - -All packages whose names or descriptions contain Documentation: - -.. command-output:: spack list --search-description Documentation - -All packages whose names contain documentation case insensitive: +All packages whose names or descriptions contain documentation: .. command-output:: spack list --search-description documentation @@ -1333,6 +1324,41 @@ load two or more versions of the same software at the same time. .. note:: The ``conflict`` option is ``tcl`` specific +The names of environment modules generated by spack are not always easy to +fully comprehend due to the long hash in the name. There are two module +configuration options to help with that. The first is a global setting to +adjust the hash length. It can be set anywhere from 0 to 32 and has a default +length of 7. This is the representation of the hash in the module file name and +does not affect the size of the package hash. Be aware that the smaller the +hash length the more likely naming conflicts will occur. The following snippet +shows how to set hash length in the module file names: + +.. code-block:: yaml + + modules: + tcl: + hash_length: 7 + +To help make module names more readable, and to help alleviate name conflicts +with a short hash, one can use the ``suffixes`` option in the modules +configuration file. This option will add strings to modules that match a spec. +For instance, the following config options, + +.. code-block:: yaml + + modules: + tcl: + all: + suffixes: + ^python@2.7.12: 'python-2.7.12' + ^openblas: 'openblas' + +will add a ``python-2.7.12`` version string to any packages compiled with +python matching the spec, ``python@2.7.12``. This is useful to know which +version of python a set of python extensions is associated with. Likewise, the +``openblas`` string is attached to any program that has openblas in the spec, +most likely via the ``+blas`` variant specification. + ^^^^^^^^^^^^^^^^^^^^^^^^^ Regenerating Module files ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/contribution_guide.rst b/lib/spack/docs/contribution_guide.rst index 108b23a6d0..ebfab854d5 100644 --- a/lib/spack/docs/contribution_guide.rst +++ b/lib/spack/docs/contribution_guide.rst @@ -1,247 +1,521 @@ .. _contribution-guide: +================== Contribution Guide -===================== +================== This guide is intended for developers or administrators who want to contribute a new package, feature, or bugfix to Spack. It assumes that you have at least some familiarity with Git VCS and Github. -The guide will show a few examples of contributing workflow and discuss -the granularity of pull-requests (PRs). +The guide will show a few examples of contributing workflows and discuss +the granularity of pull-requests (PRs). It will also discuss the tests your +PR must pass in order to be accepted into Spack. First, what is a PR? Quoting `Bitbucket's tutorials <https://www.atlassian.com/git/tutorials/making-a-pull-request/>`_: - Pull requests are a mechanism for a developer to notify team members that they have **completed a feature**. - The pull request is more than just a notification—it’s a dedicated forum for discussing the proposed feature + Pull requests are a mechanism for a developer to notify team members that + they have **completed a feature**. The pull request is more than just a + notification—it’s a dedicated forum for discussing the proposed feature. -Important is completed feature, i.e. the changes one propose in a PR should +Important is **completed feature**. The changes one proposes in a PR should correspond to one feature/bugfix/extension/etc. One can create PRs with changes relevant to different ideas, however reviewing such PRs becomes tedious -and error prone. If possible, try to follow the rule **one-PR-one-package/feature.** +and error prone. If possible, try to follow the **one-PR-one-package/feature** rule. -Spack uses a rough approximation of the `Git Flow <http://nvie.com/posts/a-successful-git-branching-model/>`_ branching -model. The develop branch contains the latest contributions, and master is -always tagged and points to the latest stable release. Thereby when you send -your request, make ``develop`` the destination branch on the +Spack uses a rough approximation of the `Git Flow <http://nvie.com/posts/a-successful-git-branching-model/>`_ +branching model. The develop branch contains the latest contributions, and +master is always tagged and points to the latest stable release. Therefore, when +you send your request, make ``develop`` the destination branch on the `Spack repository <https://github.com/LLNL/spack>`_. -Let's assume that the current (patched) state of your fork of Spack is only -relevant to yourself. Now you come across a bug in a package or would like to -extend a package and contribute this fix to Spack. It is important that -whenever you change something that might be of importance upstream, -create a pull-request (PR) as soon as possible. Do not wait for weeks/months to -do this: a) you might forget why did you modified certain files; b) it could get -difficult to isolate this change into a stand-alone clean PR. +---------------------- +Continuous Integration +---------------------- -Now let us discuss several approaches one may use to submit a PR while -also keeping your local version of Spack patched. +Spack uses `Travis CI <https://travis-ci.org/LLNL/spack>`_ for Continuous Integration +testing. This means that every time you submit a pull request, a series of tests will +be run to make sure you didn't accidentally introduce any bugs into Spack. Your PR +will not be accepted until it passes all of these tests. While you can certainly wait +for the results of these tests after submitting a PR, we recommend that you run them +locally to speed up the review process. +If you take a look in ``$SPACK_ROOT/.travis.yml``, you'll notice that we test +against Python 2.6 and 2.7. We currently perform 3 types of tests: -First approach (cherry-picking): --------------------------------- +^^^^^^^^^^ +Unit Tests +^^^^^^^^^^ -First approach is as follows. -You checkout your local develop branch, which for the purpose of this guide -will be called ``develop_modified``: +Unit tests ensure that core Spack features like fetching or spec resolution are +working as expected. If your PR only adds new packages or modifies existing ones, +there's very little chance that your changes could cause the unit tests to fail. +However, if you make changes to Spack's core libraries, you should run the unit +tests to make sure you didn't break anything. + +Since they test things like fetching from VCS repos, the unit tests require +`git <https://git-scm.com/>`_, `mercurial <https://www.mercurial-scm.org/>`_, +and `subversion <https://subversion.apache.org/>`_ to run. Make sure these are +installed on your system and can be found in your ``PATH``. All of these can be +installed with Spack or with your system package manager. + +To run *all* of the unit tests, use: .. code-block:: console - $ git checkout develop_modified + $ spack test -Let us assume that lines in files you will be modifying -are the same in `develop_modified` branch and upstream ``develop``. -Next edit files, make sure they work for you and create a commit +These tests may take several minutes to complete. If you know you are only +modifying a single Spack feature, you can run a single unit test at a time: .. code-block:: console - $ git add <files_to_be_commited> - $ git commit -m <descriptive note about changes> + $ spack test architecture -Normally we prefer that commits pertaining to a package ``<package-name>``` have -a message ``<package-name>: descriptive message``. It is important to add -descriptive message so that others, who might be looking at your changes later -(in a year or maybe two), would understand the rationale behind. +This allows you to develop iteratively: make a change, test that change, make +another change, test that change, etc. To get a list of all available unit +tests, run: + +.. command-output:: spack test --list + +Unit tests are crucial to making sure bugs aren't introduced into Spack. If you +are modifying core Spack libraries or adding new functionality, please consider +adding new unit tests or strengthening existing tests. + +.. note:: + + There is also a ``run-unit-tests`` script in ``share/spack/qa`` that runs the + unit tests. Afterwards, it reports back to Coverage with the percentage of Spack + that is covered by unit tests. This script is designed for Travis CI. If you + want to run the unit tests yourself, we suggest you use ``spack test``. + +^^^^^^^^^^^^ +Flake8 Tests +^^^^^^^^^^^^ + +Spack uses `Flake8 <http://flake8.pycqa.org/en/latest/>`_ to test for +`PEP 8 <https://www.python.org/dev/peps/pep-0008/>`_ conformance. PEP 8 is +a series of style guides for Python that provide suggestions for everything +from variable naming to indentation. In order to limit the number of PRs that +were mostly style changes, we decided to enforce PEP 8 conformance. Your PR +needs to comply with PEP 8 in order to be accepted. + +Testing for PEP 8 compliance is easy. Simply add the quality assurance +directory to your ``PATH`` and run the flake8 script: + +.. code-block:: console + + $ export PATH+=":$SPACK_ROOT/share/spack/qa" + $ run-flake8-tests +``run-flake8-tests`` has a couple advantages over running ``flake8`` by hand: -Next we will create a branch off upstream's ``develop`` and copy this commit. -Before doing this, while still on your modified branch, get the hash of the -last commit +#. It only tests files that you have modified since branching off of develop. + +#. It works regardless of what directory you are in. + +#. It automatically adds approved exemptions from the flake8 checks. For example, + URLs are often longer than 80 characters, so we exempt them from the line + length checks. We also exempt lines that start with "homepage", "url", "version", + "variant", "depends_on", and "extends" in the ``package.py`` files. + +More approved flake8 exemptions can be found +`here <https://github.com/LLNL/spack/blob/develop/.flake8>`_. + +If all is well, you'll see something like this: .. code-block:: console - $ git log -1 + $ run-flake8-tests + Dependencies found. + ======================================================= + flake8: running flake8 code checks on spack. -and copy-paste this ``<hash>`` to the buffer. Now switch to upstream's ``develop``, -make sure it's updated, checkout the new branch, apply the patch and push to -GitHub: + Modified files: + + var/spack/repos/builtin/packages/hdf5/package.py + var/spack/repos/builtin/packages/hdf/package.py + var/spack/repos/builtin/packages/netcdf/package.py + ======================================================= + Flake8 checks were clean. + +However, if you aren't compliant with PEP 8, flake8 will complain: .. code-block:: console - $ git checkout develop - $ git pull upstream develop - $ git checkout -b <descriptive_branch_name> - $ git cherry-pick <hash> - $ git push <your_origin> <descriptive_branch_name> -u + var/spack/repos/builtin/packages/netcdf/package.py:26: [F401] 'os' imported but unused + var/spack/repos/builtin/packages/netcdf/package.py:61: [E303] too many blank lines (2) + var/spack/repos/builtin/packages/netcdf/package.py:106: [E501] line too long (92 > 79 characters) + Flake8 found errors. -Here we assume that local ``develop`` branch tracks upstream develop branch of -Spack. This is not a requirement and you could also do the same with remote -branches. Yet to some it is more convenient to have a local branch that -tracks upstream. +Most of the error messages are straightforward, but if you don't understand what +they mean, just ask questions about them when you submit your PR. The line numbers +will change if you add or delete lines, so simply run ``run-flake8-tests`` again +to update them. -Now you can create a PR from web-interface of GitHub. The net result is as -follows: +.. tip:: -#. You patched your local version of Spack and can use it further -#. You "cherry-picked" these changes in a stand-alone branch and submitted it - as a PR upstream. + Try fixing flake8 errors in reverse order. This eliminates the need for + multiple runs of ``flake8`` just to re-compute line numbers and makes it + much easier to fix errors directly off of the Travis output. +.. warning:: -Should you have several commits to contribute, you could follow the same -procedure by getting hashes of all of them and cherry-picking to the PR branch. -This could get tedious and therefore there is another way: + Flake8 requires setuptools in order to run. If you installed ``py-flake8`` + with Spack, make sure to add ``py-setuptools`` to your ``PYTHONPATH``. + Otherwise, you will get an error message like: + + .. code-block:: console + + Traceback (most recent call last): + File: "/usr/bin/flake8", line 5, in <module> + from pkg_resources import load_entry_point + ImportError: No module named pkg_resources +^^^^^^^^^^^^^^^^^^^ +Documentation Tests +^^^^^^^^^^^^^^^^^^^ -Second approach: ----------------- +Spack uses `Sphinx <http://www.sphinx-doc.org/en/stable/>`_ to build its +documentation. In order to prevent things like broken links and missing imports, +we added documentation tests that build the documentation and fail if there +are any warning or error messages. -In the second approach we start from upstream ``develop`` (again assuming -that your local branch `develop` tracks upstream): +Building the documentation requires several dependencies, all of which can be +installed with Spack: + +* sphinx +* graphviz +* git +* mercurial +* subversion + +.. warning:: + + Sphinx has `several required dependencies <https://github.com/LLNL/spack/blob/develop/var/spack/repos/builtin/packages/py-sphinx/package.py>`_. + If you installed ``py-sphinx`` with Spack, make sure to add all of these + dependencies to your ``PYTHONPATH``. The easiest way to do this is to run + ``spack activate py-sphinx`` so that all of the dependencies are symlinked + to a central location. If you see an error message like: + + .. code-block:: console + + Traceback (most recent call last): + File: "/usr/bin/flake8", line 5, in <module> + from pkg_resources import load_entry_point + ImportError: No module named pkg_resources + + that means Sphinx couldn't find setuptools in your ``PYTHONPATH``. + +Once all of the dependencies are installed, you can try building the documentation: .. code-block:: console - $ git checkout develop - $ git pull upstream develop - $ git checkout -b <descriptive_branch_name> + $ cd "$SPACK_ROOT/lib/spack/docs" + $ make clean + $ make -Next edit a few files and create a few commits by +If you see any warning or error messages, you will have to correct those before +your PR is accepted. + +.. note:: + + There is also a ``run-doc-tests`` script in the Quality Assurance directory. + The only difference between running this script and running ``make`` by hand + is that the script will exit immediately if it encounters an error or warning. + This is necessary for Travis CI. If you made a lot of documentation tests, it + is much quicker to run ``make`` by hand so that you can see all of the warnings + at once. + +If you are editing the documentation, you should obviously be running the +documentation tests. But even if you are simply adding a new package, your +changes could cause the documentation tests to fail: .. code-block:: console - $ git add <files_to_be_part_of_the_commit> - $ git commit -m <descriptive_message_of_this_particular_commit> + package_list.rst:8745: WARNING: Block quote ends without a blank line; unexpected unindent. + +At first, this error message will mean nothing to you, since you didn't edit +that file. Until you look at line 8745 of the file in question: + +.. code-block:: rst + + Description: + NetCDF is a set of software libraries and self-describing, machine- + independent data formats that support the creation, access, and sharing + of array-oriented scientific data. + +Our documentation includes :ref:`a list of all Spack packages <package-list>`. +If you add a new package, its docstring is added to this page. The problem in +this case was that the docstring looked like: -Now you can push it to your fork and create a PR +.. code-block:: python + + class Netcdf(Package): + """ + NetCDF is a set of software libraries and self-describing, + machine-independent data formats that support the creation, + access, and sharing of array-oriented scientific data. + """ + +Docstrings cannot start with a newline character, or else Sphinx will complain. +Instead, they should look like: + +.. code-block:: python + + class Netcdf(Package): + """NetCDF is a set of software libraries and self-describing, + machine-independent data formats that support the creation, + access, and sharing of array-oriented scientific data.""" + +Documentation changes can result in much more obfuscated warning messages. +If you don't understand what they mean, feel free to ask when you submit +your PR. + +------------- +Git Workflows +------------- + +Spack is still in the beta stages of development. Most of our users run off of +the develop branch, and fixes and new features are constantly being merged. So +how do you keep up-to-date with upstream while maintaining your own local +differences and contributing PRs to Spack? + +^^^^^^^^^ +Branching +^^^^^^^^^ + +The easiest way to contribute a pull request is to make all of your changes on +new branches. Make sure your ``develop`` is up-to-date and create a new branch +off of it: .. code-block:: console - $ git push <your_origin> <descriptive_branch_name> -u + $ git checkout develop + $ git pull upstream develop + $ git branch <descriptive_branch_name> + $ git checkout <descriptive_branch_name> + +Here we assume that the local ``develop`` branch tracks the upstream develop +branch of Spack. This is not a requirement and you could also do the same with +remote branches. But for some it is more convenient to have a local branch that +tracks upstream. -Most likely you would want to have those changes in your (modified) local -version of Spack. To that end you need to merge this branch +Normally we prefer that commits pertaining to a package ``<package-name>`` have +a message ``<package-name>: descriptive message``. It is important to add +descriptive message so that others, who might be looking at your changes later +(in a year or maybe two), would understand the rationale behind them. + +Now, you can make your changes while keeping the ``develop`` branch pure. +Edit a few files and commit them by running: .. code-block:: console - $ git checkout develop_modified - $ git merge <descriptive_branch_name> + $ git add <files_to_be_part_of_the_commit> + $ git commit --message <descriptive_message_of_this_particular_commit> -The net result is similar to the first approach with a minor difference that -you would also merge upstream develop into you modified version in the last -step. Should this not be desirable, you have to follow the first approach. +Next, push it to your remote fork and create a PR: +.. code-block:: console + $ git push origin <descriptive_branch_name> --set-upstream -How to clean-up a branch by rewriting history: ------------------------------------------------ +GitHub provides a `tutorial <https://help.github.com/articles/about-pull-requests/>`_ +on how to file a pull request. When you send the request, make ``develop`` the +destination branch. -Sometimes you may end up on a branch that has a lot of commits, merges of -upstream branch and alike but it can't be rebased on ``develop`` due to a long -and convoluted history. If the current commits history is more of an experimental -nature and only the net result is important, you may rewrite the history. -To that end you need to first merge upstream `develop` and reset you branch to -it. So on the branch in question do: +If you need this change immediately and don't have time to wait for your PR to +be merged, you can always work on this branch. But if you have multiple PRs, +another option is to maintain a Frankenstein branch that combines all of your +other branches: .. code-block:: console - $ git merge develop - $ git reset develop + $ git co develop + $ git branch <your_modified_develop_branch> + $ git checkout <your_modified_develop_branch> + $ git merge <descriptive_branch_name> -At this point you your branch will point to the same commit as develop and -thereby the two are indistinguishable. However, all the files that were -previously modified will stay as such. In other words, you do not loose the -changes you made. Changes can be reviewed by looking at diffs +This can be done with each new PR you submit. Just make sure to keep this local +branch up-to-date with upstream ``develop`` too. + +^^^^^^^^^^^^^^ +Cherry-Picking +^^^^^^^^^^^^^^ + +What if you made some changes to your local modified develop branch and already +committed them, but later decided to contribute them to Spack? You can use +cherry-picking to create a new branch with only these commits. + +First, check out your local modified develop branch: .. code-block:: console - $ git status - $ git diff + $ git checkout <your_modified_develop_branch> -One can also run GUI to visualize the current changes +Now, get the hashes of the commits you want from the output of: .. code-block:: console - $ git difftool + $ git log -Next step is to rewrite the history by adding files and creating commits +Next, create a new branch off of upstream ``develop`` and copy the commits +that you want in your PR: .. code-block:: console - $ git add <files_to_be_part_of_commit> - $ git commit -m <descriptive_message> + $ git checkout develop + $ git pull upstream develop + $ git branch <descriptive_branch_name> + $ git checkout <descriptive_branch_name> + $ git cherry-pick <hash> + $ git push origin <descriptive_branch_name> --set-upstream +Now you can create a PR from the web-interface of GitHub. The net result is as +follows: -Shall you need to split changes within a file into separate commits, use +#. You patched your local version of Spack and can use it further. +#. You "cherry-picked" these changes in a stand-alone branch and submitted it + as a PR upstream. + +Should you have several commits to contribute, you could follow the same +procedure by getting hashes of all of them and cherry-picking to the PR branch. + +.. note:: + + It is important that whenever you change something that might be of + importance upstream, create a pull request as soon as possible. Do not wait + for weeks/months to do this, because: + + #. you might forget why you modified certain files + #. it could get difficult to isolate this change into a stand-alone clean PR. + +^^^^^^^^ +Rebasing +^^^^^^^^ + +Other developers are constantly making contributions to Spack, possibly on the +same files that your PR changed. If their PR is merged before yours, it can +create a merge conflict. This means that your PR can no longer be automatically +merged without a chance of breaking your changes. In this case, you will be +asked to rebase on top of the latest upstream ``develop``. + +First, make sure your develop branch is up-to-date: .. code-block:: console - $ git add <file> -p + $ git checkout develop + $ git pull upstream develop -After all changed files are committed, you can push the branch to your fork -and create a PR +Now, we need to switch to the branch you submitted for your PR and rebase it +on top of develop: + +.. code-block:: console + + $ git checkout <descriptive_branch_name> + $ git rebase develop + +Git will likely ask you to resolve conflicts. Edit the file that it says can't +be merged automatically and resolve the conflict. Then, run: .. code-block:: console - $ git push <you_origin> -u + $ git add <file_that_could_not_be_merged> + $ git rebase --continue +You may have to repeat this process multiple times until all conflicts are resolved. +Once this is done, simply force push your rebased branch to your remote fork: +.. code-block:: console -How to fix a bad rebase by "cherry-picking" commits: ----------------------------------------------------- + $ git push --force origin <descriptive_branch_name> -Say you are working on a branch ``feature1``. It has several commits and is -ready to be merged. However, there are a few minor merge conflicts and so -you are asked to rebase onto ``develop`` upstream branch. Occasionally, it -happens so that a contributor rebases not on top of the upstream branch, but -on his/her local outdated copy of it. This would lead to an inclusion of the -whole lot of duplicated history and of course can not be merged as-is. +^^^^^^^^^^^^^^^^^^^^^^^^^ +Rebasing with cherry-pick +^^^^^^^^^^^^^^^^^^^^^^^^^ -One way to get out of troubles is to ``cherry-pick`` important commits. To -do that, first checkout a temporary back-up branch: +You can also perform a rebase using ``cherry-pick``. First, create a temporary +backup branch: .. code-block:: console - git checkout -b tmp + $ git checkout <descriptive_branch_name> + $ git branch tmp -Now look at logs and save hashes of commits you would like to keep +If anything goes wrong, you can always go back to your ``tmp`` branch. +Now, look at the logs and save the hashes of any commits you would like to keep: .. code-block:: console - git log + $ git log Next, go back to the original branch and reset it to ``develop``. Before doing so, make sure that you local ``develop`` branch is up-to-date -with the upstream. +with upstream: + +.. code-block:: console + + $ git checkout develop + $ git pull upstream develop + $ git checkout <descriptive_branch_name> + $ git reset --hard develop + +Now you can cherry-pick relevant commits: + +.. code-block:: console + + $ git cherry-pick <hash1> + $ git cherry-pick <hash2> + +Push the modified branch to your fork: .. code-block:: console - git checkout feature1 - git reset --hard develop + $ git push --force origin <descriptive_branch_name> -Now you can cherry-pick relevant commits +If everything looks good, delete the backup branch: .. code-block:: console - git cherry-pick <hash1> - git cherry-pick <hash2> + $ git branch --delete --force tmp + +^^^^^^^^^^^^^^^^^^ +Re-writing History +^^^^^^^^^^^^^^^^^^ +Sometimes you may end up on a branch that has diverged so much from develop +that it cannot easily be rebased. If the current commits history is more of +an experimental nature and only the net result is important, you may rewrite +the history. -push the modified branch to your fork +First, merge upstream ``develop`` and reset you branch to it. On the branch +in question, run: .. code-block:: console - git push -f + $ git merge develop + $ git reset develop -and if everything looks good, delete the back-up: +At this point your branch will point to the same commit as develop and +thereby the two are indistinguishable. However, all the files that were +previously modified will stay as such. In other words, you do not lose the +changes you made. Changes can be reviewed by looking at diffs: .. code-block:: console - git branch -D tmp + $ git status + $ git diff + +The next step is to rewrite the history by adding files and creating commits: + +.. code-block:: console + + $ git add <files_to_be_part_of_commit> + $ git commit --message <descriptive_message> + +After all changed files are committed, you can push the branch to your fork +and create a PR: + +.. code-block:: console + + $ git push origin --set-upstream + diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 155bbcdb08..efd4c459a2 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -3004,13 +3004,18 @@ Cleans up all of Spack's temporary and cached files. This can be used to recover disk space if temporary files from interrupted or failed installs accumulate in the staging area. -When called with ``--stage`` or ``--all`` (or without arguments, in which case -the default is ``--all``) this removes all staged files; this is equivalent to -running ``spack clean`` for every package you have fetched or staged. +When called with ``--stage`` or without arguments this removes all staged +files and will be equivalent to running ``spack clean`` for every package +you have fetched or staged. -When called with ``--cache`` or ``--all`` this will clear all resources +When called with ``--downloads`` this will clear all resources :ref:`cached <caching>` during installs. +When called with ``--user-cache`` this will remove caches in the user home +directory, including cached virtual indices. + +To remove all of the above, the command can be called with ``--all``. + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Keeping the stage directory on success ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index c9c29a32cc..c7d592befb 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -129,8 +129,9 @@ from spack.util.executable import Executable, which # User's editor from the environment editor = Executable(os.environ.get("EDITOR", "vi")) -# Curl tool for fetching files. -curl = which("curl", required=True) +# If this is enabled, tools that use SSL should not verify +# certifiates. e.g., curl should use the -k option. +insecure = False # Whether to build in tmp space or directly in the stage_path. # If this is true, then spack will make stage directories in diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 57e266722e..b14de35109 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -83,7 +83,6 @@ from llnl.util.filesystem import join_path import llnl.util.tty as tty import spack -import spack.compilers from spack.util.naming import mod_to_class from spack.util.environment import get_path from spack.util.multiproc import parmap @@ -276,6 +275,8 @@ class OperatingSystem(object): # Once the paths are cleaned up, do a search for each type of # compiler. We can spawn a bunch of parallel searches to reduce # the overhead of spelunking all these directories. + # NOTE: we import spack.compilers here to avoid init order cycles + import spack.compilers types = spack.compilers.all_compiler_types() compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *filtered_path), diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index 98fb3b6917..7579f9adc6 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -221,6 +221,8 @@ def set_compiler_environment_variables(pkg, env): for mod in compiler.modules: load_module(mod) + compiler.setup_custom_environment(env) + return env diff --git a/lib/spack/spack/cmd/config.py b/lib/spack/spack/cmd/config.py index c189e37036..5e041192e2 100644 --- a/lib/spack/spack/cmd/config.py +++ b/lib/spack/spack/cmd/config.py @@ -29,13 +29,8 @@ description = "Get and set configuration options." def setup_parser(subparser): # User can only choose one - scope_group = subparser.add_mutually_exclusive_group() - scope_group.add_argument( - '--user', action='store_const', const='user', dest='scope', - help="Use config file in user home directory (default).") - scope_group.add_argument( - '--site', action='store_const', const='site', dest='scope', - help="Use config file in spack prefix.") + subparser.add_argument('--scope', choices=spack.config.config_scopes, + help="Configuration scope to read/modify.") sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='config_command') diff --git a/lib/spack/spack/cmd/create.py b/lib/spack/spack/cmd/create.py index aa7c42def0..741a320ea7 100644 --- a/lib/spack/spack/cmd/create.py +++ b/lib/spack/spack/cmd/create.py @@ -116,6 +116,10 @@ dependencies_dict = { # FIXME: Add additional dependencies if required. depends_on('scons', type='build')""", + 'bazel': """\ + # FIXME: Add additional dependencies if required. + depends_on('bazel', type='build')""", + 'python': """\ extends('python') @@ -164,6 +168,10 @@ install_dict = { scons('prefix={0}'.format(prefix)) scons('install')""", + 'bazel': """\ + # FIXME: Add logic to build and install here. + bazel()""", + 'python': """\ # FIXME: Add logic to build and install here. setup_py('install', '--prefix={0}'.format(prefix))""", @@ -238,7 +246,8 @@ class BuildSystemGuesser(object): (r'/CMakeLists.txt$', 'cmake'), (r'/SConstruct$', 'scons'), (r'/setup.py$', 'python'), - (r'/NAMESPACE$', 'R') + (r'/NAMESPACE$', 'R'), + (r'/WORKSPACE$', 'bazel') ] # Peek inside the compressed file. diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py index cc9b6c73ee..6ca75d7999 100644 --- a/lib/spack/spack/cmd/install.py +++ b/lib/spack/spack/cmd/install.py @@ -34,12 +34,15 @@ description = "Build and install packages" def setup_parser(subparser): subparser.add_argument( - '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps', - help="Do not try to install dependencies of requested packages.") - subparser.add_argument( - '-d', '--dependencies-only', action='store_true', dest='deps_only', - help='Install dependencies of this package, ' + - 'but not the package itself.') + '--only', + default='package,dependencies', + dest='things_to_install', + choices=['package', 'dependencies', 'package,dependencies'], + help="""Select the mode of installation. +The default is to install the package along with all its dependencies. +Alternatively one can decide to install only the package or only +the dependencies.""" + ) subparser.add_argument( '-j', '--jobs', action='store', type=int, help="Explicitly set number of make jobs. Default is #cpus.") @@ -62,18 +65,17 @@ def setup_parser(subparser): '--dirty', action='store_true', dest='dirty', help="Install a package *without* cleaning the environment.") subparser.add_argument( - '--stop-at', help="Stop at a particular phase of installation" + 'package', + nargs=argparse.REMAINDER, + help="spec of the package to install" ) subparser.add_argument( - 'packages', nargs=argparse.REMAINDER, - help="specs of packages to install") - subparser.add_argument( '--run-tests', action='store_true', dest='run_tests', help="Run tests during installation of a package.") def install(parser, args): - if not args.packages: + if not args.package: tty.die("install requires at least one package argument") if args.jobs is not None: @@ -83,19 +85,33 @@ def install(parser, args): if args.no_checksum: spack.do_checksum = False # TODO: remove this global. - specs = spack.cmd.parse_specs(args.packages, concretize=True) - for spec in specs: + # Parse cli arguments and construct a dictionary + # that will be passed to Package.do_install API + kwargs = { + 'keep_prefix': args.keep_prefix, + 'keep_stage': args.keep_stage, + 'install_deps': 'dependencies' in args.things_to_install, + 'make_jobs': args.jobs, + 'run_tests': args.run_tests, + 'verbose': args.verbose, + 'fake': args.fake, + 'dirty': args.dirty + } + + # Spec from cli + specs = spack.cmd.parse_specs(args.package, concretize=True) + if len(specs) != 1: + tty.error('only one spec can be installed at a time.') + spec = specs.pop() + + if args.things_to_install == 'dependencies': + # Install dependencies as-if they were installed + # for root (explicit=False in the DB) + kwargs['explicit'] = False + for s in spec.dependencies(): + p = spack.repo.get(s) + p.do_install(**kwargs) + else: package = spack.repo.get(spec) - package.do_install( - keep_prefix=args.keep_prefix, - keep_stage=args.keep_stage, - install_deps=not args.ignore_deps, - install_self=not args.deps_only, - make_jobs=args.jobs, - run_tests=args.run_tests, - verbose=args.verbose, - fake=args.fake, - dirty=args.dirty, - explicit=True, - stop_at=args.stop_at - ) + kwargs['explicit'] = True + package.do_install(**kwargs) diff --git a/lib/spack/spack/cmd/test_install.py b/lib/spack/spack/cmd/test_install.py index c35f2740a0..f962c5988a 100644 --- a/lib/spack/spack/cmd/test_install.py +++ b/lib/spack/spack/cmd/test_install.py @@ -181,7 +181,6 @@ def install_single_spec(spec, number_of_jobs): package.do_install(keep_prefix=False, keep_stage=True, install_deps=True, - install_self=True, make_jobs=number_of_jobs, verbose=True, fake=False) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index a77991e4dc..fc663ea646 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -114,9 +114,15 @@ class Compiler(object): def __init__(self, cspec, operating_system, paths, modules=[], alias=None, **kwargs): + self.operating_system = operating_system + self.spec = cspec + self.modules = modules + self.alias = alias + def check(exe): if exe is None: return None + exe = self._find_full_path(exe) _verify_executables(exe) return exe @@ -138,11 +144,6 @@ class Compiler(object): if value is not None: self.flags[flag] = value.split() - self.operating_system = operating_system - self.spec = cspec - self.modules = modules - self.alias = alias - @property def version(self): return self.spec.version @@ -269,6 +270,21 @@ class Compiler(object): successful.reverse() return dict(((v, p, s), path) for v, p, s, path in successful) + def _find_full_path(self, path): + """Return the actual path for a tool. + + Some toolchains use forwarding executables (particularly Xcode-based + toolchains) which can be manipulated by external environment variables. + This method should be used to extract the actual path used for a tool + by finding out the end executable the forwarding executables end up + running. + """ + return path + + def setup_custom_environment(self, env): + """Set any environment variables necessary to use the compiler.""" + pass + def __repr__(self): """Return a string representation of the compiler toolchain.""" return self.__str__() diff --git a/lib/spack/spack/compilers/clang.py b/lib/spack/spack/compilers/clang.py index f7f1bf106b..34eec4ea7b 100644 --- a/lib/spack/spack/compilers/clang.py +++ b/lib/spack/spack/compilers/clang.py @@ -23,11 +23,14 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import re +import os +import spack import spack.compiler as cpr from spack.compiler import * from spack.util.executable import * import llnl.util.tty as tty from spack.version import ver +from shutil import copytree, ignore_patterns class Clang(Compiler): @@ -107,3 +110,79 @@ class Clang(Compiler): cpr._version_cache[comp] = ver return cpr._version_cache[comp] + + def _find_full_path(self, path): + basename = os.path.basename(path) + + if not self.is_apple or basename not in ('clang', 'clang++'): + return super(Clang, self)._find_full_path(path) + + xcrun = Executable('xcrun') + full_path = xcrun('-f', basename, output=str) + return full_path.strip() + + def setup_custom_environment(self, env): + """Set the DEVELOPER_DIR environment for the Xcode toolchain. + + On macOS, not all buildsystems support querying CC and CXX for the + compilers to use and instead query the Xcode toolchain for what + compiler to run. This side-steps the spack wrappers. In order to inject + spack into this setup, we need to copy (a subset of) Xcode.app and + replace the compiler executables with symlinks to the spack wrapper. + Currently, the stage is used to store the Xcode.app copies. We then set + the 'DEVELOPER_DIR' environment variables to cause the xcrun and + related tools to use this Xcode.app. + """ + super(Clang, self).setup_custom_environment(env) + + if not self.is_apple: + return + + xcode_select = Executable('xcode-select') + real_root = xcode_select('--print-path', output=str).strip() + real_root = os.path.dirname(os.path.dirname(real_root)) + developer_root = os.path.join(spack.stage_path, + 'xcode-select', + self.name, + str(self.version)) + xcode_link = os.path.join(developer_root, 'Xcode.app') + + if not os.path.exists(developer_root): + tty.warn('Copying Xcode from %s to %s in order to add spack ' + 'wrappers to it. Please do not interrupt.' + % (real_root, developer_root)) + + # We need to make a new Xcode.app instance, but with symlinks to + # the spack wrappers for the compilers it ships. This is necessary + # because some projects insist on just asking xcrun and related + # tools where the compiler runs. These tools are very hard to trick + # as they do realpath and end up ignoring the symlinks in a + # "softer" tree of nothing but symlinks in the right places. + copytree(real_root, developer_root, symlinks=True, + ignore=ignore_patterns('AppleTV*.platform', + 'Watch*.platform', + 'iPhone*.platform', + 'Documentation', + 'swift*')) + + real_dirs = [ + 'Toolchains/XcodeDefault.xctoolchain/usr/bin', + 'usr/bin', + ] + + bins = ['c++', 'c89', 'c99', 'cc', 'clang', 'clang++', 'cpp'] + + for real_dir in real_dirs: + dev_dir = os.path.join(developer_root, + 'Contents', + 'Developer', + real_dir) + for fname in os.listdir(dev_dir): + if fname in bins: + os.unlink(os.path.join(dev_dir, fname)) + os.symlink(os.path.join(spack.build_env_path, 'cc'), + os.path.join(dev_dir, fname)) + + os.symlink(developer_root, xcode_link) + + env.set('DEVELOPER_DIR', xcode_link) diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index dec43726a0..6a67f01d66 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -30,18 +30,47 @@ Configuration file scopes When Spack runs, it pulls configuration data from several config directories, each of which contains configuration files. In Spack, -there are two configuration scopes: - -1. ``site``: Spack loads site-wide configuration options from - ``$(prefix)/etc/spack/``. - -2. ``user``: Spack next loads per-user configuration options from - ``~/.spack/``. - -Spack may read configuration files from both of these locations. When -configurations conflict, the user config options take precedence over -the site configurations. Each configuration directory may contain -several configuration files, such as compilers.yaml or mirrors.yaml. +there are three configuration scopes (lowest to highest): + +1. ``defaults``: Spack loads default configuration settings from + ``$(prefix)/etc/spack/defaults/``. These settings are the "out of the + box" settings Spack will use without site- or user- modification, and + this is where settings that are versioned with Spack should go. + +2. ``site``: This scope affects only this *instance* of Spack, and + overrides the ``defaults`` scope. Configuration files in + ``$(prefix)/etc/spack/`` determine site scope. These can be used for + per-project settings (for users with their own spack instance) or for + site-wide settings (for admins maintaining a common spack instance). + +3. ``user``: User configuration goes in the user's home directory, + specifically in ``~/.spack/``. + +Spack may read configuration files from any of these locations. When +configurations conflict, settings from higher-precedence scopes override +lower-precedence settings. + +fCommands that modify scopes (``spack compilers``, ``spack config``, +etc.) take a ``--scope=<name>`` parameter that you can use to control +which scope is modified. + +For each scope above, there can *also* be platform-specific +overrides. For example, on Blue Gene/Q machines, Spack needs to know the +location of cross-compilers for the compute nodes. This configuration is +in ``etc/spack/defaults/bgq/compilers.yaml``. It will take precedence +over settings in the ``defaults`` scope, but can still be overridden by +settings in ``site``, ``site/bgq``, ``user``, or ``user/bgq``. So, the +full list of scopes and their precedence is: + +1. ``defaults`` +2. ``defaults/<platform>`` +3. ``site`` +4. ``site/<platform>`` +5. ``user`` +6. ``user/<platform>`` + +Each configuration directory may contain several configuration files, +such as compilers.yaml or mirrors.yaml. ========================= Configuration file format @@ -118,6 +147,7 @@ a key in a configuration file. For example, this:: Will make Spack take compilers *only* from the user configuration, and the site configuration will be ignored. + """ import copy @@ -135,6 +165,7 @@ import llnl.util.tty as tty from llnl.util.filesystem import mkdirp import spack +import spack.architecture from spack.error import SpackError import spack.schema @@ -267,16 +298,30 @@ class ConfigScope(object): """Empty cached config information.""" self.sections = {} +# +# Below are configuration scopes. +# +# Each scope can have per-platfom overrides in subdirectories of the +# configuration directory. +# +_platform = spack.architecture.platform().name + """Default configuration scope is the lowest-level scope. These are versioned with Spack and can be overridden by sites or users.""" -ConfigScope('defaults', os.path.join(spack.etc_path, 'spack', 'defaults')) +_defaults_path = os.path.join(spack.etc_path, 'spack', 'defaults') +ConfigScope('defaults', _defaults_path) +ConfigScope('defaults/%s' % _platform, os.path.join(_defaults_path, _platform)) """Site configuration is per spack instance, for sites or projects. No site-level configs should be checked into spack by default.""" -ConfigScope('site', os.path.join(spack.etc_path, 'spack')) +_site_path = os.path.join(spack.etc_path, 'spack') +ConfigScope('site', _site_path) +ConfigScope('site/%s' % _platform, os.path.join(_site_path, _platform)) """User configuration can override both spack defaults and site config.""" -ConfigScope('user', spack.user_config_path) +_user_path = spack.user_config_path +ConfigScope('user', _user_path) +ConfigScope('user/%s' % _platform, os.path.join(_user_path, _platform)) def highest_precedence_scope(): diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index b0eadef7a6..de30a9c7be 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -271,8 +271,12 @@ class EnvironmentModifications(object): env = EnvironmentModifications() # Check if the files are actually there - if not all(os.path.isfile(file) for file in args): - raise RuntimeError('trying to source non-existing files') + files = [line.split(' ')[0] for line in args] + non_existing = [file for file in files if not os.path.isfile(file)] + if non_existing: + message = 'trying to source non-existing files\n' + message += '\n'.join(non_existing) + raise RuntimeError(message) # Relevant kwd parameters and formats info = dict(kwargs) info.setdefault('shell', '/bin/bash') diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 4b8829c32f..2becf9ed04 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -158,12 +158,20 @@ class URLFetchStrategy(FetchStrategy): self.digest = digest self.expand_archive = kwargs.get('expand', True) + self.extra_curl_options = kwargs.get('curl_options', []) + self._curl = None self.extension = kwargs.get('extension', None) if not self.url: raise ValueError("URLFetchStrategy requires a url for fetching.") + @property + def curl(self): + if not self._curl: + self._curl = which('curl', required=True) + return self._curl + @_needs_stage def fetch(self): self.stage.chdir() @@ -196,15 +204,21 @@ class URLFetchStrategy(FetchStrategy): self.url, ] + if spack.insecure: + curl_args.append('-k') + if sys.stdout.isatty(): curl_args.append('-#') # status bar when using a tty else: curl_args.append('-sS') # just errors when not. + curl_args += self.extra_curl_options + # Run curl but grab the mime type from the http headers - headers = spack.curl(*curl_args, output=str, fail_on_error=False) + curl = self.curl + headers = curl(*curl_args, output=str, fail_on_error=False) - if spack.curl.returncode != 0: + if curl.returncode != 0: # clean up archive on failure. if self.archive_file: os.remove(self.archive_file) @@ -212,12 +226,12 @@ class URLFetchStrategy(FetchStrategy): if partial_file and os.path.exists(partial_file): os.remove(partial_file) - if spack.curl.returncode == 22: + if curl.returncode == 22: # This is a 404. Curl will print the error. raise FailedDownloadError( self.url, "URL %s was not found!" % self.url) - elif spack.curl.returncode == 60: + elif curl.returncode == 60: # This is a certificate error. Suggest spack -k raise FailedDownloadError( self.url, @@ -233,7 +247,7 @@ class URLFetchStrategy(FetchStrategy): # error, but print a spack message too raise FailedDownloadError( self.url, - "Curl failed with error %d" % spack.curl.returncode) + "Curl failed with error %d" % curl.returncode) # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle @@ -530,6 +544,12 @@ class GitFetchStrategy(VCSFetchStrategy): def git(self): if not self._git: self._git = which('git', required=True) + + # If the user asked for insecure fetching, make that work + # with git as well. + if spack.insecure: + self._git.add_default_env('GIT_SSL_NO_VERIFY', 'true') + return self._git @_needs_stage diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 7a45988808..3d1f87d599 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -1115,8 +1115,6 @@ class PackageBase(object): even with exceptions. :param install_deps: Install dependencies before installing this \ package - :param install_self: Install this package once dependencies have \ - been installed. :param fake: Don't really build; install fake stub files instead. :param skip_patch: Skip patch stage of build if True. :param verbose: Display verbose build output (by default, suppresses \ @@ -1161,7 +1159,6 @@ class PackageBase(object): keep_prefix=keep_prefix, keep_stage=keep_stage, install_deps=install_deps, - install_self=True, fake=fake, skip_patch=skip_patch, verbose=verbose, @@ -1169,11 +1166,6 @@ class PackageBase(object): run_tests=run_tests, dirty=dirty) - # The rest of this function is to install ourself, - # once deps have been installed. - if not install_self: - return - # Set run_tests flag before starting build. self.run_tests = run_tests diff --git a/lib/spack/spack/platforms/darwin.py b/lib/spack/spack/platforms/darwin.py index 974ce3a3f9..ffefd36546 100644 --- a/lib/spack/spack/platforms/darwin.py +++ b/lib/spack/spack/platforms/darwin.py @@ -1,4 +1,4 @@ -import subprocess +import platform from spack.architecture import Platform, Target from spack.operating_systems.mac_os import MacOs @@ -22,6 +22,4 @@ class Darwin(Platform): @classmethod def detect(self): - platform = subprocess.Popen(['uname', '-a'], stdout=subprocess.PIPE) - platform, _ = platform.communicate() - return 'darwin' in platform.strip().lower() + return 'darwin' in platform.system().lower() diff --git a/lib/spack/spack/platforms/linux.py b/lib/spack/spack/platforms/linux.py index 38d2cdbfec..cee0cfd6e9 100644 --- a/lib/spack/spack/platforms/linux.py +++ b/lib/spack/spack/platforms/linux.py @@ -1,4 +1,3 @@ -import subprocess import platform from spack.architecture import Platform, Target from spack.operating_systems.linux_distro import LinuxDistro @@ -27,6 +26,4 @@ class Linux(Platform): @classmethod def detect(self): - platform = subprocess.Popen(['uname', '-a'], stdout=subprocess.PIPE) - platform, _ = platform.communicate() - return 'linux' in platform.strip().lower() + return 'linux' in platform.system().lower() diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index fc4bf41e34..6950025502 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -98,7 +98,7 @@ expansion when it is the first character in an id typed on the command line. import base64 import hashlib import imp -import sys +import ctypes from StringIO import StringIO from operator import attrgetter @@ -203,6 +203,9 @@ special_types = { legal_deps = tuple(special_types) + alldeps +"""Max integer helps avoid passing too large a value to cyaml.""" +maxint = 2 ** (ctypes.sizeof(ctypes.c_int) * 8 - 1) - 1 + def validate_deptype(deptype): if isinstance(deptype, str): @@ -969,12 +972,12 @@ class Spec(object): return self._hash[:length] else: yaml_text = syaml.dump( - self.to_node_dict(), default_flow_style=True, width=sys.maxint) + self.to_node_dict(), default_flow_style=True, width=maxint) sha = hashlib.sha1(yaml_text) - b32_hash = base64.b32encode(sha.digest()).lower()[:length] + b32_hash = base64.b32encode(sha.digest()).lower() if self.concrete: self._hash = b32_hash - return b32_hash + return b32_hash[:length] def dag_hash_bit_prefix(self, bits): """Get the first <bits> bits of the DAG hash as an integer type.""" diff --git a/lib/spack/spack/test/data/sourceme_parameters.sh b/lib/spack/spack/test/data/sourceme_parameters.sh new file mode 100644 index 0000000000..3826eb092b --- /dev/null +++ b/lib/spack/spack/test/data/sourceme_parameters.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +if [[ "$1" == "intel64" ]] ; then + export FOO='intel64' +else + export FOO='default' +fi
\ No newline at end of file diff --git a/lib/spack/spack/test/environment.py b/lib/spack/spack/test/environment.py index 9b5d75f273..c404433a56 100644 --- a/lib/spack/spack/test/environment.py +++ b/lib/spack/spack/test/environment.py @@ -119,7 +119,8 @@ class EnvironmentTest(unittest.TestCase): 'spack', 'test', 'data') files = [ join_path(datadir, 'sourceme_first.sh'), - join_path(datadir, 'sourceme_second.sh') + join_path(datadir, 'sourceme_second.sh'), + join_path(datadir, 'sourceme_parameters.sh intel64') ] env = EnvironmentModifications.from_sourcing_files(*files) modifications = env.group_by_name() @@ -134,6 +135,11 @@ class EnvironmentTest(unittest.TestCase): self.assertEqual(len(modifications['NEW_VAR']), 1) self.assertTrue(isinstance(modifications['NEW_VAR'][0], SetEnv)) self.assertEqual(modifications['NEW_VAR'][0].value, 'new') + + self.assertEqual(len(modifications['FOO']), 1) + self.assertTrue(isinstance(modifications['FOO'][0], SetEnv)) + self.assertEqual(modifications['FOO'][0].value, 'intel64') + # Unset variables self.assertEqual(len(modifications['EMPTY_PATH_LIST']), 1) self.assertTrue(isinstance( diff --git a/lib/spack/spack/url.py b/lib/spack/spack/url.py index af4b8a51ef..8842495613 100644 --- a/lib/spack/spack/url.py +++ b/lib/spack/spack/url.py @@ -323,7 +323,7 @@ def parse_name_and_version(path): def insensitize(string): """Change upper and lowercase letters to be case insensitive in - the provided string. e.g., 'a' because '[Aa]', 'B' becomes + the provided string. e.g., 'a' becomes '[Aa]', 'B' becomes '[bB]', etc. Use for building regexes.""" def to_ins(match): char = match.group(1) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 4fe4bd26ba..2790508ee8 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -40,6 +40,7 @@ class Executable(object): def __init__(self, name): self.exe = name.split(' ') + self.default_env = {} self.returncode = None if not self.exe: @@ -48,6 +49,9 @@ class Executable(object): def add_default_arg(self, arg): self.exe.append(arg) + def add_default_env(self, key, value): + self.default_env[key] = value + @property def command(self): return ' '.join(self.exe) @@ -103,7 +107,13 @@ class Executable(object): fail_on_error = kwargs.pop("fail_on_error", True) ignore_errors = kwargs.pop("ignore_errors", ()) + # environment env = kwargs.get('env', None) + if env is None: + env = os.environ.copy() + env.update(self.default_env) + else: + env = self.default_env.copy().update(env) # TODO: This is deprecated. Remove in a future version. return_output = kwargs.pop("return_output", False) @@ -149,6 +159,7 @@ class Executable(object): cmd_line = "'%s'" % "' '".join( map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) + tty.debug(cmd_line) try: diff --git a/lib/spack/spack/util/spack_yaml.py b/lib/spack/spack/util/spack_yaml.py index 7bcdf2d61f..506f56633a 100644 --- a/lib/spack/spack/util/spack_yaml.py +++ b/lib/spack/spack/util/spack_yaml.py @@ -32,6 +32,10 @@ """ import yaml +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError as e: + from yaml import Loader, Dumper from yaml.nodes import * from yaml.constructor import ConstructorError from ordereddict_backport import OrderedDict @@ -64,7 +68,7 @@ def mark(obj, node): obj._end_mark = node.end_mark -class OrderedLineLoader(yaml.Loader): +class OrderedLineLoader(Loader): """YAML loader that preserves order and line numbers. Mappings read in by this loader behave like an ordered dict. @@ -156,7 +160,7 @@ OrderedLineLoader.add_constructor( u'tag:yaml.org,2002:str', OrderedLineLoader.construct_yaml_str) -class OrderedLineDumper(yaml.Dumper): +class OrderedLineDumper(Dumper): """Dumper that preserves ordering and formats ``syaml_*`` objects. This dumper preserves insertion ordering ``syaml_dict`` objects diff --git a/lib/spack/spack/util/web.py b/lib/spack/spack/util/web.py index 29ed6e0d32..935532266f 100644 --- a/lib/spack/spack/util/web.py +++ b/lib/spack/spack/util/web.py @@ -227,7 +227,16 @@ def find_versions_of_archive(*archive_urls, **kwargs): # We'll be a bit more liberal and just look for the archive # part, not the full path. - regexes.append(os.path.basename(url_regex)) + url_regex = os.path.basename(url_regex) + + # We need to add a $ anchor to the end of the regex to prevent + # Spack from picking up signature files like: + # .asc + # .md5 + # .sha256 + # .sig + # However, SourceForge downloads still need to end in '/download'. + regexes.append(url_regex + '(\/download)?$') # Build a dict version -> URL from any links that match the wildcards. versions = {} |