add `spack dev-build` command; deprecate `spack diy` (#13374)

Rename the `spack diy` command to `spack dev-build` to make the use case clearer.

The `spack diy` command has some useful functionality for developers using Spack to build their dependencies and configure/build/install the code they are developing.  Developers do not notice it, partly because of the obscure name.

The `spack dev-build` command has a `-u/--until PHASE` option to stop after a given phase of the build. This can be used to configure your project, run cmake on your project, or similarly stop after any stage of the build the user wants. These options are analogous to the existing `spack configure` and `spack build` commands, but for developer builds.

To unify the syntax, we have deprecated the `spack configure` and `spack build` commands, and added a `-u/--until PHASE` option to the `spack install` command as well.

The functionality in `spack dev-build` (specifically `spack dev-build -u cmake`) may be able to supersede the `spack setup` command, but this PR does not deprecate that command as that will require slightly more thought.

7 files changed, 216 insertions, 86 deletions

diff --git a/lib/spack/spack/cmd/build.py b/lib/spack/spack/cmd/build.py
index 84d8221090..8687e9a741 100644
--- a/lib/spack/spack/cmd/build.py
+++ b/lib/spack/spack/cmd/build.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import spack.cmd.configure as cfg
+import llnl.util.tty as tty
 
 from spack.build_systems.autotools import AutotoolsPackage
 from spack.build_systems.cmake import CMakePackage
@@ -15,7 +16,7 @@ from spack.build_systems.perl import PerlPackage
 from spack.build_systems.meson import MesonPackage
 from spack.build_systems.sip import SIPPackage
 
-description = 'stops at build stage when installing a package, if possible'
+description = 'DEPRECATED: stops at build stage when installing a package'
 section = "build"
 level = "long"
 
@@ -38,4 +39,7 @@ def setup_parser(subparser):
 
 
 def build(parser, args):
+    tty.warn("This command is deprecated. Use `spack install --until` to"
+             " select an end phase instead. The `spack build` command will be"
+             " removed in a future version of Spack")
     cfg._stop_at_phase_during_install(args, build, build_system_to_phase)
diff --git a/lib/spack/spack/cmd/configure.py b/lib/spack/spack/cmd/configure.py
index b06d4edf1c..10a1294a41 100644
--- a/lib/spack/spack/cmd/configure.py
+++ b/lib/spack/spack/cmd/configure.py
@@ -18,7 +18,7 @@ from spack.build_systems.intel import IntelPackage
 from spack.build_systems.meson import MesonPackage
 from spack.build_systems.sip import SIPPackage
 
-description = 'stage and configure a package but do not install'
+description = 'DEPRECATED: stage and configure a package but do not install'
 section = "build"
 level = "long"
 
@@ -82,4 +82,7 @@ def _stop_at_phase_during_install(args, calling_fn, phase_mapping):
 
 
 def configure(parser, args):
+    tty.warn("This command is deprecated. Use `spack install --until` to"
+             " select an end phase instead. The `spack configure` command will"
+             " be removed in a future version of Spack.")
     _stop_at_phase_during_install(args, configure, build_system_to_phase)
diff --git a/lib/spack/spack/cmd/dev_build.py b/lib/spack/spack/cmd/dev_build.py
new file mode 100644
index 0000000000..7c92b004ca
--- /dev/null
+++ b/lib/spack/spack/cmd/dev_build.py
@@ -0,0 +1,97 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import sys
+import os
+import argparse
+
+import llnl.util.tty as tty
+
+import spack.config
+import spack.cmd
+import spack.repo
+import spack.cmd.common.arguments as arguments
+from spack.stage import DIYStage
+
+description = "developer build: build from code in current working directory"
+section = "build"
+level = "long"
+
+
+def setup_parser(subparser):
+    arguments.add_common_arguments(subparser, ['jobs'])
+    subparser.add_argument(
+        '-d', '--source-path', dest='source_path', default=None,
+        help="path to source directory. defaults to the current directory")
+    subparser.add_argument(
+        '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps',
+        help="don't try to install dependencies of requested packages")
+    arguments.add_common_arguments(subparser, ['no_checksum'])
+    subparser.add_argument(
+        '--keep-prefix', action='store_true',
+        help="do not remove the install prefix if installation fails")
+    subparser.add_argument(
+        '--skip-patch', action='store_true',
+        help="skip patching for the developer build")
+    subparser.add_argument(
+        '-q', '--quiet', action='store_true', dest='quiet',
+        help="do not display verbose build output while installing")
+    subparser.add_argument(
+        '-u', '--until', type=str, dest='until', default=None,
+        help="phase to stop after when installing (default None)")
+    subparser.add_argument(
+        'spec', nargs=argparse.REMAINDER,
+        help="specs to use for install. must contain package AND version")
+
+    cd_group = subparser.add_mutually_exclusive_group()
+    arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
+
+
+def dev_build(self, args):
+    if not args.spec:
+        tty.die("spack dev-build requires a package spec argument.")
+
+    specs = spack.cmd.parse_specs(args.spec)
+    if len(specs) > 1:
+        tty.die("spack dev-build only takes one spec.")
+
+    spec = specs[0]
+    if not spack.repo.path.exists(spec.name):
+        tty.die("No package for '{0}' was found.".format(spec.name),
+                "  Use `spack create` to create a new package")
+
+    if not spec.versions.concrete:
+        tty.die(
+            "spack dev-build spec must have a single, concrete version. "
+            "Did you forget a package version number?")
+
+    spec.concretize()
+    package = spack.repo.get(spec)
+
+    if package.installed:
+        tty.error("Already installed in %s" % package.prefix)
+        tty.msg("Uninstall or try adding a version suffix for this dev build.")
+        sys.exit(1)
+
+    source_path = args.source_path
+    if source_path is None:
+        source_path = os.getcwd()
+    source_path = os.path.abspath(source_path)
+
+    # Forces the build to run out of the current directory.
+    package.stage = DIYStage(source_path)
+
+    # disable checksumming if requested
+    if args.no_checksum:
+        spack.config.set('config:checksum', False, scope='command_line')
+
+    package.do_install(
+        make_jobs=args.jobs,
+        keep_prefix=args.keep_prefix,
+        install_deps=not args.ignore_deps,
+        verbose=not args.quiet,
+        keep_stage=True,   # don't remove source dir for dev build.
+        dirty=args.dirty,
+        stop_at=args.until)
diff --git a/lib/spack/spack/cmd/diy.py b/lib/spack/spack/cmd/diy.py
index 20e2206657..127a6bbed1 100644
--- a/lib/spack/spack/cmd/diy.py
+++ b/lib/spack/spack/cmd/diy.py
@@ -2,96 +2,19 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
-import sys
-import os
-import argparse
-
+import spack.cmd.dev_build
 import llnl.util.tty as tty
 
-import spack.config
-import spack.cmd
-import spack.repo
-import spack.cmd.common.arguments as arguments
-from spack.stage import DIYStage
-
-description = "do-it-yourself: build from an existing source directory"
+description = "DEPRECATED: do-it-yourself: build from local source directory"
 section = "build"
 level = "long"
 
 
 def setup_parser(subparser):
-    arguments.add_common_arguments(subparser, ['jobs'])
-    subparser.add_argument(
-        '-d', '--source-path', dest='source_path', default=None,
-        help="path to source directory. defaults to the current directory")
-    subparser.add_argument(
-        '-i', '--ignore-dependencies', action='store_true', dest='ignore_deps',
-        help="don't try to install dependencies of requested packages")
-    arguments.add_common_arguments(subparser, ['no_checksum'])
-    subparser.add_argument(
-        '--keep-prefix', action='store_true',
-        help="do not remove the install prefix if installation fails")
-    subparser.add_argument(
-        '--skip-patch', action='store_true',
-        help="skip patching for the DIY build")
-    subparser.add_argument(
-        '-q', '--quiet', action='store_true', dest='quiet',
-        help="do not display verbose build output while installing")
-    subparser.add_argument(
-        '-u', '--until', type=str, dest='until', default=None,
-        help="phase to stop after when installing (default None)")
-    subparser.add_argument(
-        'spec', nargs=argparse.REMAINDER,
-        help="specs to use for install. must contain package AND version")
-
-    cd_group = subparser.add_mutually_exclusive_group()
-    arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
+    spack.cmd.dev_build.setup_parser(subparser)
 
 
 def diy(self, args):
-    if not args.spec:
-        tty.die("spack diy requires a package spec argument.")
-
-    specs = spack.cmd.parse_specs(args.spec)
-    if len(specs) > 1:
-        tty.die("spack diy only takes one spec.")
-
-    spec = specs[0]
-    if not spack.repo.path.exists(spec.name):
-        tty.die("No package for '{0}' was found.".format(spec.name),
-                "  Use `spack create` to create a new package")
-
-    if not spec.versions.concrete:
-        tty.die(
-            "spack diy spec must have a single, concrete version. "
-            "Did you forget a package version number?")
-
-    spec.concretize()
-    package = spack.repo.get(spec)
-
-    if package.installed:
-        tty.error("Already installed in %s" % package.prefix)
-        tty.msg("Uninstall or try adding a version suffix for this DIY build.")
-        sys.exit(1)
-
-    source_path = args.source_path
-    if source_path is None:
-        source_path = os.getcwd()
-    source_path = os.path.abspath(source_path)
-
-    # Forces the build to run out of the current directory.
-    package.stage = DIYStage(source_path)
-
-    # disable checksumming if requested
-    if args.no_checksum:
-        spack.config.set('config:checksum', False, scope='command_line')
-
-    package.do_install(
-        make_jobs=args.jobs,
-        keep_prefix=args.keep_prefix,
-        install_deps=not args.ignore_deps,
-        verbose=not args.quiet,
-        keep_stage=True,   # don't remove source dir for DIY.
-        dirty=args.dirty,
-        stop_at=args.until)
+    tty.warn("`spack diy` has been renamed to `spack dev-build`."
+             "The `diy` command will be removed in a future version of Spack")
+    spack.cmd.dev_build.dev_build(self, args)
diff --git a/lib/spack/spack/cmd/install.py b/lib/spack/spack/cmd/install.py
index 876fb58f51..8fd63beede 100644
--- a/lib/spack/spack/cmd/install.py
+++ b/lib/spack/spack/cmd/install.py
@@ -40,7 +40,8 @@ def update_kwargs_from_args(args, kwargs):
         'dirty': args.dirty,
         'use_cache': args.use_cache,
         'cache_only': args.cache_only,
-        'explicit': True  # Always true for install command
+        'explicit': True,  # Always true for install command
+        'stop_at': args.until
     })
 
     kwargs.update({
@@ -68,6 +69,9 @@ the default is to install the package along with all its dependencies.
 alternatively one can decide to install only the package or only
 the dependencies"""
     )
+    subparser.add_argument(
+        '-u', '--until', type=str, dest='until', default=None,
+        help="phase to stop after when installing (default None)")
     arguments.add_common_arguments(subparser, ['jobs', 'install_status'])
     subparser.add_argument(
         '--overwrite', action='store_true',
diff --git a/lib/spack/spack/test/cmd/dev_build.py b/lib/spack/spack/test/cmd/dev_build.py
new file mode 100644
index 0000000000..835edac138
--- /dev/null
+++ b/lib/spack/spack/test/cmd/dev_build.py
@@ -0,0 +1,72 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import os
+import spack.spec
+from spack.main import SpackCommand
+
+dev_build = SpackCommand('dev-build')
+
+
+def test_dev_build_basics(tmpdir, mock_packages, install_mockery):
+    spec = spack.spec.Spec('dev-build-test-install@0.0.0').concretized()
+
+    with tmpdir.as_cwd():
+        with open(spec.package.filename, 'w') as f:
+            f.write(spec.package.original_string)
+
+        dev_build('dev-build-test-install@0.0.0')
+
+    assert spec.package.filename in os.listdir(spec.prefix)
+    with open(os.path.join(spec.prefix, spec.package.filename), 'r') as f:
+        assert f.read() == spec.package.replacement_string
+
+
+def test_dev_build_until(tmpdir, mock_packages, install_mockery):
+    spec = spack.spec.Spec('dev-build-test-install@0.0.0').concretized()
+
+    with tmpdir.as_cwd():
+        with open(spec.package.filename, 'w') as f:
+            f.write(spec.package.original_string)
+
+        dev_build('-u', 'edit', 'dev-build-test-install@0.0.0')
+
+        assert spec.package.filename in os.listdir(os.getcwd())
+        with open(spec.package.filename, 'r') as f:
+            assert f.read() == spec.package.replacement_string
+
+    assert not os.path.exists(spec.prefix)
+
+
+def test_dev_build_fails_already_installed(tmpdir, mock_packages,
+                                           install_mockery):
+    spec = spack.spec.Spec('dev-build-test-install@0.0.0').concretized()
+
+    with tmpdir.as_cwd():
+        with open(spec.package.filename, 'w') as f:
+            f.write(spec.package.original_string)
+
+        dev_build('dev-build-test-install@0.0.0')
+        output = dev_build('dev-build-test-install@0.0.0', fail_on_error=False)
+        assert 'Already installed in %s' % spec.prefix in output
+
+
+def test_dev_build_fails_no_spec():
+    output = dev_build(fail_on_error=False)
+    assert 'requires a package spec argument' in output
+
+
+def test_dev_build_fails_multiple_specs(mock_packages):
+    output = dev_build('libelf', 'libdwarf', fail_on_error=False)
+    assert 'only takes one spec' in output
+
+
+def test_dev_build_fails_nonexistent_package_name(mock_packages):
+    output = dev_build('no_such_package', fail_on_error=False)
+    assert "No package for 'no_such_package' was found" in output
+
+
+def test_dev_build_fails_no_version(mock_packages):
+    output = dev_build('dev-build-test-install', fail_on_error=False)
+    assert 'dev-build spec must have a single, concrete version' in output
diff --git a/var/spack/repos/builtin.mock/packages/dev-build-test-install/package.py b/var/spack/repos/builtin.mock/packages/dev-build-test-install/package.py
new file mode 100644
index 0000000000..0eb9648941
--- /dev/null
+++ b/var/spack/repos/builtin.mock/packages/dev-build-test-install/package.py
@@ -0,0 +1,27 @@
+# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+
+class DevBuildTestInstall(Package):
+    homepage = "example.com"
+    url = "fake.com"
+
+    version('0.0.0', sha256='0123456789abcdefgh')
+
+    phases = ['edit', 'install']
+
+    filename = 'dev-build-test-file.txt'
+    original_string = "This file should be edited"
+    replacement_string = "This file has been edited"
+
+    def edit(self, spec, prefix):
+        with open(self.filename, 'r+') as f:
+            assert f.read() == self.original_string
+            f.seek(0)
+            f.truncate()
+            f.write(self.replacement_string)
+
+    def install(self, spec, prefix):
+        install(self.filename, prefix)