`spack find -c`: search all concretized-but-not-installed specs (#44713)

Originally if you had `x -> y -> z`, and an env with `x` in its speclist that is concretized but not installed, then `spack find -c y` would not show anything. This was intended: `spack find` has up-until-now only ever listed out installed specs (and `-c` was for adding a preamble section about roots).

This changes `spack find` so:

* `-c` makes it search through all concretized specs in the env (in a sense it is anticipated that a concretized environment would serve as a "speculative" DB and users may want to query it like they query the DB outside of envs)
* Adds a `-i/--install-status` option, equivalent to `-I` from `spack spec`
* Shows install status for either `-c` or `-i`
* As a side effect to prior point, `spack find -i` can now distinguish different installation states (upstream/external)

Examples:

```
$ spack find -r
==> In environment findtest
==> 1 root specs
 -  raja

==> 6 installed packages (not shown)
==> 12 concretized packages to be installed (not shown)
```

```
$ spack find
==> In environment findtest
==> 1 root specs
 -  raja

-- darwin-ventura-m1 / apple-clang@14.0.3 -----------------------
berkeley-db@18.1.40  bzip2@1.0.8  diffutils@3.10  gmake@4.4.1  gnuconfig@2022-09-17  libiconv@1.17
==> 6 installed packages
==> 12 concretized packages to be installed (show with `spack find -c`)
```

```
$ spack find -c
==> In environment findtest
==> 1 root specs
 -  raja

-- darwin-ventura-m1 / apple-clang@14.0.3 -----------------------
[+]  berkeley-db@18.1.40  [+]  bzip2@1.0.8      -   cmake@3.29.4  [+]  diffutils@3.10  [+]  gmake@4.4.1           [+]  libiconv@1.17   -   nghttp2@1.62.0   -   pkgconf@2.2.0    -   readline@8.2
 -   blt@0.6.2             -   camp@2024.02.1   -   curl@8.7.1     -   gdbm@1.23       [+]  gnuconfig@2022-09-17   -   ncurses@6.5     -   perl@5.38.2      -   raja@2024.02.2   -   zlib-ng@2.1.6
==> 6 installed packages
==> 12 concretized packages to be installed


```
$ spack -E find
...
==> 82 installed packages
```

5 files changed, 62 insertions, 21 deletions

diff --git a/lib/spack/spack/cmd/__init__.py b/lib/spack/spack/cmd/__init__.py
index 023b7ae73e..00e30a551d 100644
--- a/lib/spack/spack/cmd/__init__.py
+++ b/lib/spack/spack/cmd/__init__.py
@@ -336,6 +336,7 @@ def display_specs(specs, args=None, **kwargs):
         groups (bool): display specs grouped by arch/compiler (default True)
         decorator (typing.Callable): function to call to decorate specs
         all_headers (bool): show headers even when arch/compiler aren't defined
+        status_fn (typing.Callable): if provided, prepend install-status info
         output (typing.IO): A file object to write to. Default is ``sys.stdout``
 
     """
@@ -359,6 +360,7 @@ def display_specs(specs, args=None, **kwargs):
     groups = get_arg("groups", True)
     all_headers = get_arg("all_headers", False)
     output = get_arg("output", sys.stdout)
+    status_fn = get_arg("status_fn", None)
 
     decorator = get_arg("decorator", None)
     if decorator is None:
@@ -386,6 +388,13 @@ def display_specs(specs, args=None, **kwargs):
     def fmt(s, depth=0):
         """Formatter function for all output specs"""
         string = ""
+
+        if status_fn:
+            # This was copied from spec.tree's colorization logic
+            # then shortened because it seems like status_fn should
+            # always return an InstallStatus
+            string += colorize(status_fn(s).value)
+
         if hashes:
             string += gray_hash(s, hlen) + " "
         string += depth * "    "
diff --git a/lib/spack/spack/cmd/find.py b/lib/spack/spack/cmd/find.py
index c4e2c77552..d09b2d8423 100644
--- a/lib/spack/spack/cmd/find.py
+++ b/lib/spack/spack/cmd/find.py
@@ -47,6 +47,10 @@ def setup_parser(subparser):
     )
 
     subparser.add_argument(
+        "-I", "--install-status", action="store_true", help="show install status of packages"
+    )
+
+    subparser.add_argument(
         "-d", "--deps", action="store_true", help="output dependencies along with found specs"
     )
 
@@ -293,25 +297,24 @@ def display_env(env, args, decorator, results):
         )
         print()
 
-    if args.show_concretized:
-        tty.msg("Concretized roots")
-        cmd.display_specs(env.specs_by_hash.values(), args, decorator=decorator)
-        print()
-
-    # Display a header for the installed packages section IF there are installed
-    # packages. If there aren't any, we'll just end up printing "0 installed packages"
-    # later.
-    if results and not args.only_roots:
-        tty.msg("Installed packages")
-
 
 def find(parser, args):
-    q_args = query_arguments(args)
-    results = args.specs(**q_args)
-
     env = ev.active_environment()
+
     if not env and args.only_roots:
         tty.die("-r / --only-roots requires an active environment")
+    if not env and args.show_concretized:
+        tty.die("-c / --show-concretized requires an active environment")
+
+    if env:
+        if args.constraint:
+            init_specs = spack.cmd.parse_specs(args.constraint)
+            results = env.all_matching_specs(*init_specs)
+        else:
+            results = env.all_specs()
+    else:
+        q_args = query_arguments(args)
+        results = args.specs(**q_args)
 
     decorator = make_env_decorator(env) if env else lambda s, f: f
 
@@ -332,6 +335,11 @@ def find(parser, args):
     if args.loaded:
         results = spack.cmd.filter_loaded_specs(results)
 
+    if args.install_status or args.show_concretized:
+        status_fn = spack.spec.Spec.install_status
+    else:
+        status_fn = None
+
     # Display the result
     if args.json:
         cmd.display_specs_as_json(results, deps=args.deps)
@@ -340,12 +348,34 @@ def find(parser, args):
             if env:
                 display_env(env, args, decorator, results)
 
-        count_suffix = " (not shown)"
         if not args.only_roots:
-            cmd.display_specs(results, args, decorator=decorator, all_headers=True)
-            count_suffix = ""
+            display_results = results
+            if not args.show_concretized:
+                display_results = list(x for x in results if x.installed)
+            cmd.display_specs(
+                display_results, args, decorator=decorator, all_headers=True, status_fn=status_fn
+            )
 
         # print number of installed packages last (as the list may be long)
         if sys.stdout.isatty() and args.groups:
+            installed_suffix = ""
+            concretized_suffix = " to be installed"
+
+            if args.only_roots:
+                installed_suffix += " (not shown)"
+                concretized_suffix += " (not shown)"
+            else:
+                if env and not args.show_concretized:
+                    concretized_suffix += " (show with `spack find -c`)"
+
             pkg_type = "loaded" if args.loaded else "installed"
-            spack.cmd.print_how_many_pkgs(results, pkg_type, suffix=count_suffix)
+            spack.cmd.print_how_many_pkgs(
+                list(x for x in results if x.installed), pkg_type, suffix=installed_suffix
+            )
+
+            if env:
+                spack.cmd.print_how_many_pkgs(
+                    list(x for x in results if not x.installed),
+                    "concretized",
+                    suffix=concretized_suffix,
+                )
diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py
index 78477a8894..d35163c638 100644
--- a/lib/spack/spack/spec.py
+++ b/lib/spack/spack/spec.py
@@ -4650,7 +4650,7 @@ class Spec:
         spec_str = " ^".join(root_str + sorted_dependencies)
         return spec_str.strip()
 
-    def install_status(self):
+    def install_status(self) -> InstallStatus:
         """Helper for tree to print DB install status."""
         if not self.concrete:
             return InstallStatus.absent
diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash
index 1f73849fc8..02cb496f6d 100755
--- a/share/spack/spack-completion.bash
+++ b/share/spack/spack-completion.bash
@@ -1197,7 +1197,7 @@ _spack_fetch() {
 _spack_find() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --format -H --hashes --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tag -N --namespaces -r --only-roots -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated --install-tree --start-date --end-date"
+        SPACK_COMPREPLY="-h --help --format -H --hashes --json -I --install-status -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tag -N --namespaces -r --only-roots -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated --install-tree --start-date --end-date"
     else
         _installed_packages
     fi
diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish
index f02926e449..54be36e912 100755
--- a/share/spack/spack-completion.fish
+++ b/share/spack/spack-completion.fish
@@ -1753,7 +1753,7 @@ complete -c spack -n '__fish_spack_using_command fetch' -l deprecated -f -a conf
 complete -c spack -n '__fish_spack_using_command fetch' -l deprecated -d 'allow concretizer to select deprecated versions'
 
 # spack find
-set -g __fish_spack_optspecs_spack_find h/help format= H/hashes json d/deps p/paths groups no-groups l/long L/very-long t/tag= N/namespaces r/only-roots c/show-concretized f/show-flags show-full-compiler x/explicit X/implicit u/unknown m/missing v/variants loaded M/only-missing deprecated only-deprecated install-tree= start-date= end-date=
+set -g __fish_spack_optspecs_spack_find h/help format= H/hashes json I/install-status d/deps p/paths groups no-groups l/long L/very-long t/tag= N/namespaces r/only-roots c/show-concretized f/show-flags show-full-compiler x/explicit X/implicit u/unknown m/missing v/variants loaded M/only-missing deprecated only-deprecated install-tree= start-date= end-date=
 complete -c spack -n '__fish_spack_using_command_pos_remainder 0 find' -f -a '(__fish_spack_installed_specs)'
 complete -c spack -n '__fish_spack_using_command find' -s h -l help -f -a help
 complete -c spack -n '__fish_spack_using_command find' -s h -l help -d 'show this help message and exit'
@@ -1763,6 +1763,8 @@ complete -c spack -n '__fish_spack_using_command find' -s H -l hashes -f -a form
 complete -c spack -n '__fish_spack_using_command find' -s H -l hashes -d 'same as \'--format {/hash}\'; use with xargs or $()'
 complete -c spack -n '__fish_spack_using_command find' -l json -f -a json
 complete -c spack -n '__fish_spack_using_command find' -l json -d 'output specs as machine-readable json records'
+complete -c spack -n '__fish_spack_using_command find' -s I -l install-status -f -a install_status
+complete -c spack -n '__fish_spack_using_command find' -s I -l install-status -d 'show install status of packages'
 complete -c spack -n '__fish_spack_using_command find' -s d -l deps -f -a deps
 complete -c spack -n '__fish_spack_using_command find' -s d -l deps -d 'output dependencies along with found specs'
 complete -c spack -n '__fish_spack_using_command find' -s p -l paths -f -a paths