From d1e03329c5c16ba38b082c3473e7c6970f168990 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Sun, 15 Feb 2015 11:50:13 -0800 Subject: Memoize all_specs() and exists() for better performance. - Real bottleneck is calling normalize() for every spec when we read it. - Need to store graph information in spec files to avoid the need for this. - Also, normalizing old specs isn't always possible, so we need to do this anyway. --- lib/spack/spack/directory_layout.py | 11 +++++++++-- lib/spack/spack/packages.py | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/spack/spack/directory_layout.py b/lib/spack/spack/directory_layout.py index 562c0bd3ed..5b80e93d6b 100644 --- a/lib/spack/spack/directory_layout.py +++ b/lib/spack/spack/directory_layout.py @@ -31,6 +31,7 @@ import tempfile from contextlib import closing import llnl.util.tty as tty +from llnl.util.lang import memoized from llnl.util.filesystem import join_path, mkdirp import spack @@ -223,6 +224,9 @@ class SpecHashDirectoryLayout(DirectoryLayout): if all(spack.db.exists(s.name) for s in spec.traverse()): copy = spec.copy() + + # TODO: It takes a lot of time to normalize every spec on read. + # TODO: Storing graph info with spec files would fix this. copy.normalize() if copy.concrete: return copy # These are specs spack still understands. @@ -276,17 +280,20 @@ class SpecHashDirectoryLayout(DirectoryLayout): self.write_spec(spec, spec_file_path) + @memoized def all_specs(self): if not os.path.isdir(self.root): - return + return [] + specs = [] for path in traverse_dirs_at_depth(self.root, 3): arch, compiler, last_dir = path spec_file_path = join_path( self.root, arch, compiler, last_dir, self.spec_file_name) if os.path.exists(spec_file_path): spec = self.read_spec(spec_file_path) - yield spec + specs.append(spec) + return specs def extension_file_path(self, spec): diff --git a/lib/spack/spack/packages.py b/lib/spack/spack/packages.py index 3c81863c11..43c4c191c1 100644 --- a/lib/spack/spack/packages.py +++ b/lib/spack/spack/packages.py @@ -192,6 +192,7 @@ class PackageDB(object): yield self.get(name) + @memoized def exists(self, pkg_name): """Whether a package with the supplied name exists .""" return os.path.exists(self.filename_for_package_name(pkg_name)) -- cgit v1.2.3-70-g09d2