From 2379ed54b9e8183bec8487c08d66e5f4c51eeb64 Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Tue, 6 Feb 2018 10:48:58 -0500 Subject: package_hash: add code to generate a hash for a package file This will be included in the full hash of packages. --- lib/spack/spack/test/package_hash.py | 84 +++++++++++++++++++ lib/spack/spack/util/package_hash.py | 151 +++++++++++++++++++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 lib/spack/spack/test/package_hash.py create mode 100644 lib/spack/spack/util/package_hash.py (limited to 'lib') diff --git a/lib/spack/spack/test/package_hash.py b/lib/spack/spack/test/package_hash.py new file mode 100644 index 0000000000..77768bb648 --- /dev/null +++ b/lib/spack/spack/test/package_hash.py @@ -0,0 +1,84 @@ +############################################################################## +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://software.llnl.gov/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from spack.util.package_hash import package_hash, package_content +from spack.spec import Spec + + +def test_hash(tmpdir, builtin_mock, config): + package_hash("hash-test1@1.2") + + +def test_different_variants(tmpdir, builtin_mock, config): + spec1 = Spec("hash-test1@1.2 +variantx") + spec2 = Spec("hash-test1@1.2 +varianty") + assert package_hash(spec1) == package_hash(spec2) + + +def test_all_same_but_name(tmpdir, builtin_mock, config): + spec1 = Spec("hash-test1@1.2") + spec2 = Spec("hash-test2@1.2") + compare_sans_name(True, spec1, spec2) + + spec1 = Spec("hash-test1@1.2 +varianty") + spec2 = Spec("hash-test2@1.2 +varianty") + compare_sans_name(True, spec1, spec2) + + +def test_all_same_but_archive_hash(tmpdir, builtin_mock, config): + """ + Archive hash is not intended to be reflected in Package hash. + """ + spec1 = Spec("hash-test1@1.3") + spec2 = Spec("hash-test2@1.3") + compare_sans_name(True, spec1, spec2) + + +def test_all_same_but_patch_contents(tmpdir, builtin_mock, config): + spec1 = Spec("hash-test1@1.1") + spec2 = Spec("hash-test2@1.1") + compare_sans_name(True, spec1, spec2) + + +def test_all_same_but_patches_to_apply(tmpdir, builtin_mock, config): + spec1 = Spec("hash-test1@1.4") + spec2 = Spec("hash-test2@1.4") + compare_sans_name(True, spec1, spec2) + + +def test_all_same_but_install(tmpdir, builtin_mock, config): + spec1 = Spec("hash-test1@1.5") + spec2 = Spec("hash-test2@1.5") + compare_sans_name(False, spec1, spec2) + + +def compare_sans_name(eq, spec1, spec2): + content1 = package_content(spec1) + content1 = content1.replace(spec1.package.__class__.__name__, '') + content2 = package_content(spec2) + content2 = content2.replace(spec2.package.__class__.__name__, '') + if eq: + assert content1 == content2 + else: + assert content1 != content2 diff --git a/lib/spack/spack/util/package_hash.py b/lib/spack/spack/util/package_hash.py new file mode 100644 index 0000000000..6362d9d7bb --- /dev/null +++ b/lib/spack/spack/util/package_hash.py @@ -0,0 +1,151 @@ +############################################################################## +# Copyright (c) 2016, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://software.llnl.gov/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +import spack +from spack import directives +from spack.error import SpackError +from spack.spec import Spec +from spack.util.naming import mod_to_class + +import ast +import hashlib + + +class RemoveDocstrings(ast.NodeTransformer): + """Transformer that removes docstrings from a Python AST.""" + def remove_docstring(self, node): + if node.body: + if isinstance(node.body[0], ast.Expr) and \ + isinstance(node.body[0].value, ast.Str): + node.body.pop(0) + + self.generic_visit(node) + return node + + def visit_FunctionDef(self, node): + return self.remove_docstring(node) + + def visit_ClassDef(self, node): + return self.remove_docstring(node) + + def visit_Module(self, node): + return self.remove_docstring(node) + + +class RemoveDirectives(ast.NodeTransformer): + """Remove Spack directives from a package AST.""" + def __init__(self, spec): + self.spec = spec + + def is_directive(self, node): + return (isinstance(node, ast.Expr) and + node.value and isinstance(node.value, ast.Call) and + node.value.func.id in directives.__all__) + + def is_spack_attr(self, node): + return (isinstance(node, ast.Assign) and + node.targets and isinstance(node.targets[0], ast.Name) and + node.targets[0].id in spack.Package.metadata_attrs) + + def visit_ClassDef(self, node): + if node.name == mod_to_class(self.spec.name): + node.body = [ + c for c in node.body + if (not self.is_directive(c) and not self.is_spack_attr(c))] + return node + + +class TagMultiMethods(ast.NodeVisitor): + """Tag @when-decorated methods in a spec.""" + def __init__(self, spec): + self.spec = spec + self.methods = {} + + def visit_FunctionDef(self, node): + nodes = self.methods.setdefault(node.name, []) + if node.decorator_list: + dec = node.decorator_list[0] + if isinstance(dec, ast.Call) and dec.func.id == 'when': + cond = dec.args[0].s + nodes.append((node, self.spec.satisfies(cond, strict=True))) + else: + nodes.append((node, None)) + + +class ResolveMultiMethods(ast.NodeTransformer): + """Remove methods which do not exist if their @when is not satisfied.""" + def __init__(self, methods): + self.methods = methods + + def resolve(self, node): + if node.name not in self.methods: + raise PackageHashError( + "Future traversal visited new node: %s" % node.name) + + result = None + for n, cond in self.methods[node.name]: + if cond: + return n + if cond is None: + result = n + return result + + def visit_FunctionDef(self, node): + if self.resolve(node) is node: + node.decorator_list = [] + return node + return None + + +def package_content(spec): + return ast.dump(package_ast(spec)) + + +def package_hash(spec, content=None): + if content is None: + content = package_content(spec) + return hashlib.sha256(content.encode('utf-8')).digest().lower() + + +def package_ast(spec): + spec = Spec(spec) + + filename = spack.repo.filename_for_package_name(spec.name) + with open(filename) as f: + text = f.read() + root = ast.parse(text) + + root = RemoveDocstrings().visit(root) + + RemoveDirectives(spec).visit(root) + + fmm = TagMultiMethods(spec) + fmm.visit(root) + + root = ResolveMultiMethods(fmm.methods).visit(root) + return root + + +class PackageHashError(SpackError): + """Raised for all errors encountered during package hashing.""" -- cgit v1.2.3-70-g09d2