From 045a5e80cb546b08d89cf53914a7c67c5aa25665 Mon Sep 17 00:00:00 2001 From: psakievich Date: Wed, 14 Sep 2022 11:16:28 -0600 Subject: Allow version to accept the '=' token without activating lexer switch (#32257) --- lib/spack/spack/spec.py | 44 +++++++----- lib/spack/spack/test/spec_syntax.py | 136 ++++++++++++++++++++++++++---------- 2 files changed, 126 insertions(+), 54 deletions(-) diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index b7ff676fe2..09769e4860 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -4981,7 +4981,7 @@ class LazySpecCache(collections.defaultdict): #: These are possible token types in the spec grammar. -HASH, DEP, AT, COLON, COMMA, ON, OFF, PCT, EQ, ID, VAL, FILE = range(12) +HASH, DEP, VER, COLON, COMMA, ON, OFF, PCT, EQ, ID, VAL, FILE = range(12) #: Regex for fully qualified spec names. (e.g., builtin.hdf5) spec_id_re = r"\w[\w.-]*" @@ -5001,10 +5001,13 @@ class SpecLexer(spack.parse.Lexer): ) super(SpecLexer, self).__init__( [ - (r"\^", lambda scanner, val: self.token(DEP, val)), - (r"\@", lambda scanner, val: self.token(AT, val)), + ( + r"\@([\w.\-]*\s*)*(\s*\=\s*\w[\w.\-]*)?", + lambda scanner, val: self.token(VER, val), + ), (r"\:", lambda scanner, val: self.token(COLON, val)), (r"\,", lambda scanner, val: self.token(COMMA, val)), + (r"\^", lambda scanner, val: self.token(DEP, val)), (r"\+", lambda scanner, val: self.token(ON, val)), (r"\-", lambda scanner, val: self.token(OFF, val)), (r"\~", lambda scanner, val: self.token(OFF, val)), @@ -5142,7 +5145,7 @@ class SpecParser(spack.parse.Parser): else: # If the next token can be part of a valid anonymous spec, # create the anonymous spec - if self.next.type in (AT, ON, OFF, PCT): + if self.next.type in (VER, ON, OFF, PCT): # Raise an error if the previous spec is already concrete if specs and specs[-1].concrete: raise RedundantSpecError(specs[-1], "compiler, version, " "or variant") @@ -5250,7 +5253,7 @@ class SpecParser(spack.parse.Parser): spec.name = spec_name while self.next: - if self.accept(AT): + if self.accept(VER): vlist = self.version_list() spec._add_versions(vlist) @@ -5268,7 +5271,6 @@ class SpecParser(spack.parse.Parser): elif self.accept(ID): self.previous = self.token if self.accept(EQ): - # We're adding a key-value pair to the spec self.expect(VAL) spec._add_flag(self.previous.value, self.token.value) self.previous = None @@ -5304,16 +5306,24 @@ class SpecParser(spack.parse.Parser): return self.token.value def version(self): + start = None end = None - if self.accept(ID): - start = self.token.value - if self.accept(EQ): - # This is for versions that are associated with a hash - # i.e. @[40 char hash]=version - start += self.token.value - self.expect(VAL) - start += self.token.value + + def str_translate(value): + # return None for empty strings since we can end up with `'@'.strip('@')` + if not (value and value.strip()): + return None + else: + return value + + if self.token.type is COMMA: + # need to increment commas, could be ID or COLON + self.accept(ID) + + if self.token.type in (VER, ID): + version_spec = self.token.value.lstrip("@") + start = str_translate(version_spec) if self.accept(COLON): if self.accept(ID): @@ -5323,10 +5333,10 @@ class SpecParser(spack.parse.Parser): else: end = self.token.value elif start: - # No colon, but there was a version. + # No colon, but there was a version return vn.Version(start) else: - # No colon and no id: invalid version. + # No colon and no id: invalid version self.next_token_error("Invalid version specifier") if start: @@ -5349,7 +5359,7 @@ class SpecParser(spack.parse.Parser): compiler = CompilerSpec.__new__(CompilerSpec) compiler.name = self.token.value compiler.versions = vn.VersionList() - if self.accept(AT): + if self.accept(VER): vlist = self.version_list() compiler._add_versions(vlist) else: diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index ec92cc877e..41a75a3fcb 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -31,63 +31,97 @@ from spack.spec import ( ) from spack.variant import DuplicateVariantError -# Sample output for a complex lexing. -complex_lex = [ +# Building blocks for complex lexing. +complex_root = [ Token(sp.ID, "mvapich_foo"), - Token(sp.DEP), - Token(sp.ID, "_openmpi"), - Token(sp.AT), - Token(sp.ID, "1.2"), - Token(sp.COLON), - Token(sp.ID, "1.4"), - Token(sp.COMMA), - Token(sp.ID, "1.6"), +] + +kv_root = [ + Token(sp.ID, "mvapich_foo"), + Token(sp.ID, "debug"), + Token(sp.EQ), + Token(sp.VAL, "4"), +] + +complex_compiler = [ Token(sp.PCT), Token(sp.ID, "intel"), - Token(sp.AT), +] + +complex_compiler_v = [ + Token(sp.VER, "@12.1"), + Token(sp.COLON), + Token(sp.ID, "12.6"), +] + +complex_compiler_v_space = [ + Token(sp.VER, "@"), Token(sp.ID, "12.1"), Token(sp.COLON), Token(sp.ID, "12.6"), - Token(sp.ON), - Token(sp.ID, "debug"), - Token(sp.OFF), - Token(sp.ID, "qt_4"), +] + +complex_dep1 = [ Token(sp.DEP), - Token(sp.ID, "stackwalker"), - Token(sp.AT), - Token(sp.ID, "8.1_1e"), + Token(sp.ID, "_openmpi"), + Token(sp.VER, "@1.2"), + Token(sp.COLON), + Token(sp.ID, "1.4"), + Token(sp.COMMA), + Token(sp.ID, "1.6"), ] -# Another sample lexer output with a kv pair. -kv_lex = [ - Token(sp.ID, "mvapich_foo"), - Token(sp.ID, "debug"), - Token(sp.EQ), - Token(sp.VAL, "4"), +complex_dep1_space = [ Token(sp.DEP), Token(sp.ID, "_openmpi"), - Token(sp.AT), + Token(sp.VER, "@"), Token(sp.ID, "1.2"), Token(sp.COLON), Token(sp.ID, "1.4"), Token(sp.COMMA), Token(sp.ID, "1.6"), - Token(sp.PCT), - Token(sp.ID, "intel"), - Token(sp.AT), - Token(sp.ID, "12.1"), - Token(sp.COLON), - Token(sp.ID, "12.6"), +] + +complex_dep1_var = [ Token(sp.ON), Token(sp.ID, "debug"), Token(sp.OFF), Token(sp.ID, "qt_4"), +] + +complex_dep2 = [ + Token(sp.DEP), + Token(sp.ID, "stackwalker"), + Token(sp.VER, "@8.1_1e"), +] + +complex_dep2_space = [ Token(sp.DEP), Token(sp.ID, "stackwalker"), - Token(sp.AT), + Token(sp.VER, "@"), Token(sp.ID, "8.1_1e"), ] +# Sample output from complex lexing +complex_lex = ( + complex_root + + complex_dep1 + + complex_compiler + + complex_compiler_v + + complex_dep1_var + + complex_dep2 +) + +# Another sample lexer output with a kv pair. +kv_lex = ( + kv_root + + complex_dep1 + + complex_compiler + + complex_compiler_v_space + + complex_dep1_var + + complex_dep2_space +) + class TestSpecSyntax(object): # ======================================================================== @@ -120,7 +154,7 @@ class TestSpecSyntax(object): lex_output = sp.SpecLexer().lex(spec) assert len(tokens) == len(lex_output), "unexpected number of tokens" for tok, spec_tok in zip(tokens, lex_output): - if tok.type == sp.ID or tok.type == sp.VAL: + if tok.type in (sp.ID, sp.VAL, sp.VER): assert tok == spec_tok else: # Only check the type for non-identifiers. @@ -716,14 +750,22 @@ class TestSpecSyntax(object): ) def test_spaces_between_dependences(self): + lex_key = ( + complex_root + + complex_dep1 + + complex_compiler + + complex_compiler_v + + complex_dep1_var + + complex_dep2_space + ) self.check_lex( - complex_lex, + lex_key, "mvapich_foo " "^_openmpi@1.2:1.4,1.6%intel@12.1:12.6+debug -qt_4 " "^stackwalker @ 8.1_1e", ) self.check_lex( - complex_lex, + lex_key, "mvapich_foo " "^_openmpi@1.2:1.4,1.6%intel@12.1:12.6+debug~qt_4 " "^stackwalker @ 8.1_1e", @@ -738,14 +780,30 @@ class TestSpecSyntax(object): ) def test_way_too_many_spaces(self): + lex_key = ( + complex_root + + complex_dep1 + + complex_compiler + + complex_compiler_v_space + + complex_dep1_var + + complex_dep2_space + ) self.check_lex( - complex_lex, + lex_key, "mvapich_foo " "^ _openmpi @1.2 : 1.4 , 1.6 % intel @ 12.1 : 12.6 + debug - qt_4 " "^ stackwalker @ 8.1_1e", ) + lex_key = ( + complex_root + + complex_dep1 + + complex_compiler + + complex_compiler_v_space + + complex_dep1_var + + complex_dep2_space + ) self.check_lex( - complex_lex, + lex_key, "mvapich_foo " "^ _openmpi @1.2 : 1.4 , 1.6 % intel @ 12.1 : 12.6 + debug ~ qt_4 " "^ stackwalker @ 8.1_1e", @@ -838,6 +896,10 @@ class TestSpecSyntax(object): # Check that we can compare without raising an error assert a <= b or b < a + def test_git_ref_specs_with_variants(self): + spec_str = "develop-branch-version@git.{h}=develop+var1+var2".format(h="a" * 40) + self.check_parse(spec_str) + def test_git_ref_spec_equivalences(self, mock_packages, mock_stage): s1 = sp.Spec("develop-branch-version@git.{hash}=develop".format(hash="a" * 40)) s2 = sp.Spec("develop-branch-version@git.{hash}=develop".format(hash="b" * 40)) -- cgit v1.2.3-70-g09d2