From a22686279c6e45729c0ce6dc4aee0df2d280b513 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Wed, 7 Jul 2021 02:55:38 -0700 Subject: cvs tests: don't use dateutil at all `dateutil.parser` was an optional dependency for CVS tests. It was failing on macOS beacuse the dateutil types were not being installed, and mypy was failing *even when the CVS tests were skipped*. This seems like it was an oversight on macOS -- `types-dateutil-parser` was not installed there, though it was on Linux unit tests. It takes 6 lines of YAML and some weird test-skipping logic to get `python-dateutil` and `types-python-dateutil` installed in all the tests where we need them, but it only takes 4 lines of code to write the date parser we need for CVS, so I just did that instead. Note that CVS date format can vary from system to system, but it seems like it's always pretty similar for the parts we care about. - [x] Replace dateutil.parser with a simpler date regex - [x] Lose the dependency on `dateutil.parser` --- .github/workflows/unit_tests.yaml | 11 +++++------ lib/spack/spack/test/conftest.py | 39 +++++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index a4790fc25e..7c8971a048 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -39,7 +39,7 @@ jobs: python-version: 3.9 - name: Install Python packages run: | - pip install --upgrade pip six setuptools flake8 isort>=4.3.5 mypy>=0.800 black types-six types-python-dateutil + pip install --upgrade pip six setuptools flake8 isort>=4.3.5 mypy>=0.800 black types-six - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -62,7 +62,7 @@ jobs: sudo apt-get install -y coreutils ninja-build graphviz - name: Install Python packages run: | - pip install --upgrade pip six setuptools python-dateutil + pip install --upgrade pip six setuptools pip install --upgrade -r lib/spack/docs/requirements.txt - name: Build documentation run: | @@ -137,7 +137,7 @@ jobs: sudo apt-get -y install zlib1g-dev libdw-dev libiberty-dev - name: Install Python packages run: | - pip install --upgrade pip python-dateutil six setuptools codecov coverage + pip install --upgrade pip six setuptools codecov coverage - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -205,7 +205,7 @@ jobs: sudo apt-get -y install zlib1g-dev libdw-dev libiberty-dev - name: Install Python packages run: | - pip install --upgrade pip six setuptools codecov coverage python-dateutil + pip install --upgrade pip six setuptools codecov coverage - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -326,7 +326,7 @@ jobs: make -C ${KCOV_ROOT}/build && sudo make -C ${KCOV_ROOT}/build install - name: Install Python packages run: | - pip install --upgrade pip six setuptools codecov coverage clingo python-dateutil + pip install --upgrade pip six setuptools codecov coverage clingo - name: Setup git configuration run: | # Need this for the git tests to succeed. @@ -371,7 +371,6 @@ jobs: pip install --upgrade pip six setuptools pip install --upgrade codecov coverage pip install --upgrade flake8 isort>=4.3.5 pep8-naming mypy>=0.800 - pip install --upgrade python-dateutil - name: Setup Homebrew packages run: | brew install dash fish gcc gnupg2 kcov diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 3d9f9429dc..d579fe616f 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) import collections +import datetime import errno import inspect import itertools @@ -15,21 +16,13 @@ import shutil import tempfile import xml.etree.ElementTree -try: - # CVS outputs dates in different formats on different systems. We are using - # the dateutil package to parse these dates. This package does not exist - # for Python <2.7. That means that we cannot test checkouts "by date" for - # CVS respositories. (We can still use CVS repos with all features, only - # our tests break.) - from dateutil.parser import parse as parse_date -except ImportError: - def parse_date(string): # type: ignore - pytest.skip("dateutil package not available") +import py +import pytest import archspec.cpu.microarchitecture import archspec.cpu.schema -import py -import pytest + +from llnl.util.filesystem import mkdirp, remove_linked_tree, working_dir import spack.architecture import spack.caches @@ -49,9 +42,7 @@ import spack.subprocess_context import spack.util.executable import spack.util.gpg import spack.util.spack_yaml as syaml -from llnl.util.filesystem import mkdirp, remove_linked_tree, working_dir -from spack.fetch_strategy import FetchError -from spack.fetch_strategy import FetchStrategyComposite, URLFetchStrategy +from spack.fetch_strategy import FetchError, FetchStrategyComposite, URLFetchStrategy from spack.util.pattern import Bunch @@ -917,6 +908,19 @@ def mock_archive(request, tmpdir_factory): expanded_archive_basedir=spack.stage._source_path_subdir) +def _parse_cvs_date(line): + """Turn a CVS log date into a datetime.datetime""" + # dates in CVS logs can have slashes or dashes and may omit the time zone: + # date: 2021-07-07 02:43:33 -0700; ... + # date: 2021-07-07 02:43:33; ... + # date: 2021/07/07 02:43:33; ... + m = re.search(r'date:\s+(\d+)[/-](\d+)[/-](\d+)\s+(\d+):(\d+):(\d+)', line) + if not m: + return None + year, month, day, hour, minute, second = [int(g) for g in m.groups()] + return datetime.datetime(year, month, day, hour, minute, second) + + @pytest.fixture(scope='session') def mock_cvs_repository(tmpdir_factory): """Creates a very simple CVS repository with two commits and a branch.""" @@ -943,9 +947,8 @@ def mock_cvs_repository(tmpdir_factory): """Find the most recent CVS time stamp in a `cvs log` output""" latest_timestamp = None for line in output.splitlines(): - m = re.search(r'date:\s+([^;]*);', line) - if m: - timestamp = parse_date(m.group(1)) + timestamp = _parse_cvs_date(line) + if timestamp: if latest_timestamp is None: latest_timestamp = timestamp else: -- cgit v1.2.3-70-g09d2