From f7d080b7fb1069c45d9f5d31e27ff6b833b012d8 Mon Sep 17 00:00:00 2001 From: Zack Galbreath Date: Wed, 9 May 2018 13:00:49 -0400 Subject: Move report writers to separate classes --- lib/spack/spack/report.py | 210 ++++------------------------------ lib/spack/spack/reporter.py | 40 +++++++ lib/spack/spack/reporters/__init__.py | 24 ++++ lib/spack/spack/reporters/cdash.py | 203 ++++++++++++++++++++++++++++++++ lib/spack/spack/reporters/junit.py | 48 ++++++++ 5 files changed, 335 insertions(+), 190 deletions(-) create mode 100644 lib/spack/spack/reporter.py create mode 100644 lib/spack/spack/reporters/__init__.py create mode 100644 lib/spack/spack/reporters/cdash.py create mode 100644 lib/spack/spack/reporters/junit.py diff --git a/lib/spack/spack/report.py b/lib/spack/spack/report.py index 9403b76c1e..dc68b5e583 100644 --- a/lib/spack/spack/report.py +++ b/lib/spack/spack/report.py @@ -26,33 +26,25 @@ import codecs import collections import functools -import hashlib -import itertools -import os.path -import platform -import re -import socket import time import traceback -import xml.sax.saxutils -from six import text_type -from six.moves.urllib.request import build_opener, HTTPHandler, Request import llnl.util.lang import spack.build_environment import spack.fetch_strategy import spack.package -from spack.util.crypto import checksum -from spack.util.log_parse import parse_log_events - - -templates = { - 'junit': os.path.join('reports', 'junit.xml'), - 'cdash': os.path.join('reports', 'cdash') +from spack.reporter import Reporter +from spack.reporters.cdash import CDash +from spack.reporters.junit import JUnit + +report_writers = { + None: Reporter, + 'junit': JUnit, + 'cdash': CDash } #: Allowed report formats -valid_formats = list(templates.keys()) +valid_formats = list(report_writers.keys()) __all__ = [ 'valid_formats', @@ -261,21 +253,17 @@ class collect_info(object): ValueError: when ``format_name`` is not in ``valid_formats`` """ def __init__(self, format_name, install_command, cdash_upload_url): - self.format_name = format_name self.filename = None - self.install_command = install_command - self.cdash_upload_url = cdash_upload_url - self.hostname = socket.gethostname() - self.osname = platform.system() - self.starttime = int(time.time()) - # TODO: remove hardcoded use of Experimental here. - # Make the submission model configurable. - self.buildstamp = time.strftime("%Y%m%d-%H%M-Experimental", - time.localtime(self.starttime)) - - # Check that the format is valid - if format_name not in itertools.chain(valid_formats, [None]): - raise ValueError('invalid report type: {0}'.format(format_name)) + self.format_name = format_name + # Check that the format is valid. + if self.format_name not in valid_formats: + raise ValueError('invalid report type: {0}' + .format(self.format_name)) + self.report_writer = report_writers[self.format_name]( + install_command, cdash_upload_url) + + def concretization_report(self, msg): + self.report_writer.concretization_report(self.filename, msg) def __enter__(self): if self.format_name: @@ -283,155 +271,6 @@ class collect_info(object): self.collector = InfoCollector(self.specs) self.collector.__enter__() - def cdash_initialize_report(self, report_data): - if not os.path.exists(self.filename): - os.mkdir(self.filename) - report_data['install_command'] = self.install_command - report_data['buildstamp'] = self.buildstamp - report_data['hostname'] = self.hostname - report_data['osname'] = self.osname - - def cdash_build_report(self, report_data): - self.cdash_initialize_report(report_data) - - # Mapping Spack phases to the corresponding CTest/CDash phase. - map_phases_to_cdash = { - 'autoreconf': 'configure', - 'cmake': 'configure', - 'configure': 'configure', - 'edit': 'configure', - 'build': 'build', - 'install': 'build' - } - - # Initialize data structures common to each phase's report. - cdash_phases = set(map_phases_to_cdash.values()) - for phase in cdash_phases: - report_data[phase] = {} - report_data[phase]['log'] = "" - report_data[phase]['status'] = 0 - report_data[phase]['starttime'] = self.starttime - report_data[phase]['endtime'] = self.starttime - - # Track the phases we perform so we know what reports to create. - phases_encountered = [] - - # Parse output phase-by-phase. - phase_regexp = re.compile(r"Executing phase: '(.*)'") - for spec in self.collector.specs: - for package in spec['packages']: - if 'stdout' in package: - current_phase = '' - for line in package['stdout'].splitlines(): - match = phase_regexp.search(line) - if match: - current_phase = match.group(1) - if current_phase not in map_phases_to_cdash: - current_phase = '' - continue - beginning_of_phase = True - else: - if beginning_of_phase: - cdash_phase = \ - map_phases_to_cdash[current_phase] - if cdash_phase not in phases_encountered: - phases_encountered.append(cdash_phase) - report_data[cdash_phase]['log'] += \ - text_type("{0} output for {1}:\n".format( - cdash_phase, package['name'])) - beginning_of_phase = False - report_data[cdash_phase]['log'] += \ - xml.sax.saxutils.escape(line) + "\n" - - for phase in phases_encountered: - errors, warnings = parse_log_events( - report_data[phase]['log'].splitlines()) - nerrors = len(errors) - - if phase == 'configure' and nerrors > 0: - report_data[phase]['status'] = 1 - - if phase == 'build': - # Convert log output from ASCII to Unicode and escape for XML. - def clean_log_event(event): - event = vars(event) - event['text'] = xml.sax.saxutils.escape(event['text']) - event['pre_context'] = xml.sax.saxutils.escape( - '\n'.join(event['pre_context'])) - event['post_context'] = xml.sax.saxutils.escape( - '\n'.join(event['post_context'])) - # source_file and source_line_no are either strings or - # the tuple (None,). Distinguish between these two cases. - if event['source_file'][0] is None: - event['source_file'] = '' - event['source_line_no'] = '' - else: - event['source_file'] = xml.sax.saxutils.escape( - event['source_file']) - return event - - report_data[phase]['errors'] = [] - report_data[phase]['warnings'] = [] - for error in errors: - report_data[phase]['errors'].append(clean_log_event(error)) - for warning in warnings: - report_data[phase]['warnings'].append( - clean_log_event(warning)) - - # Write the report. - report_name = phase.capitalize() + ".xml" - phase_report = os.path.join(self.filename, report_name) - - with codecs.open(phase_report, 'w', 'utf-8') as f: - env = spack.tengine.make_environment() - site_template = os.path.join(templates[self.format_name], - 'Site.xml') - t = env.get_template(site_template) - f.write(t.render(report_data)) - - phase_template = os.path.join(templates[self.format_name], - report_name) - t = env.get_template(phase_template) - f.write(t.render(report_data)) - self.upload_to_cdash(phase_report) - - def concretization_report(self, msg): - if not self.format_name == 'cdash': - return - - report_data = {} - report_data['starttime'] = self.starttime - report_data['endtime'] = self.starttime - self.cdash_initialize_report(report_data) - - report_data['msg'] = msg - env = spack.tengine.make_environment() - update_template = os.path.join(templates[self.format_name], - 'Update.xml') - t = env.get_template(update_template) - output_filename = os.path.join(self.filename, 'Update.xml') - with open(output_filename, 'w') as f: - f.write(t.render(report_data)) - self.upload_to_cdash(output_filename) - - def upload_to_cdash(self, filename): - if not self.cdash_upload_url: - return - - # Compute md5 checksum for the contents of this file. - md5sum = checksum(hashlib.md5, filename, block_size=8192) - - opener = build_opener(HTTPHandler) - with open(filename, 'rb') as f: - url = "{0}&MD5={1}".format(self.cdash_upload_url, md5sum) - request = Request(url, data=f) - request.add_header('Content-Type', 'text/xml') - request.add_header('Content-Length', os.path.getsize(filename)) - # By default, urllib2 only support GET and POST. - # CDash needs expects this file to be uploaded via PUT. - request.get_method = lambda: 'PUT' - url = opener.open(request) - def __exit__(self, exc_type, exc_val, exc_tb): if self.format_name: # Close the collector and restore the @@ -439,13 +278,4 @@ class collect_info(object): self.collector.__exit__(exc_type, exc_val, exc_tb) report_data = {'specs': self.collector.specs} - - if self.format_name == 'cdash': - # CDash reporting results are split across multiple files. - self.cdash_build_report(report_data) - else: - # Write the report - with open(self.filename, 'w') as f: - env = spack.tengine.make_environment() - t = env.get_template(templates[self.format_name]) - f.write(t.render(report_data)) + self.report_writer.build_report(self.filename, report_data) diff --git a/lib/spack/spack/reporter.py b/lib/spack/spack/reporter.py new file mode 100644 index 0000000000..36efc44e6a --- /dev/null +++ b/lib/spack/spack/reporter.py @@ -0,0 +1,40 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +__all__ = ['Reporter'] + + +class Reporter(object): + """Base class for report writers.""" + + def __init__(self, install_command, cdash_upload_url): + self.install_command = install_command + self.cdash_upload_url = cdash_upload_url + + def build_report(self, filename, report_data): + pass + + def concretization_report(self, filename, msg): + pass diff --git a/lib/spack/spack/reporters/__init__.py b/lib/spack/spack/reporters/__init__.py new file mode 100644 index 0000000000..0e81cb3b36 --- /dev/null +++ b/lib/spack/spack/reporters/__init__.py @@ -0,0 +1,24 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## diff --git a/lib/spack/spack/reporters/cdash.py b/lib/spack/spack/reporters/cdash.py new file mode 100644 index 0000000000..eeb1224fc7 --- /dev/null +++ b/lib/spack/spack/reporters/cdash.py @@ -0,0 +1,203 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import codecs +import hashlib +import os.path +import platform +import re +import socket +import time +import xml.sax.saxutils +from six import text_type +from six.moves.urllib.request import build_opener, HTTPHandler, Request + +import spack.build_environment +import spack.fetch_strategy +import spack.package +from spack.reporter import Reporter +from spack.util.crypto import checksum +from spack.util.log_parse import parse_log_events + +__all__ = ['CDash'] + +# Mapping Spack phases to the corresponding CTest/CDash phase. +map_phases_to_cdash = { + 'autoreconf': 'configure', + 'cmake': 'configure', + 'configure': 'configure', + 'edit': 'configure', + 'build': 'build', + 'install': 'build' +} + +# Initialize data structures common to each phase's report. +cdash_phases = set(map_phases_to_cdash.values()) + + +class CDash(Reporter): + """Generate reports of spec installations for CDash.""" + + def __init__(self, install_command, cdash_upload_url): + Reporter.__init__(self, install_command, cdash_upload_url) + self.template_dir = os.path.join('reports', 'cdash') + self.hostname = socket.gethostname() + self.osname = platform.system() + self.starttime = int(time.time()) + # TODO: remove hardcoded use of Experimental here. + # Make the submission model configurable. + self.buildstamp = time.strftime("%Y%m%d-%H%M-Experimental", + time.localtime(self.starttime)) + + def build_report(self, filename, report_data): + self.initialize_report(filename, report_data) + + for phase in cdash_phases: + report_data[phase] = {} + report_data[phase]['log'] = "" + report_data[phase]['status'] = 0 + report_data[phase]['starttime'] = self.starttime + report_data[phase]['endtime'] = self.starttime + + # Track the phases we perform so we know what reports to create. + phases_encountered = [] + + # Parse output phase-by-phase. + phase_regexp = re.compile(r"Executing phase: '(.*)'") + for spec in report_data['specs']: + for package in spec['packages']: + if 'stdout' in package: + current_phase = '' + for line in package['stdout'].splitlines(): + match = phase_regexp.search(line) + if match: + current_phase = match.group(1) + if current_phase not in map_phases_to_cdash: + current_phase = '' + continue + beginning_of_phase = True + else: + if beginning_of_phase: + cdash_phase = \ + map_phases_to_cdash[current_phase] + if cdash_phase not in phases_encountered: + phases_encountered.append(cdash_phase) + report_data[cdash_phase]['log'] += \ + text_type("{0} output for {1}:\n".format( + cdash_phase, package['name'])) + beginning_of_phase = False + report_data[cdash_phase]['log'] += \ + xml.sax.saxutils.escape(line) + "\n" + + for phase in phases_encountered: + errors, warnings = parse_log_events( + report_data[phase]['log'].splitlines()) + nerrors = len(errors) + + if phase == 'configure' and nerrors > 0: + report_data[phase]['status'] = 1 + + if phase == 'build': + # Convert log output from ASCII to Unicode and escape for XML. + def clean_log_event(event): + event = vars(event) + event['text'] = xml.sax.saxutils.escape(event['text']) + event['pre_context'] = xml.sax.saxutils.escape( + '\n'.join(event['pre_context'])) + event['post_context'] = xml.sax.saxutils.escape( + '\n'.join(event['post_context'])) + # source_file and source_line_no are either strings or + # the tuple (None,). Distinguish between these two cases. + if event['source_file'][0] is None: + event['source_file'] = '' + event['source_line_no'] = '' + else: + event['source_file'] = xml.sax.saxutils.escape( + event['source_file']) + return event + + report_data[phase]['errors'] = [] + report_data[phase]['warnings'] = [] + for error in errors: + report_data[phase]['errors'].append(clean_log_event(error)) + for warning in warnings: + report_data[phase]['warnings'].append( + clean_log_event(warning)) + + # Write the report. + report_name = phase.capitalize() + ".xml" + phase_report = os.path.join(filename, report_name) + + with codecs.open(phase_report, 'w', 'utf-8') as f: + env = spack.tengine.make_environment() + site_template = os.path.join(self.template_dir, 'Site.xml') + t = env.get_template(site_template) + f.write(t.render(report_data)) + + phase_template = os.path.join(self.template_dir, report_name) + t = env.get_template(phase_template) + f.write(t.render(report_data)) + self.upload(phase_report) + + def concretization_report(self, filename, msg): + report_data = {} + self.initialize_report(filename, report_data) + report_data['starttime'] = self.starttime + report_data['endtime'] = self.starttime + report_data['msg'] = msg + + env = spack.tengine.make_environment() + update_template = os.path.join(self.template_dir, 'Update.xml') + t = env.get_template(update_template) + output_filename = os.path.join(filename, 'Update.xml') + with open(output_filename, 'w') as f: + f.write(t.render(report_data)) + self.upload(output_filename) + + def initialize_report(self, filename, report_data): + if not os.path.exists(filename): + os.mkdir(filename) + report_data['install_command'] = self.install_command + report_data['buildstamp'] = self.buildstamp + report_data['hostname'] = self.hostname + report_data['osname'] = self.osname + + def upload(self, filename): + if not self.cdash_upload_url: + return + + # Compute md5 checksum for the contents of this file. + md5sum = checksum(hashlib.md5, filename, block_size=8192) + + opener = build_opener(HTTPHandler) + with open(filename, 'rb') as f: + url = "{0}&MD5={1}".format(self.cdash_upload_url, md5sum) + request = Request(url, data=f) + request.add_header('Content-Type', 'text/xml') + request.add_header('Content-Length', os.path.getsize(filename)) + # By default, urllib2 only support GET and POST. + # CDash needs expects this file to be uploaded via PUT. + request.get_method = lambda: 'PUT' + url = opener.open(request) diff --git a/lib/spack/spack/reporters/junit.py b/lib/spack/spack/reporters/junit.py new file mode 100644 index 0000000000..e8b69f8240 --- /dev/null +++ b/lib/spack/spack/reporters/junit.py @@ -0,0 +1,48 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## + +import os.path + +import spack.build_environment +import spack.fetch_strategy +import spack.package +from spack.reporter import Reporter + +__all__ = ['JUnit'] + + +class JUnit(Reporter): + """Generate reports of spec installations for JUnit.""" + + def __init__(self, install_command, cdash_upload_url): + Reporter.__init__(self, install_command, cdash_upload_url) + self.template_file = os.path.join('reports', 'junit.xml') + + def build_report(self, filename, report_data): + # Write the report + with open(filename, 'w') as f: + env = spack.tengine.make_environment() + t = env.get_template(self.template_file) + f.write(t.render(report_data)) -- cgit v1.2.3-70-g09d2