From 0c33e8ac4859b3ce559055af87479418b8cc8fe4 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Wed, 30 Dec 2015 11:22:11 -0800 Subject: go and a basic go package --- var/spack/packages/go/package.py | 49 ++++++++++++++++++++++ .../packages/the_platinum_searcher/package.py | 19 +++++++++ 2 files changed, 68 insertions(+) create mode 100644 var/spack/packages/go/package.py create mode 100644 var/spack/packages/the_platinum_searcher/package.py diff --git a/var/spack/packages/go/package.py b/var/spack/packages/go/package.py new file mode 100644 index 0000000000..4e3f6fa8ec --- /dev/null +++ b/var/spack/packages/go/package.py @@ -0,0 +1,49 @@ +import os +from spack import * + +class Go(Package): + """The golang compiler and build environment""" + homepage = "https://golang.org" + url = "https://go.googlesource.com/go" + + extendable = True + + # temporary fix until tags are pulled correctly + version('1.4.2', git='https://go.googlesource.com/go', tag='go1.4.2') + + # to-do, make non-c self-hosting compilers possible + # depends_on('go@:1.4.2', when='@1.5:') + + def install(self, spec, prefix): + os.environ['GOROOT'] = os.getcwd() + os.environ['GOBIN'] = join_path(os.getcwd(), 'bin') + os.environ['GOROOT_FINAL'] = prefix + bash = which('bash') + bash('-c', 'env') + bash('-c', 'pwd') + with working_dir('src'): + #TODO: crutch until the read-only-filesystem bug is fixed upstream + bash('all.bash', fail_on_error=False) + cp = which('cp') + bash('-c', 'cp -r ./* "{}"'.format(prefix)) + + def setup_dependent_environment(self, module, spec, ext_spec): + """Called before go modules' install() methods. + + In most cases, extensions will only need to have one line:: + + go('get', '') + """ + # Add a go command for extensions + module.go = Executable(join_path(spec.prefix.bin, 'go')) + os.environ['GOROOT'] = spec.prefix + + stage_path = os.path.realpath(ext_spec.package.stage.source_path) + print "PREFIX: {}".format(stage_path) + go_paths = [stage_path] + for d in ext_spec.traverse(): + if d.package.extends(self.spec): + go_paths.append(d.prefix) + os.environ['GOPATH'] = ':'.join(go_paths) + + diff --git a/var/spack/packages/the_platinum_searcher/package.py b/var/spack/packages/the_platinum_searcher/package.py new file mode 100644 index 0000000000..1dd54d3b98 --- /dev/null +++ b/var/spack/packages/the_platinum_searcher/package.py @@ -0,0 +1,19 @@ +from spack import * + +class ThePlatinumSearcher(Package): + """Fast parallel recursive grep alternative""" + # FIXME: add a proper url for your package's homepage here. + homepage = "https://github.com/monochromegane/the_platinum_searcher" + url = "https://github.com/monochromegane/the_platinum_searcher/archive/v1.7.7.tar.gz" + + version('1.7.7', '08d7265e101bc1427d5d4b9903aa1166') + + depends_on("go") + + def install(self, spec, prefix): + env = which('env') + env() + # Fetch all dependencies + go('get', './...') + # Build pt + go('build') -- cgit v1.2.3-70-g09d2 From 915f91a6199dfe77481742330194e0f6ee17fb8b Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Thu, 12 May 2016 10:45:58 -0700 Subject: moving go packages to the new path --- var/spack/packages/go/package.py | 49 --------------------- .../packages/the_platinum_searcher/package.py | 19 -------- var/spack/repos/builtin/packages/go/package.py | 50 ++++++++++++++++++++++ .../packages/the_platinum_searcher/package.py | 19 ++++++++ 4 files changed, 69 insertions(+), 68 deletions(-) delete mode 100644 var/spack/packages/go/package.py delete mode 100644 var/spack/packages/the_platinum_searcher/package.py create mode 100644 var/spack/repos/builtin/packages/go/package.py create mode 100644 var/spack/repos/builtin/packages/the_platinum_searcher/package.py diff --git a/var/spack/packages/go/package.py b/var/spack/packages/go/package.py deleted file mode 100644 index 4e3f6fa8ec..0000000000 --- a/var/spack/packages/go/package.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -from spack import * - -class Go(Package): - """The golang compiler and build environment""" - homepage = "https://golang.org" - url = "https://go.googlesource.com/go" - - extendable = True - - # temporary fix until tags are pulled correctly - version('1.4.2', git='https://go.googlesource.com/go', tag='go1.4.2') - - # to-do, make non-c self-hosting compilers possible - # depends_on('go@:1.4.2', when='@1.5:') - - def install(self, spec, prefix): - os.environ['GOROOT'] = os.getcwd() - os.environ['GOBIN'] = join_path(os.getcwd(), 'bin') - os.environ['GOROOT_FINAL'] = prefix - bash = which('bash') - bash('-c', 'env') - bash('-c', 'pwd') - with working_dir('src'): - #TODO: crutch until the read-only-filesystem bug is fixed upstream - bash('all.bash', fail_on_error=False) - cp = which('cp') - bash('-c', 'cp -r ./* "{}"'.format(prefix)) - - def setup_dependent_environment(self, module, spec, ext_spec): - """Called before go modules' install() methods. - - In most cases, extensions will only need to have one line:: - - go('get', '') - """ - # Add a go command for extensions - module.go = Executable(join_path(spec.prefix.bin, 'go')) - os.environ['GOROOT'] = spec.prefix - - stage_path = os.path.realpath(ext_spec.package.stage.source_path) - print "PREFIX: {}".format(stage_path) - go_paths = [stage_path] - for d in ext_spec.traverse(): - if d.package.extends(self.spec): - go_paths.append(d.prefix) - os.environ['GOPATH'] = ':'.join(go_paths) - - diff --git a/var/spack/packages/the_platinum_searcher/package.py b/var/spack/packages/the_platinum_searcher/package.py deleted file mode 100644 index 1dd54d3b98..0000000000 --- a/var/spack/packages/the_platinum_searcher/package.py +++ /dev/null @@ -1,19 +0,0 @@ -from spack import * - -class ThePlatinumSearcher(Package): - """Fast parallel recursive grep alternative""" - # FIXME: add a proper url for your package's homepage here. - homepage = "https://github.com/monochromegane/the_platinum_searcher" - url = "https://github.com/monochromegane/the_platinum_searcher/archive/v1.7.7.tar.gz" - - version('1.7.7', '08d7265e101bc1427d5d4b9903aa1166') - - depends_on("go") - - def install(self, spec, prefix): - env = which('env') - env() - # Fetch all dependencies - go('get', './...') - # Build pt - go('build') diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py new file mode 100644 index 0000000000..05fce90534 --- /dev/null +++ b/var/spack/repos/builtin/packages/go/package.py @@ -0,0 +1,50 @@ +import os +from spack import * + +class Go(Package): + """The golang compiler and build environment""" + homepage = "https://golang.org" + url = "https://go.googlesource.com/go" + + extendable = True + + # temporary fix until tags are pulled correctly + version('1.4.2', git='https://go.googlesource.com/go', tag='go1.4.2') + version('1.5.0', git='https://go.googlesource.com/go', tag='go1.5.0') + + # to-do, make non-c self-hosting compilers possible + depends_on('go@:1.4.2', when='@1.5:') + + def install(self, spec, prefix): + os.environ['GOROOT'] = os.getcwd() + os.environ['GOBIN'] = join_path(os.getcwd(), 'bin') + os.environ['GOROOT_FINAL'] = prefix + bash = which('bash') + bash('-c', 'env') + bash('-c', 'pwd') + with working_dir('src'): + #TODO: crutch until the read-only-filesystem bug is fixed upstream + bash('all.bash', fail_on_error=False) + cp = which('cp') + bash('-c', 'cp -r ./* "{}"'.format(prefix)) + + def setup_dependent_environment(self, module, spec, ext_spec): + """Called before go modules' install() methods. + + In most cases, extensions will only need to have one line:: + + go('get', '') + """ + # Add a go command for extensions + module.go = Executable(join_path(spec.prefix.bin, 'go')) + os.environ['GOROOT'] = spec.prefix + + stage_path = os.path.realpath(ext_spec.package.stage.source_path) + print "PREFIX: {}".format(stage_path) + go_paths = [stage_path] + for d in ext_spec.traverse(): + if d.package.extends(self.spec): + go_paths.append(d.prefix) + os.environ['GOPATH'] = ':'.join(go_paths) + + diff --git a/var/spack/repos/builtin/packages/the_platinum_searcher/package.py b/var/spack/repos/builtin/packages/the_platinum_searcher/package.py new file mode 100644 index 0000000000..1dd54d3b98 --- /dev/null +++ b/var/spack/repos/builtin/packages/the_platinum_searcher/package.py @@ -0,0 +1,19 @@ +from spack import * + +class ThePlatinumSearcher(Package): + """Fast parallel recursive grep alternative""" + # FIXME: add a proper url for your package's homepage here. + homepage = "https://github.com/monochromegane/the_platinum_searcher" + url = "https://github.com/monochromegane/the_platinum_searcher/archive/v1.7.7.tar.gz" + + version('1.7.7', '08d7265e101bc1427d5d4b9903aa1166') + + depends_on("go") + + def install(self, spec, prefix): + env = which('env') + env() + # Fetch all dependencies + go('get', './...') + # Build pt + go('build') -- cgit v1.2.3-70-g09d2 From 0816e9554d108498cdb419d71f3bac1375622ad1 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sat, 14 May 2016 18:06:22 -0700 Subject: adding go_compiler providers --- var/spack/repos/builtin/packages/gcc/package.py | 3 +++ var/spack/repos/builtin/packages/go/package.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 47dbeb2a99..f630b9b61a 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -62,6 +62,9 @@ class Gcc(Package): depends_on("binutils~libiberty", when='+binutils ~gold') depends_on("binutils~libiberty+gold", when='+binutils +gold') + if sys.platform != 'darwin': + provides('go_compiler' when='@4.7.1:') + # TODO: integrate these libraries. #depends_on("ppl") #depends_on("cloog") diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py index 05fce90534..71b1b9c18f 100644 --- a/var/spack/repos/builtin/packages/go/package.py +++ b/var/spack/repos/builtin/packages/go/package.py @@ -12,12 +12,14 @@ class Go(Package): version('1.4.2', git='https://go.googlesource.com/go', tag='go1.4.2') version('1.5.0', git='https://go.googlesource.com/go', tag='go1.5.0') + + provides('go_compiler') # to-do, make non-c self-hosting compilers possible - depends_on('go@:1.4.2', when='@1.5:') + # should be go_compiler, but that creates an infinite loop + depends_on('gcc', when='@1.5:') def install(self, spec, prefix): os.environ['GOROOT'] = os.getcwd() - os.environ['GOBIN'] = join_path(os.getcwd(), 'bin') os.environ['GOROOT_FINAL'] = prefix bash = which('bash') bash('-c', 'env') -- cgit v1.2.3-70-g09d2 From 577657b3f7eec000c172ac4f495a9480cd1d4038 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sat, 14 May 2016 22:35:55 -0700 Subject: go rework This commit includes: * a new go package that uses gccgo to bootstrap the go toolchain * env support added to Executable * a new Go fetch strategy that uses `go get` to fetch a package and all of its deps * A platinum searcher package that leverages the new go package and fetch strategy --- lib/spack/spack/fetch_strategy.py | 54 ++++++++++++++ lib/spack/spack/util/executable.py | 4 +- var/spack/repos/builtin/packages/gcc/package.py | 84 +++++++++++----------- var/spack/repos/builtin/packages/go/package.py | 81 +++++++++++++-------- .../packages/the_platinum_searcher/package.py | 19 ++--- 5 files changed, 160 insertions(+), 82 deletions(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index e05cb13c1e..0144d26057 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -381,6 +381,60 @@ class VCSFetchStrategy(FetchStrategy): def __repr__(self): return "%s<%s>" % (self.__class__, self.url) +class GoFetchStrategy(VCSFetchStrategy): + """Fetch strategy that employs the `go get` infrastructure + Use like this in a package: + + version('name', go='github.com/monochromegane/the_platinum_searcher/...') + + Go get does not natively support versions, they can be faked with git + """ + enabled = True + required_attributes = ('go',) + + def __init__(self, **kwargs): + # Discards the keywords in kwargs that may conflict with the next call to __init__ + forwarded_args = copy.copy(kwargs) + forwarded_args.pop('name', None) + + super(GoFetchStrategy, self).__init__('go', **forwarded_args) + self._go = None + + @property + def go_version(self): + vstring = self.go('version', output=str).split(' ')[2] + return Version(vstring) + + @property + def go(self): + if not self._go: + self._go = which('go', required=True) + return self._go + + @_needs_stage + def fetch(self): + self.stage.chdir() + + tty.msg("Trying to get go resource:", self.url) + + try: + os.mkdir('go') + except OSError: + pass + env = dict(os.environ) + env['GOPATH'] = os.path.join(os.getcwd(),'go') + self.go('get', '-v', '-d', self.url, env=env) + + def archive(self, destination): + super(GoFetchStrategy, self).archive(destination, exclude='.git') + + @_needs_stage + def reset(self): + self.stage.chdir_to_source() + self.go('clean') + + def __str__(self): + return "[go] %s" % self.url class GitFetchStrategy(VCSFetchStrategy): """Fetch strategy that gets source code from a git repository. diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d2ccfde69b..0b91b6c360 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -105,6 +105,8 @@ class Executable(object): fail_on_error = kwargs.pop("fail_on_error", True) ignore_errors = kwargs.pop("ignore_errors", ()) + env = kwargs.get('env', None) + # TODO: This is deprecated. Remove in a future version. return_output = kwargs.pop("return_output", False) @@ -149,7 +151,7 @@ class Executable(object): try: proc = subprocess.Popen( - cmd, stdin=istream, stderr=estream, stdout=ostream) + cmd, stdin=istream, stderr=estream, stdout=ostream, env=env) out, err = proc.communicate() rc = self.returncode = proc.returncode diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index f630b9b61a..0ef6b731ed 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -1,33 +1,33 @@ ############################################################################## -# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC. +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. # Produced at the Lawrence Livermore National Laboratory. # # This file is part of Spack. -# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. # LLNL-CODE-647188 # # For details, see https://github.com/llnl/spack # Please also see the LICENSE file for our notice and the LGPL. # # This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License (as -# published by the Free Software Foundation) version 2.1, February 1999. +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU Lesser General Public License for more details. +# conditions of the GNU General Public License for more details. # -# You should have received a copy of the GNU Lesser General Public -# License along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## from spack import * from contextlib import closing from glob import glob import sys -import os + class Gcc(Package): """The GNU Compiler Collection includes front ends for C, C++, @@ -50,10 +50,12 @@ class Gcc(Package): version('4.6.4', 'b407a3d1480c11667f293bfb1f17d1a4') version('4.5.4', '27e459c2566b8209ab064570e1b378f7') - variant('binutils', default=sys.platform != 'darwin', - description="Build via binutils") - variant('gold', default=sys.platform != 'darwin', - description="Build the gold linker plugin for ld-based LTO") + variant('binutils', + default=sys.platform != 'darwin', + description="Build via binutils") + variant('gold', + default=sys.platform != 'darwin', + description="Build the gold linker plugin for ld-based LTO") depends_on("mpfr") depends_on("gmp") @@ -62,12 +64,9 @@ class Gcc(Package): depends_on("binutils~libiberty", when='+binutils ~gold') depends_on("binutils~libiberty+gold", when='+binutils +gold') - if sys.platform != 'darwin': - provides('go_compiler' when='@4.7.1:') - # TODO: integrate these libraries. - #depends_on("ppl") - #depends_on("cloog") + # depends_on("ppl") + # depends_on("cloog") if sys.platform == 'darwin': patch('darwin/gcc-4.9.patch1', when='@4.9.3') patch('darwin/gcc-4.9.patch2', when='@4.9.3') @@ -75,7 +74,7 @@ class Gcc(Package): def install(self, spec, prefix): # libjava/configure needs a minor fix to install into spack paths. filter_file(r"'@.*@'", "'@[[:alnum:]]*@'", 'libjava/configure', - string=True) + string=True) enabled_languages = set(('c', 'c++', 'fortran', 'java', 'objc')) @@ -83,62 +82,59 @@ class Gcc(Package): enabled_languages.add('go') # Generic options to compile GCC - options = ["--prefix=%s" % prefix, - "--libdir=%s/lib64" % prefix, + options = ["--prefix=%s" % prefix, "--libdir=%s/lib64" % prefix, "--disable-multilib", "--enable-languages=" + ','.join(enabled_languages), - "--with-mpc=%s" % spec['mpc'].prefix, - "--with-mpfr=%s" % spec['mpfr'].prefix, - "--with-gmp=%s" % spec['gmp'].prefix, - "--enable-lto", - "--with-quad"] + "--with-mpc=%s" % spec['mpc'].prefix, "--with-mpfr=%s" % + spec['mpfr'].prefix, "--with-gmp=%s" % spec['gmp'].prefix, + "--enable-lto", "--with-quad"] # Binutils if spec.satisfies('+binutils'): static_bootstrap_flags = "-static-libstdc++ -static-libgcc" - binutils_options = ["--with-sysroot=/", - "--with-stage1-ldflags=%s %s" % - (self.rpath_args, static_bootstrap_flags), - "--with-boot-ldflags=%s %s" % - (self.rpath_args, static_bootstrap_flags), - "--with-gnu-ld", - "--with-ld=%s/bin/ld" % spec['binutils'].prefix, - "--with-gnu-as", - "--with-as=%s/bin/as" % spec['binutils'].prefix] + binutils_options = [ + "--with-sysroot=/", "--with-stage1-ldflags=%s %s" % + (self.rpath_args, static_bootstrap_flags), + "--with-boot-ldflags=%s %s" % + (self.rpath_args, static_bootstrap_flags), "--with-gnu-ld", + "--with-ld=%s/bin/ld" % spec['binutils'].prefix, + "--with-gnu-as", + "--with-as=%s/bin/as" % spec['binutils'].prefix + ] options.extend(binutils_options) # Isl if 'isl' in spec: isl_options = ["--with-isl=%s" % spec['isl'].prefix] options.extend(isl_options) - if sys.platform == 'darwin' : - darwin_options = [ "--with-build-config=bootstrap-debug" ] + if sys.platform == 'darwin': + darwin_options = ["--with-build-config=bootstrap-debug"] options.extend(darwin_options) build_dir = join_path(self.stage.path, 'spack-build') - configure = Executable( join_path(self.stage.source_path, 'configure') ) + configure = Executable(join_path(self.stage.source_path, 'configure')) with working_dir(build_dir, create=True): # Rest of install is straightforward. configure(*options) - if sys.platform == 'darwin' : make("bootstrap") - else: make() + if sys.platform == 'darwin': + make("bootstrap") + else: + make() make("install") self.write_rpath_specs() - @property def spec_dir(self): # e.g. lib64/gcc/x86_64-unknown-linux-gnu/4.9.2 spec_dir = glob("%s/lib64/gcc/*/*" % self.prefix) return spec_dir[0] if spec_dir else None - def write_rpath_specs(self): """Generate a spec file so the linker adds a rpath to the libs the compiler used to build the executable.""" if not self.spec_dir: tty.warn("Could not install specs for %s." % - self.spec.format('$_$@')) + self.spec.format('$_$@')) return gcc = Executable(join_path(self.prefix.bin, 'gcc')) @@ -149,5 +145,5 @@ class Gcc(Package): out.write(line + "\n") if line.startswith("*link:"): out.write("-rpath %s/lib:%s/lib64 \\\n" % - (self.prefix, self.prefix)) + (self.prefix, self.prefix)) set_install_permissions(specs_file) diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py index 71b1b9c18f..6fe212c800 100644 --- a/var/spack/repos/builtin/packages/go/package.py +++ b/var/spack/repos/builtin/packages/go/package.py @@ -1,6 +1,9 @@ import os +import shutil +import glob from spack import * + class Go(Package): """The golang compiler and build environment""" homepage = "https://golang.org" @@ -10,43 +13,65 @@ class Go(Package): # temporary fix until tags are pulled correctly version('1.4.2', git='https://go.googlesource.com/go', tag='go1.4.2') - version('1.5.0', git='https://go.googlesource.com/go', tag='go1.5.0') + version('1.5.4', git='https://go.googlesource.com/go', tag='go1.5.4') + version('1.6.2', git='https://go.googlesource.com/go', tag='go1.6.2') + + variant('test', + default=True, + description="Run tests as part of build, a good idea but quite" + " time consuming") + provides('golang') - provides('go_compiler') - # to-do, make non-c self-hosting compilers possible + # to-do, make non-c self-hosting compilers feasible without backflips # should be go_compiler, but that creates an infinite loop - depends_on('gcc', when='@1.5:') + depends_on('gcc@5:', when='@1.5:') + depends_on('git') def install(self, spec, prefix): - os.environ['GOROOT'] = os.getcwd() - os.environ['GOROOT_FINAL'] = prefix - bash = which('bash') - bash('-c', 'env') - bash('-c', 'pwd') - with working_dir('src'): - #TODO: crutch until the read-only-filesystem bug is fixed upstream - bash('all.bash', fail_on_error=False) - cp = which('cp') - bash('-c', 'cp -r ./* "{}"'.format(prefix)) - - def setup_dependent_environment(self, module, spec, ext_spec): + bash = which('bash') + with working_dir('src'): + if '+test' in spec: + bash('all.bash') + else: + bash('make.bash') + + try: + os.makedirs(prefix) + except OSError: + pass + for f in glob.glob('*'): + if os.path.isdir(f): + shutil.copytree(f, os.path.join(prefix, f)) + else: + shutil.copy2(f, os.path.join(prefix, f)) + + def setup_environment(self, spack_env, run_env): + # spack_env.set("GOROOT", self.spec.prefix) + # run_env.set("GOROOT", self.spec.prefix) + spack_env.set('GOROOT_FINAL', self.spec.prefix) + spack_env.set('GOROOT_BOOTSTRAP', self.spec['gcc'].prefix) + + def setup_dependent_package(self, module, ext_spec): + # Add a go command/compiler for extensions + module.go = Executable(join_path(self.spec.prefix.bin, 'go')) + + def setup_dependent_environment(self, spack_env, run_env, ext_spec): """Called before go modules' install() methods. In most cases, extensions will only need to have one line:: - go('get', '') + go('get', '') """ - # Add a go command for extensions - module.go = Executable(join_path(spec.prefix.bin, 'go')) - os.environ['GOROOT'] = spec.prefix - - stage_path = os.path.realpath(ext_spec.package.stage.source_path) - print "PREFIX: {}".format(stage_path) - go_paths = [stage_path] - for d in ext_spec.traverse(): - if d.package.extends(self.spec): - go_paths.append(d.prefix) - os.environ['GOPATH'] = ':'.join(go_paths) + # spack_env.set("GOROOT", self.spec.prefix) + # run_env.set("GOROOT", self.spec.prefix) + # stage_path = os.path.realpath(ext_spec.package.stage.source_path) + # print "PREFIX: {}".format(stage_path) + # go_paths = [stage_path] + # for d in ext_spec.traverse(): + # if d.package.extends(self.spec): + # go_paths.append(d.prefix) + # spack_env.prepend_path('GOPATH', ':'.join(go_paths)) + spack_env.set('GOPATH', ext_spec.package.stage.source_path) diff --git a/var/spack/repos/builtin/packages/the_platinum_searcher/package.py b/var/spack/repos/builtin/packages/the_platinum_searcher/package.py index 1dd54d3b98..c040da3699 100644 --- a/var/spack/repos/builtin/packages/the_platinum_searcher/package.py +++ b/var/spack/repos/builtin/packages/the_platinum_searcher/package.py @@ -1,19 +1,20 @@ from spack import * +import os +import shutil + class ThePlatinumSearcher(Package): """Fast parallel recursive grep alternative""" # FIXME: add a proper url for your package's homepage here. homepage = "https://github.com/monochromegane/the_platinum_searcher" - url = "https://github.com/monochromegane/the_platinum_searcher/archive/v1.7.7.tar.gz" + url = "https://github.com/monochromegane/the_platinum_searcher" + + package = 'github.com/monochromegane/the_platinum_searcher/...' - version('1.7.7', '08d7265e101bc1427d5d4b9903aa1166') + version('head', go=package) - depends_on("go") + extends("go") def install(self, spec, prefix): - env = which('env') - env() - # Fetch all dependencies - go('get', './...') - # Build pt - go('build') + go('install', self.package) + shutil.copytree('bin', os.path.join(prefix, 'bin')) -- cgit v1.2.3-70-g09d2 From 2bfa4d7e4f0ef9dc70b0d7d12d38c2ec005de7a5 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sat, 14 May 2016 22:42:14 -0700 Subject: removing comments also randomly changed unnecessary licence block --- var/spack/repos/builtin/packages/gcc/package.py | 24 ------------------------ var/spack/repos/builtin/packages/go/package.py | 13 ++----------- 2 files changed, 2 insertions(+), 35 deletions(-) diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 0ef6b731ed..218f25b586 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -1,27 +1,3 @@ -############################################################################## -# Copyright (c) 2013, Lawrence Livermore National Security, LLC. -# Produced at the Lawrence Livermore National Laboratory. -# -# This file is part of Spack. -# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. -# LLNL-CODE-647188 -# -# For details, see https://github.com/llnl/spack -# Please also see the LICENSE file for our notice and the LGPL. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License (as published by -# the Free Software Foundation) version 2.1 dated February 1999. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and -# conditions of the GNU General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with this program; if not, write to the Free Software Foundation, -# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -############################################################################## from spack import * from contextlib import closing diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py index 6fe212c800..80924c7fad 100644 --- a/var/spack/repos/builtin/packages/go/package.py +++ b/var/spack/repos/builtin/packages/go/package.py @@ -59,19 +59,10 @@ class Go(Package): def setup_dependent_environment(self, spack_env, run_env, ext_spec): """Called before go modules' install() methods. - In most cases, extensions will only need to have one line:: + In most cases, extensions will only need to have two lines:: go('get', '') + shutil.copytree('bin', os.path.join(prefix, '/bin')) """ - # spack_env.set("GOROOT", self.spec.prefix) - # run_env.set("GOROOT", self.spec.prefix) - - # stage_path = os.path.realpath(ext_spec.package.stage.source_path) - # print "PREFIX: {}".format(stage_path) - # go_paths = [stage_path] - # for d in ext_spec.traverse(): - # if d.package.extends(self.spec): - # go_paths.append(d.prefix) - # spack_env.prepend_path('GOPATH', ':'.join(go_paths)) spack_env.set('GOPATH', ext_spec.package.stage.source_path) -- cgit v1.2.3-70-g09d2 From 71d991153928e05af21e6fefabf6ae3cd0fd627b Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Sun, 15 May 2016 10:23:33 -0700 Subject: go cleanup, adding hub --- var/spack/repos/builtin/packages/gcc/package.py | 2 ++ var/spack/repos/builtin/packages/go/package.py | 2 ++ var/spack/repos/builtin/packages/hub/package.py | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 var/spack/repos/builtin/packages/hub/package.py diff --git a/var/spack/repos/builtin/packages/gcc/package.py b/var/spack/repos/builtin/packages/gcc/package.py index 218f25b586..224105ea0f 100644 --- a/var/spack/repos/builtin/packages/gcc/package.py +++ b/var/spack/repos/builtin/packages/gcc/package.py @@ -46,6 +46,8 @@ class Gcc(Package): if sys.platform == 'darwin': patch('darwin/gcc-4.9.patch1', when='@4.9.3') patch('darwin/gcc-4.9.patch2', when='@4.9.3') + else: + provides('golang', when='@4.7.1:') def install(self, spec, prefix): # libjava/configure needs a minor fix to install into spack paths. diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py index 80924c7fad..90f34dcfc6 100644 --- a/var/spack/repos/builtin/packages/go/package.py +++ b/var/spack/repos/builtin/packages/go/package.py @@ -65,4 +65,6 @@ class Go(Package): shutil.copytree('bin', os.path.join(prefix, '/bin')) """ + if os.environ.get('GOROOT', False): + tty.warn('GOROOT is set, this is not recommended') spack_env.set('GOPATH', ext_spec.package.stage.source_path) diff --git a/var/spack/repos/builtin/packages/hub/package.py b/var/spack/repos/builtin/packages/hub/package.py new file mode 100644 index 0000000000..e82bc7a551 --- /dev/null +++ b/var/spack/repos/builtin/packages/hub/package.py @@ -0,0 +1,20 @@ +from spack import * + +class Hub(Package): + """The github git wrapper""" + homepage = "https://github.com/github/hub" + url = "https://github.com/github/hub/archive/v2.2.3.tar.gz" + + version('head' , git='https://github.com/github/hub') + version('2.2.3' , '6675992ddd16d186eac7ba4484d57f5b') + version('2.2.2' , '7edc8f5b5d3c7c392ee191dd999596fc') + version('2.2.1' , '889a31ee9d10ae9cb333480d8dbe881f') + version('2.2.0' , 'eddce830a079b8480f104aa7496f46fe') + version('1.12.4', '4f2ebb14834c9981b04e40b0d1754717') + + extends("go") + + def install(self, spec, prefix): + os.environ['GOPATH'] = os.getcwd() + bash = which('bash') + bash(os.path.join('script', 'build'), '-o', os.path.join(prefix, 'bin', 'hub')) -- cgit v1.2.3-70-g09d2 From dacedfcc0ebe2bdf15cc5bb5b49d5a61d6aca354 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Mon, 16 May 2016 05:12:30 -0700 Subject: formatting to appease flake8 --- lib/spack/spack/fetch_strategy.py | 202 ++++++++++++--------- lib/spack/spack/util/executable.py | 92 +++++----- var/spack/repos/builtin/packages/hub/package.py | 16 +- .../packages/the_platinum_searcher/package.py | 3 +- 4 files changed, 178 insertions(+), 135 deletions(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 0144d26057..7c8cebe0c9 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -57,7 +57,6 @@ from spack.version import Version, ver from spack.util.compression import decompressor_for, extension import spack.util.pattern as pattern - """List of all fetch strategies, created by FetchStrategy metaclass.""" all_strategies = [] @@ -82,13 +81,16 @@ class FetchStrategy(object): class __metaclass__(type): """This metaclass registers all fetch strategies in a list.""" + def __init__(cls, name, bases, dict): type.__init__(cls, name, bases, dict) - if cls.enabled: all_strategies.append(cls) + if cls.enabled: + all_strategies.append(cls) def __init__(self): - # The stage is initialized late, so that fetch strategies can be constructed - # at package construction time. This is where things will be fetched. + # The stage is initialized late, so that fetch strategies can be + # constructed at package construction time. This is where things + # will be fetched. self.stage = None def set_stage(self, stage): @@ -97,15 +99,20 @@ class FetchStrategy(object): self.stage = stage # Subclasses need to implement these methods - def fetch(self): pass # Return True on success, False on fail. + def fetch(self): + pass # Return True on success, False on fail. - def check(self): pass # Do checksum. + def check(self): + pass # Do checksum. - def expand(self): pass # Expand archive. + def expand(self): + pass # Expand archive. - def reset(self): pass # Revert to freshly downloaded state. + def reset(self): + pass # Revert to freshly downloaded state. - def archive(self, destination): pass # Used to create tarball for mirror. + def archive(self, destination): + pass # Used to create tarball for mirror. def __str__(self): # Should be human readable URL. return "FetchStrategy.__str___" @@ -139,10 +146,12 @@ class URLFetchStrategy(FetchStrategy): # If URL or digest are provided in the kwargs, then prefer # those values. self.url = kwargs.get('url', None) - if not self.url: self.url = url + if not self.url: + self.url = url self.digest = kwargs.get('md5', None) - if not self.digest: self.digest = digest + if not self.digest: + self.digest = digest self.expand_archive = kwargs.get('expand', True) @@ -167,16 +176,20 @@ class URLFetchStrategy(FetchStrategy): tty.msg("Trying to fetch from %s" % self.url) if partial_file: - save_args = ['-C', '-', # continue partial downloads - '-o', partial_file] # use a .part file + save_args = ['-C', + '-', # continue partial downloads + '-o', + partial_file] # use a .part file else: save_args = ['-O'] curl_args = save_args + [ - '-f', # fail on >400 errors - '-D', '-', # print out HTML headers - '-L', # resolve 3xx redirects - self.url, ] + '-f', # fail on >400 errors + '-D', + '-', # print out HTML headers + '-L', # resolve 3xx redirects + self.url, + ] if sys.stdout.isatty(): curl_args.append('-#') # status bar when using a tty @@ -184,8 +197,7 @@ class URLFetchStrategy(FetchStrategy): curl_args.append('-sS') # just errors when not. # Run curl but grab the mime type from the http headers - headers = spack.curl( - *curl_args, output=str, fail_on_error=False) + headers = spack.curl(*curl_args, output=str, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. @@ -198,33 +210,36 @@ class URLFetchStrategy(FetchStrategy): if spack.curl.returncode == 22: # This is a 404. Curl will print the error. raise FailedDownloadError( - self.url, "URL %s was not found!" % self.url) + self.url, "URL %s was not found!" % self.url) elif spack.curl.returncode == 60: # This is a certificate error. Suggest spack -k raise FailedDownloadError( - self.url, - "Curl was unable to fetch due to invalid certificate. " - "This is either an attack, or your cluster's SSL configuration " - "is bad. If you believe your SSL configuration is bad, you " - "can try running spack -k, which will not check SSL certificates." - "Use this at your own risk.") + self.url, + "Curl was unable to fetch due to invalid certificate. " + "This is either an attack, or your cluster's SSL " + "configuration is bad. If you believe your SSL " + "configuration is bad, you can try running spack -k, " + "which will not check SSL certificates." + "Use this at your own risk.") else: # This is some other curl error. Curl will print the # error, but print a spack message too raise FailedDownloadError( - self.url, "Curl failed with error %d" % spack.curl.returncode) + self.url, + "Curl failed with error %d" % spack.curl.returncode) # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle # redirects properly. content_types = re.findall(r'Content-Type:[^\r\n]+', headers) if content_types and 'text/html' in content_types[-1]: - tty.warn("The contents of " + self.archive_file + " look like HTML.", - "The checksum will likely be bad. If it is, you can use", - "'spack clean ' to remove the bad archive, then fix", - "your internet gateway issue and install again.") + tty.warn( + "The contents of " + self.archive_file + " look like HTML.", + "The checksum will likely be bad. If it is, you can use", + "'spack clean ' to remove the bad archive, then fix", + "your internet gateway issue and install again.") if save_file: os.rename(partial_file, save_file) @@ -247,14 +262,16 @@ class URLFetchStrategy(FetchStrategy): self.stage.chdir() if not self.archive_file: - raise NoArchiveFileError("URLFetchStrategy couldn't find archive file", - "Failed on expand() for URL %s" % self.url) + raise NoArchiveFileError( + "URLFetchStrategy couldn't find archive file", + "Failed on expand() for URL %s" % self.url) decompress = decompressor_for(self.archive_file) # Expand all tarballs in their own directory to contain # exploding tarballs. - tarball_container = os.path.join(self.stage.path, "spack-expanded-archive") + tarball_container = os.path.join(self.stage.path, + "spack-expanded-archive") mkdirp(tarball_container) os.chdir(tarball_container) decompress(self.archive_file) @@ -295,20 +312,25 @@ class URLFetchStrategy(FetchStrategy): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" if not self.digest: - raise NoDigestError("Attempt to check URLFetchStrategy with no digest.") + raise NoDigestError( + "Attempt to check URLFetchStrategy with no digest.") checker = crypto.Checker(self.digest) if not checker.check(self.archive_file): raise ChecksumError( - "%s checksum failed for %s" % (checker.hash_name, self.archive_file), - "Expected %s but got %s" % (self.digest, checker.sum)) + "%s checksum failed for %s" % + (checker.hash_name, self.archive_file), + "Expected %s but got %s" % (self.digest, checker.sum)) @_needs_stage def reset(self): - """Removes the source path if it exists, then re-expands the archive.""" + """ + Removes the source path if it exists, then re-expands the archive. + """ if not self.archive_file: - raise NoArchiveFileError("Tried to reset URLFetchStrategy before fetching", - "Failed on reset() for URL %s" % self.url) + raise NoArchiveFileError( + "Tried to reset URLFetchStrategy before fetching", + "Failed on reset() for URL %s" % self.url) # Remove everythigng but the archive from the stage for filename in os.listdir(self.stage.path): @@ -337,14 +359,16 @@ class VCSFetchStrategy(FetchStrategy): # Set a URL based on the type of fetch strategy. self.url = kwargs.get(name, None) - if not self.url: raise ValueError( + if not self.url: + raise ValueError( "%s requires %s argument." % (self.__class__, name)) # Ensure that there's only one of the rev_types if sum(k in kwargs for k in rev_types) > 1: raise FetchStrategyError( - "Supply only one of %s to fetch with %s" % ( - comma_or(rev_types), name)) + "Supply only one of %s to fetch with %s" % ( + comma_or(rev_types), name + )) # Set attributes for each rev type. for rt in rev_types: @@ -381,19 +405,23 @@ class VCSFetchStrategy(FetchStrategy): def __repr__(self): return "%s<%s>" % (self.__class__, self.url) + class GoFetchStrategy(VCSFetchStrategy): - """Fetch strategy that employs the `go get` infrastructure - Use like this in a package: + """ + Fetch strategy that employs the `go get` infrastructure + Use like this in a package: - version('name', go='github.com/monochromegane/the_platinum_searcher/...') + version('name', + go='github.com/monochromegane/the_platinum_searcher/...') - Go get does not natively support versions, they can be faked with git + Go get does not natively support versions, they can be faked with git """ enabled = True - required_attributes = ('go',) + required_attributes = ('go', ) def __init__(self, **kwargs): - # Discards the keywords in kwargs that may conflict with the next call to __init__ + # Discards the keywords in kwargs that may conflict with the next + # call to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop('name', None) @@ -422,8 +450,8 @@ class GoFetchStrategy(VCSFetchStrategy): except OSError: pass env = dict(os.environ) - env['GOPATH'] = os.path.join(os.getcwd(),'go') - self.go('get', '-v', '-d', self.url, env=env) + env['GOPATH'] = os.path.join(os.getcwd(), 'go') + self.go('get', '-v', '-d', self.url, env=env) def archive(self, destination): super(GoFetchStrategy, self).archive(destination, exclude='.git') @@ -436,32 +464,35 @@ class GoFetchStrategy(VCSFetchStrategy): def __str__(self): return "[go] %s" % self.url + class GitFetchStrategy(VCSFetchStrategy): - """Fetch strategy that gets source code from a git repository. - Use like this in a package: + """ + Fetch strategy that gets source code from a git repository. + Use like this in a package: - version('name', git='https://github.com/project/repo.git') + version('name', git='https://github.com/project/repo.git') - Optionally, you can provide a branch, or commit to check out, e.g.: + Optionally, you can provide a branch, or commit to check out, e.g.: - version('1.1', git='https://github.com/project/repo.git', tag='v1.1') + version('1.1', git='https://github.com/project/repo.git', tag='v1.1') - You can use these three optional attributes in addition to ``git``: + You can use these three optional attributes in addition to ``git``: - * ``branch``: Particular branch to build from (default is master) - * ``tag``: Particular tag to check out - * ``commit``: Particular commit hash in the repo + * ``branch``: Particular branch to build from (default is master) + * ``tag``: Particular tag to check out + * ``commit``: Particular commit hash in the repo """ enabled = True - required_attributes = ('git',) + required_attributes = ('git', ) def __init__(self, **kwargs): - # Discards the keywords in kwargs that may conflict with the next call to __init__ + # Discards the keywords in kwargs that may conflict with the next call + # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop('name', None) super(GitFetchStrategy, self).__init__( - 'git', 'tag', 'branch', 'commit', **forwarded_args) + 'git', 'tag', 'branch', 'commit', **forwarded_args) self._git = None @property @@ -569,12 +600,13 @@ class SvnFetchStrategy(VCSFetchStrategy): required_attributes = ['svn'] def __init__(self, **kwargs): - # Discards the keywords in kwargs that may conflict with the next call to __init__ + # Discards the keywords in kwargs that may conflict with the next call + # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop('name', None) super(SvnFetchStrategy, self).__init__( - 'svn', 'revision', **forwarded_args) + 'svn', 'revision', **forwarded_args) self._svn = None if self.revision is not None: self.revision = str(self.revision) @@ -630,32 +662,35 @@ class SvnFetchStrategy(VCSFetchStrategy): class HgFetchStrategy(VCSFetchStrategy): - """Fetch strategy that gets source code from a Mercurial repository. - Use like this in a package: + """ + Fetch strategy that gets source code from a Mercurial repository. + Use like this in a package: - version('name', hg='https://jay.grs.rwth-aachen.de/hg/lwm2') + version('name', hg='https://jay.grs.rwth-aachen.de/hg/lwm2') - Optionally, you can provide a branch, or revision to check out, e.g.: + Optionally, you can provide a branch, or revision to check out, e.g.: - version('torus', hg='https://jay.grs.rwth-aachen.de/hg/lwm2', branch='torus') + version('torus', + hg='https://jay.grs.rwth-aachen.de/hg/lwm2', branch='torus') - You can use the optional 'revision' attribute to check out a - branch, tag, or particular revision in hg. To prevent - non-reproducible builds, using a moving target like a branch is - discouraged. + You can use the optional 'revision' attribute to check out a + branch, tag, or particular revision in hg. To prevent + non-reproducible builds, using a moving target like a branch is + discouraged. - * ``revision``: Particular revision, branch, or tag. + * ``revision``: Particular revision, branch, or tag. """ enabled = True required_attributes = ['hg'] def __init__(self, **kwargs): - # Discards the keywords in kwargs that may conflict with the next call to __init__ + # Discards the keywords in kwargs that may conflict with the next call + # to __init__ forwarded_args = copy.copy(kwargs) forwarded_args.pop('name', None) super(HgFetchStrategy, self).__init__( - 'hg', 'revision', **forwarded_args) + 'hg', 'revision', **forwarded_args) self._hg = None @property @@ -729,7 +764,8 @@ def from_kwargs(**kwargs): return fetcher(**kwargs) # Raise an error in case we can't instantiate any known strategy message = "Cannot instantiate any FetchStrategy" - long_message = message + " from the given arguments : {arguments}".format(srguments=kwargs) + long_message = message + " from the given arguments : {arguments}".format( + srguments=kwargs) raise FetchError(message, long_message) @@ -741,7 +777,7 @@ def for_package_version(pkg, version): """Determine a fetch strategy based on the arguments supplied to version() in the package description.""" # If it's not a known version, extrapolate one. - if not version in pkg.versions: + if version not in pkg.versions: url = pkg.url_for_version(version) if not url: raise InvalidArgsError(pkg, version) @@ -779,7 +815,7 @@ class FailedDownloadError(FetchError): def __init__(self, url, msg=""): super(FailedDownloadError, self).__init__( - "Failed to fetch file from URL: %s" % url, msg) + "Failed to fetch file from URL: %s" % url, msg) self.url = url @@ -795,7 +831,8 @@ class NoDigestError(FetchError): class InvalidArgsError(FetchError): def __init__(self, pkg, version): - msg = "Could not construct a fetch strategy for package %s at version %s" + msg = ("Could not construct a fetch strategy for package %s at " + "version %s") msg %= (pkg.name, version) super(InvalidArgsError, self).__init__(msg) @@ -812,4 +849,5 @@ class NoStageError(FetchError): def __init__(self, method): super(NoStageError, self).__init__( - "Must call FetchStrategy.set_stage() before calling %s" % method.__name__) + "Must call FetchStrategy.set_stage() before calling %s" % + method.__name__) diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 0b91b6c360..38b778fa00 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -22,10 +22,8 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -__all__ = ['Executable', 'which', 'ProcessError'] import os -import sys import re import subprocess import inspect @@ -34,9 +32,12 @@ import llnl.util.tty as tty import spack import spack.error +__all__ = ['Executable', 'which', 'ProcessError'] + class Executable(object): """Class representing a program that can be run on the command line.""" + def __init__(self, name): self.exe = name.split(' ') self.returncode = None @@ -44,16 +45,13 @@ class Executable(object): if not self.exe: raise ProcessError("Cannot construct executable for '%s'" % name) - def add_default_arg(self, arg): self.exe.append(arg) - @property def command(self): return ' '.join(self.exe) - def __call__(self, *args, **kwargs): """Run this executable in a subprocess. @@ -105,7 +103,7 @@ class Executable(object): fail_on_error = kwargs.pop("fail_on_error", True) ignore_errors = kwargs.pop("ignore_errors", ()) - env = kwargs.get('env', None) + env = kwargs.get('env', None) # TODO: This is deprecated. Remove in a future version. return_output = kwargs.pop("return_output", False) @@ -116,8 +114,8 @@ class Executable(object): else: output = kwargs.pop("output", None) - error = kwargs.pop("error", None) - input = kwargs.pop("input", None) + error = kwargs.pop("error", None) + input = kwargs.pop("input", None) if input is str: raise ValueError("Cannot use `str` as input stream.") @@ -128,85 +126,90 @@ class Executable(object): return subprocess.PIPE, False else: return arg, False + ostream, close_ostream = streamify(output, 'w') - estream, close_estream = streamify(error, 'w') - istream, close_istream = streamify(input, 'r') + estream, close_estream = streamify(error, 'w') + istream, close_istream = streamify(input, 'r') # if they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): - ignore_errors = (ignore_errors,) + ignore_errors = (ignore_errors, ) quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] if quoted_args: - tty.warn("Quotes in command arguments can confuse scripts like configure.", - "The following arguments may cause problems when executed:", - str("\n".join([" "+arg for arg in quoted_args])), - "Quotes aren't needed because spack doesn't use a shell.", - "Consider removing them") + tty.warn( + "Quotes in command arguments can confuse scripts like" + " configure.", + "The following arguments may cause problems when executed:", + str("\n".join([" " + arg for arg in quoted_args])), + "Quotes aren't needed because spack doesn't use a shell.", + "Consider removing them") cmd = self.exe + list(args) - cmd_line = "'%s'" % "' '".join(map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) + cmd_line = "'%s'" % "' '".join( + map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) tty.debug(cmd_line) try: proc = subprocess.Popen( - cmd, stdin=istream, stderr=estream, stdout=ostream, env=env) + cmd, + stdin=istream, + stderr=estream, + stdout=ostream, + env=env) out, err = proc.communicate() rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): - raise ProcessError("Command exited with status %d:" - % proc.returncode, cmd_line) + raise ProcessError("Command exited with status %d:" % + proc.returncode, cmd_line) if output is str or error is str: result = '' - if output is str: result += out - if error is str: result += err + if output is str: + result += out + if error is str: + result += err return result except OSError, e: raise ProcessError( - "%s: %s" % (self.exe[0], e.strerror), - "Command: " + cmd_line) + "%s: %s" % (self.exe[0], e.strerror), "Command: " + cmd_line) except subprocess.CalledProcessError, e: if fail_on_error: raise ProcessError( - str(e), - "\nExit status %d when invoking command: %s" - % (proc.returncode, cmd_line)) + str(e), "\nExit status %d when invoking command: %s" % + (proc.returncode, cmd_line)) finally: - if close_ostream: output.close() - if close_estream: error.close() - if close_istream: input.close() - + if close_ostream: + output.close() + if close_estream: + error.close() + if close_istream: + input.close() def __eq__(self, other): return self.exe == other.exe - def __neq__(self, other): return not (self == other) - def __hash__(self): - return hash((type(self),) + tuple(self.exe)) - + return hash((type(self), ) + tuple(self.exe)) def __repr__(self): return "" % self.exe - def __str__(self): return ' '.join(self.exe) - def which(name, **kwargs): """Finds an executable in the path like command-line which.""" - path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) + path = kwargs.get('path', os.environ.get('PATH', '').split(os.pathsep)) required = kwargs.get('required', False) if not path: @@ -235,14 +238,16 @@ class ProcessError(spack.error.SpackError): @property def long_message(self): msg = self._long_message - if msg: msg += "\n\n" + if msg: + msg += "\n\n" if self.build_log: msg += "See build log for details:\n" msg += " %s" % self.build_log if self.package_context: - if msg: msg += "\n\n" + if msg: + msg += "\n\n" msg += '\n'.join(self.package_context) return msg @@ -269,7 +274,7 @@ def _get_package_context(): frame = f[0] # Find a frame with 'self' in the local variables. - if not 'self' in frame.f_locals: + if 'self' not in frame.f_locals: continue # Look only at a frame in a subclass of spack.Package @@ -282,9 +287,8 @@ def _get_package_context(): # Build a message showing where in install we failed. lines.append("%s:%d, in %s:" % ( - inspect.getfile(frame.f_code), - frame.f_lineno, - frame.f_code.co_name)) + inspect.getfile(frame.f_code), frame.f_lineno, frame.f_code.co_name + )) sourcelines, start = inspect.getsourcelines(frame) for i, line in enumerate(sourcelines): diff --git a/var/spack/repos/builtin/packages/hub/package.py b/var/spack/repos/builtin/packages/hub/package.py index e82bc7a551..5f3926533a 100644 --- a/var/spack/repos/builtin/packages/hub/package.py +++ b/var/spack/repos/builtin/packages/hub/package.py @@ -1,15 +1,16 @@ from spack import * + class Hub(Package): """The github git wrapper""" homepage = "https://github.com/github/hub" - url = "https://github.com/github/hub/archive/v2.2.3.tar.gz" + url = "https://github.com/github/hub/archive/v2.2.3.tar.gz" - version('head' , git='https://github.com/github/hub') - version('2.2.3' , '6675992ddd16d186eac7ba4484d57f5b') - version('2.2.2' , '7edc8f5b5d3c7c392ee191dd999596fc') - version('2.2.1' , '889a31ee9d10ae9cb333480d8dbe881f') - version('2.2.0' , 'eddce830a079b8480f104aa7496f46fe') + version('head', git='https://github.com/github/hub') + version('2.2.3', '6675992ddd16d186eac7ba4484d57f5b') + version('2.2.2', '7edc8f5b5d3c7c392ee191dd999596fc') + version('2.2.1', '889a31ee9d10ae9cb333480d8dbe881f') + version('2.2.0', 'eddce830a079b8480f104aa7496f46fe') version('1.12.4', '4f2ebb14834c9981b04e40b0d1754717') extends("go") @@ -17,4 +18,5 @@ class Hub(Package): def install(self, spec, prefix): os.environ['GOPATH'] = os.getcwd() bash = which('bash') - bash(os.path.join('script', 'build'), '-o', os.path.join(prefix, 'bin', 'hub')) + bash(os.path.join('script', 'build'), '-o', os.path.join(prefix, 'bin', + 'hub')) diff --git a/var/spack/repos/builtin/packages/the_platinum_searcher/package.py b/var/spack/repos/builtin/packages/the_platinum_searcher/package.py index c040da3699..da21a0f656 100644 --- a/var/spack/repos/builtin/packages/the_platinum_searcher/package.py +++ b/var/spack/repos/builtin/packages/the_platinum_searcher/package.py @@ -5,9 +5,8 @@ import shutil class ThePlatinumSearcher(Package): """Fast parallel recursive grep alternative""" - # FIXME: add a proper url for your package's homepage here. homepage = "https://github.com/monochromegane/the_platinum_searcher" - url = "https://github.com/monochromegane/the_platinum_searcher" + url = "https://github.com/monochromegane/the_platinum_searcher" package = 'github.com/monochromegane/the_platinum_searcher/...' -- cgit v1.2.3-70-g09d2 From 62dd040a8fd7f6f8993640156049eeb061efe955 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Mon, 16 May 2016 09:30:37 -0700 Subject: add GOPATH support Also fixes flex to depend on bison for its yacc implementation. --- var/spack/repos/builtin/packages/flex/package.py | 5 ++++- var/spack/repos/builtin/packages/go/package.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/var/spack/repos/builtin/packages/flex/package.py b/var/spack/repos/builtin/packages/flex/package.py index 926651010f..b778538606 100644 --- a/var/spack/repos/builtin/packages/flex/package.py +++ b/var/spack/repos/builtin/packages/flex/package.py @@ -24,15 +24,18 @@ ############################################################################## from spack import * + class Flex(Package): """Flex is a tool for generating scanners.""" homepage = "http://flex.sourceforge.net/" - url = "http://download.sourceforge.net/flex/flex-2.5.39.tar.gz" + url = "http://download.sourceforge.net/flex/flex-2.5.39.tar.gz" version('2.6.0', '5724bcffed4ebe39e9b55a9be80859ec') version('2.5.39', 'e133e9ead8ec0a58d81166b461244fde') + depends_on("bison") + def install(self, spec, prefix): configure("--prefix=%s" % prefix) diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py index 90f34dcfc6..f1057e6bed 100644 --- a/var/spack/repos/builtin/packages/go/package.py +++ b/var/spack/repos/builtin/packages/go/package.py @@ -67,4 +67,14 @@ class Go(Package): if os.environ.get('GOROOT', False): tty.warn('GOROOT is set, this is not recommended') - spack_env.set('GOPATH', ext_spec.package.stage.source_path) + + # Set GOPATH to include paths of dependencies + for d in extension_spec.traverse(): + if d.package.extends(self.spec): + spack_env.prepend_path('GOPATH', d.prefix) + + # This *MUST* be first, this is where new code is installed + spack_env.prepend_path('GOPATH', ext_spec.package.stage.source_path) + + # Allow packages to find this when using module or dotkit + run_env.prepend_path('GOPATH', ext_spec.prefix) -- cgit v1.2.3-70-g09d2 From af4af94203d0dc7cebd110c2bd43707e527c63cb Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 17 May 2016 05:21:11 -0700 Subject: rework for gopath and bootstrapping --- .../repos/builtin/packages/go-bootstrap/package.py | 51 ++++++++++++++++++++++ var/spack/repos/builtin/packages/go/package.py | 36 +++++++-------- var/spack/repos/builtin/packages/hub/package.py | 4 +- .../packages/the_platinum_searcher/package.py | 4 +- 4 files changed, 76 insertions(+), 19 deletions(-) create mode 100644 var/spack/repos/builtin/packages/go-bootstrap/package.py diff --git a/var/spack/repos/builtin/packages/go-bootstrap/package.py b/var/spack/repos/builtin/packages/go-bootstrap/package.py new file mode 100644 index 0000000000..fd0b5bc5d1 --- /dev/null +++ b/var/spack/repos/builtin/packages/go-bootstrap/package.py @@ -0,0 +1,51 @@ +import os +import shutil +import glob +from spack import * + +# THIS PACKAGE SHOULD NOT EXIST +# it exists to make up for the inability to: +# * use an external go compiler +# * have go depend on itself +# * have a sensible way to find gccgo without a dep on gcc + +class GoBootstrap(Package): + """Old C-bootstrapped go to bootstrap real go""" + homepage = "https://golang.org" + url = "https://go.googlesource.com/go" + + extendable = True + + # temporary fix until tags are pulled correctly + version('1.4.2', git='https://go.googlesource.com/go', tag='go1.4.2') + + variant('test', + default=True, + description="Run tests as part of build, a good idea but quite" + " time consuming") + + provides('golang@:1.4.2') + + depends_on('git') + + def install(self, spec, prefix): + bash = which('bash') + with working_dir('src'): + if '+test' in spec: + bash('all.bash') + else: + bash('make.bash') + + try: + os.makedirs(prefix) + except OSError: + pass + for f in glob.glob('*'): + if os.path.isdir(f): + shutil.copytree(f, os.path.join(prefix, f)) + else: + shutil.copy2(f, os.path.join(prefix, f)) + + def setup_environment(self, spack_env, run_env): + spack_env.set('GOROOT_FINAL', self.spec.prefix) + diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py index f1057e6bed..28e0d1ba7f 100644 --- a/var/spack/repos/builtin/packages/go/package.py +++ b/var/spack/repos/builtin/packages/go/package.py @@ -1,6 +1,7 @@ import os import shutil import glob +import llnl.util.tty as tty from spack import * @@ -11,8 +12,6 @@ class Go(Package): extendable = True - # temporary fix until tags are pulled correctly - version('1.4.2', git='https://go.googlesource.com/go', tag='go1.4.2') version('1.5.4', git='https://go.googlesource.com/go', tag='go1.5.4') version('1.6.2', git='https://go.googlesource.com/go', tag='go1.6.2') @@ -24,8 +23,8 @@ class Go(Package): provides('golang') # to-do, make non-c self-hosting compilers feasible without backflips - # should be go_compiler, but that creates an infinite loop - depends_on('gcc@5:', when='@1.5:') + # should be a dep on external go compiler + depends_on('go-bootstrap') depends_on('git') def install(self, spec, prefix): @@ -47,34 +46,37 @@ class Go(Package): shutil.copy2(f, os.path.join(prefix, f)) def setup_environment(self, spack_env, run_env): - # spack_env.set("GOROOT", self.spec.prefix) - # run_env.set("GOROOT", self.spec.prefix) spack_env.set('GOROOT_FINAL', self.spec.prefix) - spack_env.set('GOROOT_BOOTSTRAP', self.spec['gcc'].prefix) + spack_env.set('GOROOT_BOOTSTRAP', self.spec['go-bootstrap'].prefix) def setup_dependent_package(self, module, ext_spec): - # Add a go command/compiler for extensions - module.go = Executable(join_path(self.spec.prefix.bin, 'go')) - - def setup_dependent_environment(self, spack_env, run_env, ext_spec): """Called before go modules' install() methods. - In most cases, extensions will only need to have two lines:: + In most cases, extensions will only need to set GOPATH and use go:: - go('get', '') + env = os.environ + env['GOPATH'] = self.source_path + ':' + env['GOPATH'] + go('get', '', env=env) shutil.copytree('bin', os.path.join(prefix, '/bin')) """ + # Add a go command/compiler for extensions + module.go = Executable(join_path(self.spec.prefix.bin, 'go')) + def setup_dependent_environment(self, spack_env, run_env, ext_spec): if os.environ.get('GOROOT', False): tty.warn('GOROOT is set, this is not recommended') + path_components = [] # Set GOPATH to include paths of dependencies - for d in extension_spec.traverse(): + for d in ext_spec.traverse(): if d.package.extends(self.spec): - spack_env.prepend_path('GOPATH', d.prefix) + path_components.append(d.prefix) # This *MUST* be first, this is where new code is installed - spack_env.prepend_path('GOPATH', ext_spec.package.stage.source_path) + spack_env.set('GOPATH', ':'.join(path_components)) # Allow packages to find this when using module or dotkit - run_env.prepend_path('GOPATH', ext_spec.prefix) + run_env.prepend_path('GOPATH', ':'.join( + [ext_spec.prefix] + path_components)) + + diff --git a/var/spack/repos/builtin/packages/hub/package.py b/var/spack/repos/builtin/packages/hub/package.py index 5f3926533a..ed8b742e42 100644 --- a/var/spack/repos/builtin/packages/hub/package.py +++ b/var/spack/repos/builtin/packages/hub/package.py @@ -1,4 +1,5 @@ from spack import * +import os class Hub(Package): @@ -16,7 +17,8 @@ class Hub(Package): extends("go") def install(self, spec, prefix): - os.environ['GOPATH'] = os.getcwd() + env = os.environ + env['GOPATH'] = self.stage.source_path + ':' + env['GOPATH'] bash = which('bash') bash(os.path.join('script', 'build'), '-o', os.path.join(prefix, 'bin', 'hub')) diff --git a/var/spack/repos/builtin/packages/the_platinum_searcher/package.py b/var/spack/repos/builtin/packages/the_platinum_searcher/package.py index da21a0f656..9c9a66cdef 100644 --- a/var/spack/repos/builtin/packages/the_platinum_searcher/package.py +++ b/var/spack/repos/builtin/packages/the_platinum_searcher/package.py @@ -15,5 +15,7 @@ class ThePlatinumSearcher(Package): extends("go") def install(self, spec, prefix): - go('install', self.package) + env = os.environ + env['GOPATH'] = self.stage.source_path + ':' + env['GOPATH'] + go('install', self.package, env=env) shutil.copytree('bin', os.path.join(prefix, 'bin')) -- cgit v1.2.3-70-g09d2 From 80c02ad36af00b31f9b4ed4f3d1de72ae963ed98 Mon Sep 17 00:00:00 2001 From: Tom Scogland Date: Tue, 17 May 2016 05:30:35 -0700 Subject: flake8-related cleanup --- var/spack/repos/builtin/packages/go-bootstrap/package.py | 2 +- var/spack/repos/builtin/packages/go/package.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/var/spack/repos/builtin/packages/go-bootstrap/package.py b/var/spack/repos/builtin/packages/go-bootstrap/package.py index fd0b5bc5d1..b0e2109fd3 100644 --- a/var/spack/repos/builtin/packages/go-bootstrap/package.py +++ b/var/spack/repos/builtin/packages/go-bootstrap/package.py @@ -9,6 +9,7 @@ from spack import * # * have go depend on itself # * have a sensible way to find gccgo without a dep on gcc + class GoBootstrap(Package): """Old C-bootstrapped go to bootstrap real go""" homepage = "https://golang.org" @@ -48,4 +49,3 @@ class GoBootstrap(Package): def setup_environment(self, spack_env, run_env): spack_env.set('GOROOT_FINAL', self.spec.prefix) - diff --git a/var/spack/repos/builtin/packages/go/package.py b/var/spack/repos/builtin/packages/go/package.py index 28e0d1ba7f..13b83517d1 100644 --- a/var/spack/repos/builtin/packages/go/package.py +++ b/var/spack/repos/builtin/packages/go/package.py @@ -78,5 +78,3 @@ class Go(Package): # Allow packages to find this when using module or dotkit run_env.prepend_path('GOPATH', ':'.join( [ext_spec.prefix] + path_components)) - - -- cgit v1.2.3-70-g09d2