From 488e1bab2ca384863517375f47be33dca4f170f8 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Tue, 11 Oct 2016 23:13:40 -0700 Subject: Make `insecure` option work with curl AND git. (#1786) --- bin/spack | 2 +- lib/spack/spack/__init__.py | 5 +++-- lib/spack/spack/fetch_strategy.py | 30 ++++++++++++++++++++----- lib/spack/spack/util/executable.py | 11 +++++++++ var/spack/repos/builtin/packages/jdk/package.py | 26 +++++---------------- 5 files changed, 46 insertions(+), 28 deletions(-) diff --git a/bin/spack b/bin/spack index dcc25c5c83..503ef068bf 100755 --- a/bin/spack +++ b/bin/spack @@ -161,7 +161,7 @@ def main(): # If the user asked for it, don't check ssl certs. if args.insecure: tty.warn("You asked for --insecure. Will NOT check SSL certificates.") - spack.curl.add_default_arg('-k') + spack.insecure = True # Try to load the particular command asked for and run it command = spack.cmd.get_command(args.command.replace('-', '_')) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index 3d508d0fde..e284a58194 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -129,8 +129,9 @@ from spack.util.executable import Executable, which # User's editor from the environment editor = Executable(os.environ.get("EDITOR", "vi")) -# Curl tool for fetching files. -curl = which("curl", required=True) +# If this is enabled, tools that use SSL should not verify +# certifiates. e.g., curl should use the -k option. +insecure = False # Whether to build in tmp space or directly in the stage_path. # If this is true, then spack will make stage directories in diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 4b8829c32f..2becf9ed04 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -158,12 +158,20 @@ class URLFetchStrategy(FetchStrategy): self.digest = digest self.expand_archive = kwargs.get('expand', True) + self.extra_curl_options = kwargs.get('curl_options', []) + self._curl = None self.extension = kwargs.get('extension', None) if not self.url: raise ValueError("URLFetchStrategy requires a url for fetching.") + @property + def curl(self): + if not self._curl: + self._curl = which('curl', required=True) + return self._curl + @_needs_stage def fetch(self): self.stage.chdir() @@ -196,15 +204,21 @@ class URLFetchStrategy(FetchStrategy): self.url, ] + if spack.insecure: + curl_args.append('-k') + if sys.stdout.isatty(): curl_args.append('-#') # status bar when using a tty else: curl_args.append('-sS') # just errors when not. + curl_args += self.extra_curl_options + # Run curl but grab the mime type from the http headers - headers = spack.curl(*curl_args, output=str, fail_on_error=False) + curl = self.curl + headers = curl(*curl_args, output=str, fail_on_error=False) - if spack.curl.returncode != 0: + if curl.returncode != 0: # clean up archive on failure. if self.archive_file: os.remove(self.archive_file) @@ -212,12 +226,12 @@ class URLFetchStrategy(FetchStrategy): if partial_file and os.path.exists(partial_file): os.remove(partial_file) - if spack.curl.returncode == 22: + if curl.returncode == 22: # This is a 404. Curl will print the error. raise FailedDownloadError( self.url, "URL %s was not found!" % self.url) - elif spack.curl.returncode == 60: + elif curl.returncode == 60: # This is a certificate error. Suggest spack -k raise FailedDownloadError( self.url, @@ -233,7 +247,7 @@ class URLFetchStrategy(FetchStrategy): # error, but print a spack message too raise FailedDownloadError( self.url, - "Curl failed with error %d" % spack.curl.returncode) + "Curl failed with error %d" % curl.returncode) # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle @@ -530,6 +544,12 @@ class GitFetchStrategy(VCSFetchStrategy): def git(self): if not self._git: self._git = which('git', required=True) + + # If the user asked for insecure fetching, make that work + # with git as well. + if spack.insecure: + self._git.add_default_env('GIT_SSL_NO_VERIFY', 'true') + return self._git @_needs_stage diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index 4fe4bd26ba..2790508ee8 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -40,6 +40,7 @@ class Executable(object): def __init__(self, name): self.exe = name.split(' ') + self.default_env = {} self.returncode = None if not self.exe: @@ -48,6 +49,9 @@ class Executable(object): def add_default_arg(self, arg): self.exe.append(arg) + def add_default_env(self, key, value): + self.default_env[key] = value + @property def command(self): return ' '.join(self.exe) @@ -103,7 +107,13 @@ class Executable(object): fail_on_error = kwargs.pop("fail_on_error", True) ignore_errors = kwargs.pop("ignore_errors", ()) + # environment env = kwargs.get('env', None) + if env is None: + env = os.environ.copy() + env.update(self.default_env) + else: + env = self.default_env.copy().update(env) # TODO: This is deprecated. Remove in a future version. return_output = kwargs.pop("return_output", False) @@ -149,6 +159,7 @@ class Executable(object): cmd_line = "'%s'" % "' '".join( map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) + tty.debug(cmd_line) try: diff --git a/var/spack/repos/builtin/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py index bab0920434..518a469435 100644 --- a/var/spack/repos/builtin/packages/jdk/package.py +++ b/var/spack/repos/builtin/packages/jdk/package.py @@ -26,10 +26,7 @@ # Author: Justin Too # import distutils.dir_util - -import spack from spack import * -import llnl.util.tty as tty class Jdk(Package): @@ -37,11 +34,6 @@ class Jdk(Package): in the form of a binary product aimed at Java developers.""" homepage = "http://www.oracle.com/technetwork/java/javase/downloads/index.html" - version('8u66-linux-x64', '88f31f3d642c3287134297b8c10e61bf', - url="http://download.oracle.com/otn-pub/java/jdk/8u66-b17/jdk-8u66-linux-x64.tar.gz") - version('8u92-linux-x64', '65a1cc17ea362453a6e0eb4f13be76e4', - url="http://download.oracle.com/otn-pub/java/jdk/8u92-b14/jdk-8u92-linux-x64.tar.gz") - # Oracle requires that you accept their License Agreement in order # to access the Java packages in download.oracle.com. In order to # automate this process, we need to utilize these additional curl @@ -53,18 +45,12 @@ class Jdk(Package): '-H', # specify required License Agreement cookie 'Cookie: oraclelicense=accept-securebackup-cookie'] - def do_fetch(self, mirror_only=False): - # Add our custom curl commandline options - tty.msg( - "[Jdk] Adding required commandline options to curl " + - "before performing fetch: %s" % - (self.curl_options)) - - for option in self.curl_options: - spack.curl.add_default_arg(option) - - # Now perform the actual fetch - super(Jdk, self).do_fetch(mirror_only) + version('8u66-linux-x64', '88f31f3d642c3287134297b8c10e61bf', + url="http://download.oracle.com/otn-pub/java/jdk/8u66-b17/jdk-8u66-linux-x64.tar.gz", + curl_options=curl_options) + version('8u92-linux-x64', '65a1cc17ea362453a6e0eb4f13be76e4', + url="http://download.oracle.com/otn-pub/java/jdk/8u92-b14/jdk-8u92-linux-x64.tar.gz", + curl_options=curl_options) def install(self, spec, prefix): distutils.dir_util.copy_tree(".", prefix) -- cgit v1.2.3-70-g09d2