Java updates: new versions, classpath, and JAVA_HOME (#8613)

This PR includes the following changes:

* Added JDK 10
* Changed the JDK version numbers according to the consensus reached
  in #2284
* Added spec['java'].home and spec['java'].libs, similar to #3367
  (JDK and IcedTea)
* Added a check to prevent people from installing JDK on macOS
* Set CLASSPATH for packages depending on Java (JDK and IcedTea)
* Add TODO for extending virtual packages (not currently possible)
* Add TODO for adding Java dependents to views
* Add TODO for packages which extend multiple packages (e.g. Java
  and Python)

6 files changed, 194 insertions, 46 deletions

diff --git a/var/spack/repos/builtin/packages/icedtea/package.py b/var/spack/repos/builtin/packages/icedtea/package.py
index bc0b5ddc66..366606cf44 100644
--- a/var/spack/repos/builtin/packages/icedtea/package.py
+++ b/var/spack/repos/builtin/packages/icedtea/package.py
@@ -37,7 +37,6 @@ class Icedtea(AutotoolsPackage):
 
     version('3.4.0',  'eba66765b92794495e16b83f23640872')
 
-    provides('java@8', when='@3.4.0:3.99.99')
     variant('X', default=False, description="Build with GUI support.")
     variant('shenandoah', default=False,
             description="Build with the shenandoah gc. Only for version 3+")
@@ -82,6 +81,9 @@ class Icedtea(AutotoolsPackage):
     depends_on('zlib')
     depends_on('alsa-lib')
 
+    provides('java')
+    provides('java@8', when='@3.4.0:3.99.99')
+
     force_autoreconf = True
 
     resource(name='corba', placement='corba_src',
@@ -139,6 +141,20 @@ class Icedtea(AutotoolsPackage):
              url='http://icedtea.wildebeest.org/download/drops/icedtea8/3.4.0/shenandoah.tar.xz',
              when='@3.4.0')
 
+    # FIXME:
+    # 1. `extends('java')` doesn't work, you need to use `extends('icedtea')`
+    # 2. Packages cannot extend multiple packages, see #987
+    # 3. Update `YamlFilesystemView.merge` to allow a Package to completely
+    #    override how it is symlinked into a view prefix. Then, spack activate
+    #    can symlink all *.jar files to `prefix.lib.ext`
+    extendable = True
+
+    @property
+    def home(self):
+        """For compatibility with the ``jdk`` package, so that other packages
+        can say ``spec['java'].home`` regardless of the Java provider."""
+        return self.prefix
+
     def configure_args(self):
         os.environ['POTENTIAL_CXX'] = os.environ['CXX']
         os.environ['POTENTIAL_CC'] = os.environ['CC']
@@ -175,7 +191,34 @@ class Icedtea(AutotoolsPackage):
         return args
 
     def setup_environment(self, spack_env, run_env):
-        run_env.set('JAVA_HOME', self.spec.prefix)
+        """Set JAVA_HOME."""
+
+        run_env.set('JAVA_HOME', self.home)
 
     def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
-        spack_env.set('JAVA_HOME', self.prefix)
+        """Set JAVA_HOME and CLASSPATH.
+
+        CLASSPATH contains the installation prefix for the extension and any
+        other Java extensions it depends on."""
+
+        spack_env.set('JAVA_HOME', self.home)
+
+        class_paths = []
+        for d in dependent_spec.traverse(deptype=('build', 'run', 'test')):
+            if d.package.extends(self.spec):
+                class_paths.extend(find(d.prefix, '*.jar'))
+
+        classpath = os.pathsep.join(class_paths)
+        spack_env.set('CLASSPATH', classpath)
+
+        # For runtime environment set only the path for
+        # dependent_spec and prepend it to CLASSPATH
+        if dependent_spec.package.extends(self.spec):
+            class_paths = find(dependent_spec.prefix, '*.jar')
+            classpath = os.pathsep.join(class_paths)
+            run_env.prepend_path('CLASSPATH', classpath)
+
+    def setup_dependent_package(self, module, dependent_spec):
+        """Allows spec['java'].home to work."""
+
+        self.spec.home = self.home
diff --git a/var/spack/repos/builtin/packages/jdk/package.py b/var/spack/repos/builtin/packages/jdk/package.py
index f6f78b9501..ecd2690c7f 100644
--- a/var/spack/repos/builtin/packages/jdk/package.py
+++ b/var/spack/repos/builtin/packages/jdk/package.py
@@ -22,59 +22,169 @@
 # License along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-#
-# Author: Justin Too <too1@llnl.gov>
-#
+from spack.util.prefix import Prefix
 from spack import *
 
+import llnl.util.tty as tty
+import os
+
 
 class Jdk(Package):
-    """The Java Development Kit (JDK) released by Oracle Corporation
-       in the form of a binary product aimed at Java developers."""
+    """The Java Development Kit (JDK) released by Oracle Corporation in the
+    form of a binary product aimed at Java developers. Includes a complete JRE
+    plus tools for developing, debugging, and monitoring Java applications."""
+
     homepage = "http://www.oracle.com/technetwork/java/javase/downloads/index.html"
 
+    maintainers = ['justintoo']
+
     # Oracle requires that you accept their License Agreement in order
     # to access the Java packages in download.oracle.com. In order to
     # automate this process, we need to utilize these additional curl
-    # commandline options.
-    #
-    # See http://stackoverflow.com/questions/10268583/how-to-automate-download-and-installation-of-java-jdk-on-linux
+    # command-line options. See:
+    # http://stackoverflow.com/questions/10268583/how-to-automate-download-and-installation-of-java-jdk-on-linux
     curl_options = [
         '-j',  # junk cookies
         '-H',  # specify required License Agreement cookie
-        'Cookie: oraclelicense=accept-securebackup-cookie']
+        'Cookie: oraclelicense=accept-securebackup-cookie'
+    ]
 
-    # For instructions on how to find the magic URL, see:
-    # https://gist.github.com/P7h/9741922
-    # https://linuxconfig.org/how-to-install-java-se-development-kit-on-debian-linux
-    version('8u172-b11', 'eda2945e8c02b84adbf78f46c37b71c1', curl_options=curl_options,
+    # To add the latest version, go to the homepage listed above,
+    # click "JDK Download", click "Accept License Agreement", right-click the
+    # Linux .tar.gz link, and select Copy Link Address. The checksum can be
+    # found in a link above. The build number can be deciphered from the URL.
+    # Alternatively, run `bin/java -version` after extracting. Replace '+'
+    # symbol in version with '_', otherwise it will be interpreted as a variant
+    version('10.0.1_10', 'ae8ed645e6af38432a56a847597ac61d4283b7536688dbab44ab536199d1e5a4', curl_options=curl_options,
+            url='http://download.oracle.com/otn-pub/java/jdk/10.0.1+10/fb4372174a714e6b8c52526dc134031e/jdk-10.0.1_linux-x64_bin.tar.gz')
+    version('1.8.0_172-b11', 'eda2945e8c02b84adbf78f46c37b71c1', curl_options=curl_options,
             url='http://download.oracle.com/otn-pub/java/jdk/8u172-b11/a58eab1ec242421181065cdc37240b08/jdk-8u172-linux-x64.tar.gz')
-    version('8u141-b15', '8cf4c4e00744bfafc023d770cb65328c', curl_options=curl_options,
+    version('1.8.0_141-b15', '8cf4c4e00744bfafc023d770cb65328c', curl_options=curl_options,
             url='http://download.oracle.com/otn-pub/java/jdk/8u141-b15/336fa29ff2bb4ef291e347e091f7f4a7/jdk-8u141-linux-x64.tar.gz')
-    version('8u131-b11', '75b2cb2249710d822a60f83e28860053', curl_options=curl_options,
+    version('1.8.0_131-b11', '75b2cb2249710d822a60f83e28860053', curl_options=curl_options,
             url='http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.tar.gz')
-    version('8u92-b14',  '65a1cc17ea362453a6e0eb4f13be76e4', curl_options=curl_options)
-    version('8u73-b02',  '1b0120970aa8bc182606a16bf848a686', curl_options=curl_options)
-    version('8u66-b17',  '88f31f3d642c3287134297b8c10e61bf', curl_options=curl_options)
-    # The 7u80 tarball is not readily available from Oracle.  If you have
-    # the tarball, add it to your mirror as mirror/jdk/jdk-7u80.tar.gz and
-    # away you go.
-    version('7u80-b0', '6152f8a7561acf795ca4701daa10a965')
-
-    provides('java@8', when='@8u0:8u999')
-    provides('java@7', when='@7u0:7u999')
-
-    def url_for_version(self, version):
-        url = "http://download.oracle.com/otn-pub/java/jdk/{0}/jdk-{1}-linux-x64.tar.gz"
-        version = str(version)
-        minor_version = version[:version.index('-')]
-        return url.format(version, minor_version)
+    version('1.8.0_92-b14', '65a1cc17ea362453a6e0eb4f13be76e4', curl_options=curl_options)
+    version('1.8.0_73-b02', '1b0120970aa8bc182606a16bf848a686', curl_options=curl_options)
+    version('1.8.0_66-b17', '88f31f3d642c3287134297b8c10e61bf', curl_options=curl_options)
+    version('1.7.0_80-b0', '6152f8a7561acf795ca4701daa10a965', curl_options=curl_options)
+
+    provides('java')
+    provides('java@10', when='@10.0:10.999')
+    provides('java@9',  when='@9.0:9.999')
+    provides('java@8',  when='@1.8.0:1.8.999')
+    provides('java@7',  when='@1.7.0:1.7.999')
+
+    # FIXME:
+    # 1. `extends('java')` doesn't work, you need to use `extends('jdk')`
+    # 2. Packages cannot extend multiple packages, see #987
+    # 3. Update `YamlFilesystemView.merge` to allow a Package to completely
+    #    override how it is symlinked into a view prefix. Then, spack activate
+    #    can symlink all *.jar files to `prefix.lib.ext`
+    extendable = True
+
+    @property
+    def home(self):
+        """Most of the time, ``JAVA_HOME`` is simply ``spec['java'].prefix``.
+        However, if the user is using an externally installed JDK, it may be
+        symlinked. For example, on macOS, the ``java`` executable can be found
+        in ``/usr/bin``, but ``JAVA_HOME`` is actually
+        ``/Library/Java/JavaVirtualMachines/jdk-10.0.1.jdk/Contents/Home``.
+        Users may not know the actual installation directory and add ``/usr``
+        to their ``packages.yaml`` unknowingly. Run ``java_home`` if it exists
+        to determine exactly where it is installed. Specify which version we
+        are expecting in case multiple Java versions are installed.
+        See ``man java_home`` for more details."""
+
+        prefix = self.prefix
+        java_home = prefix.libexec.java_home
+        if os.path.exists(java_home):
+            java_home = Executable(java_home)
+            version = str(self.version.up_to(2))
+            prefix = java_home('--version', version, output=str).strip()
+            prefix = Prefix(prefix)
+
+        return prefix
+
+    @property
+    def libs(self):
+        """Depending on the version number and whether the full JDK or just
+        the JRE was installed, Java libraries can be in several locations:
+
+        * ``lib/libjvm.so``
+        * ``jre/lib/libjvm.dylib``
+
+        Search recursively to find the correct library location."""
+
+        return find_libraries(['libjvm'], root=self.home, recursive=True)
+
+    @run_before('install')
+    def macos_check(self):
+        if self.spec.satisfies('platform=darwin'):
+            msg = """\
+Spack's JDK package only supports Linux. If you need to install JDK on macOS,
+manually download the .dmg from:
+
+    {0}
+
+and double-click to install. Once JDK is installed, you can tell Spack where
+to find it like so. To find the JDK installation directory, run:
+
+    $ /usr/libexec/java_home
+
+If you have multiple versions of JDK installed, you can specify a particular
+version to search for with the --version flag. To find the exact version
+number, run:
+
+    $ java -version
+
+If the version number contains a '+' symbol, replace it with '_', otherwise
+Spack will think it is a variant. Add JDK as an external package by running:
+
+    $ spack config edit packages
+
+and adding entries for each installation:
+
+    packages:
+        jdk:
+            paths:
+                jdk@10.0.1_10:    /path/to/jdk/Home
+                jdk@1.7.0_45-b18: /path/to/jdk/Home
+            buildable: False""".format(self.homepage)
+
+            tty.die(msg)
 
     def install(self, spec, prefix):
         install_tree('.', prefix)
 
     def setup_environment(self, spack_env, run_env):
-        run_env.set('JAVA_HOME', self.spec.prefix)
+        """Set JAVA_HOME."""
+
+        run_env.set('JAVA_HOME', self.home)
 
     def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
-        spack_env.set('JAVA_HOME', self.spec.prefix)
+        """Set JAVA_HOME and CLASSPATH.
+
+        CLASSPATH contains the installation prefix for the extension and any
+        other Java extensions it depends on."""
+
+        spack_env.set('JAVA_HOME', self.home)
+
+        class_paths = []
+        for d in dependent_spec.traverse(deptype=('build', 'run', 'test')):
+            if d.package.extends(self.spec):
+                class_paths.extend(find(d.prefix, '*.jar'))
+
+        classpath = os.pathsep.join(class_paths)
+        spack_env.set('CLASSPATH', classpath)
+
+        # For runtime environment set only the path for
+        # dependent_spec and prepend it to CLASSPATH
+        if dependent_spec.package.extends(self.spec):
+            class_paths = find(dependent_spec.prefix, '*.jar')
+            classpath = os.pathsep.join(class_paths)
+            run_env.prepend_path('CLASSPATH', classpath)
+
+    def setup_dependent_package(self, module, dependent_spec):
+        """Allows spec['java'].home to work."""
+
+        self.spec.home = self.home
diff --git a/var/spack/repos/builtin/packages/octave/package.py b/var/spack/repos/builtin/packages/octave/package.py
index 5587e79396..4a89992dac 100644
--- a/var/spack/repos/builtin/packages/octave/package.py
+++ b/var/spack/repos/builtin/packages/octave/package.py
@@ -186,9 +186,9 @@ class Octave(AutotoolsPackage):
 
         if '+jdk' in spec:
             config_args.extend([
-                "--with-java-homedir=%s"    % spec['java'].prefix,
-                "--with-java-includedir=%s" % spec['java'].prefix.include,
-                "--with-java-libdir=%s"     % spec['java'].prefix.lib
+                "--with-java-homedir=%s"    % spec['java'].home,
+                "--with-java-includedir=%s" % spec['java'].home.include,
+                "--with-java-libdir=%s"     % spec['java'].libs.directories[0]
             ])
         else:
             config_args.append("--disable-java")
diff --git a/var/spack/repos/builtin/packages/openmpi/package.py b/var/spack/repos/builtin/packages/openmpi/package.py
index 651d36b3b9..4550c611ad 100644
--- a/var/spack/repos/builtin/packages/openmpi/package.py
+++ b/var/spack/repos/builtin/packages/openmpi/package.py
@@ -89,7 +89,7 @@ class Openmpi(AutotoolsPackage):
 
     # Still supported
     version('3.0.2', '098fa89646f5b4438d9d8534bc960cd6')  # libmpi.so.40.00.2
-    version('3.0.1', '565f5060e080b0871a64b295c3d4426a')  # libmpi.so.40.00.1    
+    version('3.0.1', '565f5060e080b0871a64b295c3d4426a')  # libmpi.so.40.00.1
     version('3.0.0', '757d51719efec08f9f1a7f32d58b3305')  # libmpi.so.40.00.0
     version('2.1.5', '6019c8b67d4975d833801e72ba290918')  # libmpi.so.20.10.3 
     version('2.1.4', '003b356a24a5b7bd1705a23ddc69d9a0')  # libmpi.so.20.10.3
@@ -394,7 +394,7 @@ class Openmpi(AutotoolsPackage):
                 config_args.extend([
                     '--enable-java',
                     '--enable-mpi-java',
-                    '--with-jdk-dir={0}'.format(spec['java'].prefix)
+                    '--with-jdk-dir={0}'.format(spec['java'].home)
                 ])
             else:
                 config_args.extend([
diff --git a/var/spack/repos/builtin/packages/spark/package.py b/var/spack/repos/builtin/packages/spark/package.py
index 1b318e8c78..9dce9b81f0 100644
--- a/var/spack/repos/builtin/packages/spark/package.py
+++ b/var/spack/repos/builtin/packages/spark/package.py
@@ -67,10 +67,6 @@ class Spark(Package):
 
     @when('+hadoop')
     def setup_environment(self, spack_env, run_env):
-
-        env['JAVA_HOME'] = self.spec['java'].prefix
-        # spack_env.set('JAVA_HOME', self.spec['jdk'].prefix)
-
         hadoop = self.spec['hadoop'].command
         hadoop_classpath = hadoop('classpath', output=str)
 
diff --git a/var/spack/repos/builtin/packages/thrift/package.py b/var/spack/repos/builtin/packages/thrift/package.py
index 262aa9b106..8ae67b6ec2 100644
--- a/var/spack/repos/builtin/packages/thrift/package.py
+++ b/var/spack/repos/builtin/packages/thrift/package.py
@@ -73,7 +73,6 @@ class Thrift(Package):
 
     def install(self, spec, prefix):
         env['PY_PREFIX'] = prefix
-        env['JAVA_HOME'] = spec['java'].prefix
 
         # configure options
         options = ['--prefix=%s' % prefix]