summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGlenn Johnson <glenn-johnson@uiowa.edu>2020-02-18 15:59:09 -0600
committerGitHub <noreply@github.com>2020-02-18 15:59:09 -0600
commit3ee0295fb29e08870e04f849de961266f95e9fea (patch)
tree79d9c0cc55396231cd7f49ea7da2bf02a6451a58
parent6f95967e95570ce6aa0c710abd3e2b522655a85e (diff)
downloadspack-3ee0295fb29e08870e04f849de961266f95e9fea.tar.gz
spack-3ee0295fb29e08870e04f849de961266f95e9fea.tar.bz2
spack-3ee0295fb29e08870e04f849de961266f95e9fea.tar.xz
spack-3ee0295fb29e08870e04f849de961266f95e9fea.zip
Update and simplify julia package (#14756)
* Update and simplify julia package The current Spack Julia package potentially installs a few julia packages, with the installation being controlled by variants. There are a couple of problems with this. First, Julia handles packages very differently from systems such as R and Python. Julia requires write access to the repository directories in order for user installs of packages to work. If spack installs julia packages then there will be a repository, DEPOT_PATH, in the installation directory. If spack is used on an individual basis this would work but would mean that package data is written to the spack installation directory after installation. If spack is used to provide packages for end users then user installs of julia packages will fail due to lack of write access to the repository in the installation directory. It seems best for spack to just install julia without any julia packages, and drop the configuration for those packages. Second, having spack install package as variants seems to be counter to how spack works for other extendable systems, like R and Python. Julia should be an extendable package in spack but it is not clear how to make that work. As pointed out above, installing user packages requires write access to the julia repositories, including the one in the install directory. Essentially, a user package installation will try to update metadata for *all* julia repositories. Furthermore, each of those repositories, assuming one per package with spack, would need to have the Project.toml files merged to present the package stack to julia. Again, it seems best for spack to just install julia itself and not try to install julia packages, at least at this time. A good discussion on this can be found at https://discourse.julialang.org/t/how-does-one-set-up-a-centralized-julia-installation/13922. This PR does the following: - adds versions 1.2.0 and 1.3.1 - removes variants that correspond to julia packages - changes python to build dependency as it seems to only be needed for LLVM - the new versions can use Python-3 - removes dependencies for packages - adds a conflict statement for Intel compiler, with comment - add a setup_build_environment method to find GCC libraries - make formatting consistent - adds JULIA_CPU_TARGET option to correspond with target to help with running julia image on hardware older than build host - added intel build options, for when they can be used - removed code for installing packages - removed code for julia config that was needed for packages Note that versions below 0.5.1 fail to build, with or without these changes. It is not clear why that is. * Update var/spack/repos/builtin/packages/julia/package.py Yes, need to use correct grammar even in the midst of numbers and symbols. Good catch! Co-Authored-By: Adam J. Stewart <ajstewart426@gmail.com> * More cleanup of Julia package This commit does more cleanup and sets more constraints. - Removed release-0.4 and release-0.5. I am not sure if those are actually useful but they are quite old and there are released versions from the same timeframe. - Remove the binutils variant. - Made cmake a build dependency for versions >= 1. - Added git as a dependency for @master. - Limit curl dependency to released versions. - Do not use external curl for master. When I checked, using the external version failed but the internal curl worked. - Versions <= 0.5.0 need an older version of openssl. - Set conflicts directive for cxx variant. - Added conflicts directive for needing +mkl with Intel compiler. - Removed configuration settings as these prevented julia from working properly in all cases that I looked at. * Fix flake8 error Remove 'import sys' that is no longer used. * More dependency tweaks This commit sets further version constraints on dependencies. It really looks like julia requires its internal dependencies more over time. - curl only up to 0.5.0 - openssl only up to 0.5.0 - override with system curl up to version 0.5.0 * Fix spec for curl certs Only depending on curl through 0.5.0. Co-authored-by: Adam J. Stewart <ajstewart426@gmail.com>
-rw-r--r--var/spack/repos/builtin/packages/julia/package.py278
1 files changed, 104 insertions, 174 deletions
diff --git a/var/spack/repos/builtin/packages/julia/package.py b/var/spack/repos/builtin/packages/julia/package.py
index a056d40558..9dae676d9a 100644
--- a/var/spack/repos/builtin/packages/julia/package.py
+++ b/var/spack/repos/builtin/packages/julia/package.py
@@ -6,7 +6,6 @@
from spack import *
import os
-import sys
class Julia(Package):
@@ -16,62 +15,55 @@ class Julia(Package):
url = "https://github.com/JuliaLang/julia/releases/download/v0.4.3/julia-0.4.3-full.tar.gz"
git = "https://github.com/JuliaLang/julia.git"
+ maintainers = ['glennpj']
+
version('master', branch='master')
- version('1.1.1', sha256='3c5395dd3419ebb82d57bcc49dc729df3b225b9094e74376f8c649ee35ed79c2')
+ version('1.3.1', sha256='053908ec2706eb76cfdc998c077de123ecb1c60c945b4b5057aa3be19147b723')
+ version('1.2.0', sha256='2419b268fc5c3666dd9aeb554815fe7cf9e0e7265bc9b94a43957c31a68d9184')
+ version('1.1.1', sha256='3c5395dd3419ebb82d57bcc49dc729df3b225b9094e74376f8c649ee35ed79c2')
version('1.0.0', sha256='1a2497977b1d43bb821a5b7475b4054b29938baae8170881c6b8dd4099d133f1')
version('0.6.2', sha256='1e34c13091c9ddb47cf87a51566d94a06613f3db3c483b8f63b276e416dd621b')
- version('release-0.5', branch='release-0.5')
version('0.5.2', sha256='f5ef56d79ed55eacba9fe968bb175317be3f61668ef93e747d76607678cc01dd')
version('0.5.1', sha256='533b6427a1b01bd38ea0601f58a32d15bf403f491b8415e9ce4305b8bc83bb21')
version('0.5.0', sha256='732478536b6dccecbf56e541eef0aed04de0e6d63ae631b136e033dda2e418a9')
- version('release-0.4', branch='release-0.4')
version('0.4.7', sha256='d658d5bd5fb79b19f3c01cadb9aba8622ca8a12a4b687acc7d99c21413623570')
version('0.4.6', sha256='4c23c9fc72398014bd39327c2f7efd3a301884567d4cb2a89105c984d4d633ba')
version('0.4.5', sha256='cbf361c23a77e7647040e8070371691083e92aa93c8a318afcc495ad1c3a71d9')
version('0.4.3', sha256='2b9df25a8f58df8e43038ec30bae195dfb160abdf925f3fa193b59d40e4113c5')
- # TODO: Split these out into jl-hdf5, jl-mpi packages etc.
- variant("cxx", default=False, description="Prepare for Julia Cxx package")
- variant("hdf5", default=False, description="Install Julia HDF5 package")
- variant("mpi", default=True, description="Install Julia MPI package")
- variant("plot", default=False,
- description="Install Julia plotting packages")
- variant("python", default=False,
- description="Install Julia Python package")
- variant("simd", default=False, description="Install Julia SIMD package")
- variant("mkl", default=False, description="Use Intel MKL")
+ variant('cxx', default=False, description='Prepare for Julia Cxx package')
+ variant('mkl', default=False, description='Use Intel MKL')
patch('gc.patch', when='@0.4:0.4.5')
patch('openblas.patch', when='@0.4:0.4.5')
patch('armgcc.patch', when='@1.0.0:1.1.1 %gcc@:5.9 target=aarch64:')
- variant('binutils', default=sys.platform != 'darwin',
- description="Build via binutils")
-
# Build-time dependencies:
- # depends_on("awk")
- depends_on("m4", type="build")
- # depends_on("pkgconfig")
+ # depends_on('awk')
+ depends_on('m4', type='build')
+ # depends_on('pkgconfig')
+ # Python only needed to build LLVM?
+ depends_on('python@2.7:2.8', type='build', when='@:1.1')
+ depends_on('python@2.7:', type='build', when='@1.2:')
+ depends_on('cmake @2.8:', type='build', when='@1.0:')
+ depends_on('git', type='build', when='@master')
# Combined build-time and run-time dependencies:
# (Yes, these are run-time dependencies used by Julia's package manager.)
- depends_on("binutils", when='+binutils')
- depends_on("cmake @2.8:")
- depends_on("curl")
- depends_on("git", when='@:0.4')
- depends_on("git", when='@release-0.4')
- depends_on("openssl")
- depends_on("python@2.7:2.8")
- depends_on("mkl", when='+mkl')
+ depends_on('cmake @2.8:', type=('build', 'run'), when='@:0.6')
+ depends_on('curl', when='@:0.5.0')
+ depends_on('git', type=('build', 'run'), when='@:0.4')
+ depends_on('openssl@:1.0', when='@:0.5.0')
+ depends_on('mkl', when='+mkl')
# Run-time dependencies:
- # depends_on("arpack")
- # depends_on("fftw +float")
- # depends_on("gmp")
- # depends_on("libgit")
- # depends_on("mpfr")
- # depends_on("openblas")
- # depends_on("pcre2")
+ # depends_on('arpack')
+ # depends_on('fftw +float')
+ # depends_on('gmp')
+ # depends_on('libgit')
+ # depends_on('mpfr')
+ # depends_on('openblas')
+ # depends_on('pcre2')
# ARPACK: Requires BLAS and LAPACK; needs to use the same version
# as Julia.
@@ -97,162 +89,100 @@ class Julia(Package):
# USE_SYSTEM_UTF8PROC=0
# USE_SYSTEM_LIBGIT2=0
- # Run-time dependencies for Julia packages:
- depends_on("hdf5", when="+hdf5", type="run")
- depends_on("mpi", when="+mpi", type="run")
- depends_on("py-matplotlib", when="+plot", type="run")
+ conflicts('+cxx', when='@:0.6', msg='Variant cxx requires Julia >= 1.0.0')
+
+ conflicts('@:0.7.0', when='target=aarch64:')
+
+ # GCC conflicts
+ conflicts('@:0.5.1', when='%gcc@8:', msg='Julia <= 0.5.1 needs GCC <= 7')
+
+ # Building recent versions of Julia with Intel is untested and unsupported
+ # by the Julia project, https://github.com/JuliaLang/julia/issues/23407.
+ conflicts('@0.6:', when='%intel',
+ msg='Only Julia <= 0.5.x can be built with the Intel compiler.')
+ conflicts('%intel', when='~mkl',
+ msg='Building with the Intel compiler requires the mkl variant '
+ '(+mkl)')
- conflicts("@:0.7.0", when="target=aarch64:")
+ def setup_build_environment(self, env):
+ # The julia build can have trouble with finding GCC libraries with the
+ # spack compiler.
+ if self.compiler.name == 'gcc':
+ gcc_base = os.path.split(os.path.split(self.compiler.cc)[0])[0]
+ env.prepend_path('LD_LIBRARY_PATH', join_path(gcc_base, 'lib64'))
def install(self, spec, prefix):
# Julia needs git tags
- if os.path.isfile(".git/shallow"):
- git = which("git")
- git("fetch", "--unshallow")
+ if os.path.isfile('.git/shallow'):
+ git = which('git')
+ git('fetch', '--unshallow')
# Explicitly setting CC, CXX, or FC breaks building libuv, one
# of Julia's dependencies. This might be a Darwin-specific
# problem. Given how Spack sets up compilers, Julia should
# still use Spack's compilers, even if we don't specify them
- # explicitly.
+ # explicitly. Potential options are
+ # 'CC=cc',
+ # 'CXX=c++',
+ # 'FC=fc',
+ # 'USE_SYSTEM_ARPACK=1',
+ # 'override USE_SYSTEM_CURL=1',
+ # 'USE_SYSTEM_FFTW=1',
+ # 'USE_SYSTEM_GMP=1',
+ # 'USE_SYSTEM_MPFR=1',
+ # 'USE_SYSTEM_PCRE=1',
options = [
- # "CC=cc",
- # "CXX=c++",
- # "FC=fc",
- # "USE_SYSTEM_ARPACK=1",
- "override USE_SYSTEM_CURL=1",
- # "USE_SYSTEM_FFTW=1",
- # "USE_SYSTEM_GMP=1",
- # "USE_SYSTEM_MPFR=1",
- # "USE_SYSTEM_PCRE=1",
- "prefix=%s" % prefix]
- if "+cxx" in spec:
- if "@master" not in spec:
- raise InstallError(
- "Variant +cxx requires the @master version of Julia")
+ 'prefix={0}'.format(prefix)
+ ]
+ if '@:0.5.0' in spec:
options += [
- "BUILD_LLVM_CLANG=1",
- "LLVM_ASSERTIONS=1",
- "USE_LLVM_SHLIB=1"]
+ 'override USE_SYSTEM_CURL=1'
+ ]
+
+ if '+cxx' in spec:
+ options += [
+ 'BUILD_LLVM_CLANG=1',
+ 'LLVM_ASSERTIONS=1',
+ 'USE_LLVM_SHLIB=1'
+ ]
if spec.target.family == 'aarch64':
options += [
'JULIA_CPU_TARGET=generic',
- 'MARCH=armv8-a+crc']
+ 'MARCH=armv8-a+crc'
+ ]
+
+ if spec.target.family == 'x86_64' or spec.target.family == 'x86':
+ if spec.target == 'x86_64':
+ options += [
+ 'JULIA_CPU_TARGET=generic'
+ ]
+ else:
+ options += [
+ 'JULIA_CPU_TARGET={0}'.format(spec.target)
+ ]
+
+ if '%intel' in spec:
+ options += [
+ 'USEICC=1',
+ 'USEIFC=1',
+ 'USE_INTEL_LIBM=1'
+ ]
+
if '+mkl' in spec:
options += [
- 'USE_INTEL_MKL=1']
+ 'USE_INTEL_MKL=1',
+ ]
with open('Make.user', 'w') as f:
f.write('\n'.join(options) + '\n')
+
make()
- make("install")
+ make('install')
# Julia's package manager needs a certificate
- cacert_dir = join_path(prefix, "etc", "curl")
- mkdirp(cacert_dir)
- cacert_file = join_path(cacert_dir, "cacert.pem")
- curl = which("curl")
- curl("--create-dirs",
- "--output", cacert_file,
- "https://curl.haxx.se/ca/cacert.pem")
-
- # Put Julia's compiler cache into a private directory
- cachedir = join_path(prefix, "var", "julia", "cache")
- mkdirp(cachedir)
-
- # Store Julia packages in a private directory
- pkgdir = join_path(prefix, "var", "julia", "pkg")
- mkdirp(pkgdir)
-
- # Configure Julia
- if spec.satisfies('@master') or spec.satisfies("@0.7:"):
- julia_config = 'startup.jl'
- cache_path = 'DEPOT_PATH'
- unshift = 'pushfirst!'
- else:
- julia_config = 'juliarc.jl'
- cache_path = 'LOAD_CACHE_PATH'
- unshift = 'unshift!'
- with open(join_path(prefix, "etc", "julia", julia_config),
- "a") as juliarc:
- if "@master" in spec or "@release-0.5" in spec or "@0.5" in spec:
- # This is required for versions @0.5:
- juliarc.write(
- '# Point package manager to working certificates\n')
- if spec.satisfies('@master') or spec.satisfies('@0,7'):
- juliarc.write('import LibGit2;')
- juliarc.write('LibGit2.set_ssl_cert_locations("%s")\n' %
- cacert_file)
- juliarc.write('\n')
- juliarc.write('# Put compiler cache into a private directory\n')
- juliarc.write('empty!(Base.%s)\n' % cache_path)
- juliarc.write('%s(Base.%s, "%s")\n' %
- (unshift, cache_path, cachedir))
- juliarc.write('\n')
- juliarc.write('# Put Julia packages into a private directory\n')
- juliarc.write('ENV["JULIA_PKGDIR"] = "%s"\n' % pkgdir)
- juliarc.write('\n')
-
- # Install some commonly used packages
- julia = spec['julia'].command
- if spec.satisfies("@:0.7"):
- julia("-e", 'Pkg.init(); Pkg.update()')
- pkgstart = ''
- else:
- pkgstart = 'import Pkg;'
-
- # Install HDF5
- if "+hdf5" in spec:
- with open(join_path(prefix, "etc", "julia", julia_config),
- "a") as juliarc:
- juliarc.write('# HDF5\n')
- juliarc.write('push!(Libdl.DL_LOAD_PATH, "%s")\n' %
- spec["hdf5"].prefix.lib)
- juliarc.write('\n')
- julia("-e", pkgstart + 'Pkg.add("HDF5"); using HDF5')
- julia("-e", pkgstart + 'Pkg.add("JLD"); using JLD')
-
- # Install MPI
- if "+mpi" in spec:
- with open(join_path(prefix, "etc", "julia", julia_config),
- "a") as juliarc:
- juliarc.write('# MPI\n')
- juliarc.write('ENV["JULIA_MPI_C_COMPILER"] = "%s"\n' %
- join_path(spec["mpi"].prefix.bin, "mpicc"))
- juliarc.write('ENV["JULIA_MPI_Fortran_COMPILER"] = "%s"\n' %
- join_path(spec["mpi"].prefix.bin, "mpifort"))
- juliarc.write('\n')
- julia("-e", pkgstart + 'Pkg.add("MPI"); using MPI')
-
- # Install Python
- if "+python" in spec or "+plot" in spec:
- with open(join_path(prefix, "etc", "julia", julia_config),
- "a") as juliarc:
- juliarc.write('# Python\n')
- juliarc.write('ENV["PYTHON"] = "%s"\n' % spec["python"].home)
- juliarc.write('\n')
- # Python's OpenSSL package installer complains:
- # Error: PREFIX too long: 166 characters, but only 128 allowed
- # Error: post-link failed for: openssl-1.0.2g-0
- julia("-e", pkgstart + 'Pkg.add("PyCall"); using PyCall')
-
- if "+plot" in spec:
- julia("-e", pkgstart + 'Pkg.add("PyPlot"); using PyPlot')
- julia("-e", pkgstart + 'Pkg.add("Colors"); using Colors')
- # These require maybe gtk and imagemagick
- julia("-e", pkgstart + 'Pkg.add("Plots"); using Plots')
- julia("-e", pkgstart + 'Pkg.add("PlotRecipes"); using PlotRecipes')
- julia(
- "-e",
- pkgstart + 'Pkg.add("UnicodePlots"); using UnicodePlots'
- )
- julia("-e", """\
-using Plots
-using UnicodePlots
-unicodeplots()
-plot(x->sin(x)*cos(x), linspace(0, 2pi))
-""")
-
- # Install SIMD
- if "+simd" in spec:
- julia("-e", pkgstart + 'Pkg.add("SIMD"); using SIMD')
-
- julia("-e", pkgstart + 'Pkg.status()')
+ if '@:0.5.0' in spec:
+ cacert_dir = join_path(prefix, 'etc', 'curl')
+ mkdirp(cacert_dir)
+ cacert_file = join_path(cacert_dir, 'cacert.pem')
+ curl = which('curl')
+ curl('--create-dirs',
+ '--output', cacert_file,
+ 'https://curl.haxx.se/ca/cacert.pem')