Updatepackage/minigmg (#26467)

* MiniGMG, add support for optimised flags + SIMDe implementation of AVX instrinsics

* Add .gitlab-ci.yml

* NVHPC fast

* remove CI

* Syntax fix

4 files changed, 136 insertions, 5 deletions

diff --git a/var/spack/repos/builtin/packages/minigmg/aarch64_time.patch b/var/spack/repos/builtin/packages/minigmg/aarch64_time.patch
new file mode 100644
index 0000000000..7d5424ef6c
--- /dev/null
+++ b/var/spack/repos/builtin/packages/minigmg/aarch64_time.patch
@@ -0,0 +1,21 @@
+diff --git a/timer.aarch64.c b/timer.aarch64.c
+new file mode 100644
+index 0000000..ca47633
+--- /dev/null
++++ b/timer.aarch64.c
+@@ -0,0 +1,15 @@
++//------------------------------------------------------------------------------------------------------------------------------
++// Samuel Williams
++// SWWilliams@lbl.gov
++// Lawrence Berkeley National Lab
++//------------------------------------------------------------------------------------------------------------------------------
++#include <stdint.h>
++
++
++uint64_t CycleTime(){
++  uint64_t val;
++    asm volatile("mrs %0, cntvct_el0" : "=r" (val));
++
++  return val;
++}
++
diff --git a/var/spack/repos/builtin/packages/minigmg/inline_static.patch b/var/spack/repos/builtin/packages/minigmg/inline_static.patch
new file mode 100644
index 0000000000..fd797d8daa
--- /dev/null
+++ b/var/spack/repos/builtin/packages/minigmg/inline_static.patch
@@ -0,0 +1,14 @@
+diff --git a/operators.ompif/exchange_boundary.c b/operators.ompif/exchange_boundary.c
+index 6cb27b2..702a2cb 100644
+--- a/operators.ompif/exchange_boundary.c
++++ b/operators.ompif/exchange_boundary.c
+@@ -6,7 +6,7 @@
+ #include <stdint.h>
+ #include "../timer.h"
+ //------------------------------------------------------------------------------------------------------------------------------
+-inline void DoBufferCopy(domain_type *domain, int level, int grid_id, int buffer){
++inline static void DoBufferCopy(domain_type *domain, int level, int grid_id, int buffer){
+   // copy 3D array from read_i,j,k of read[] to write_i,j,k in write[]
+   int   dim_i      = domain->bufferCopies[level][buffer].dim.i;
+   int   dim_j      = domain->bufferCopies[level][buffer].dim.j;
+
diff --git a/var/spack/repos/builtin/packages/minigmg/package.py b/var/spack/repos/builtin/packages/minigmg/package.py
index 262bc91eb1..d18cb7a97c 100644
--- a/var/spack/repos/builtin/packages/minigmg/package.py
+++ b/var/spack/repos/builtin/packages/minigmg/package.py
@@ -24,17 +24,73 @@ class Minigmg(Package):
 
     version('master', sha256='1c2d27496a881f655f5e849d6a7a132625e535739f82575991c511cc2cf899ac')
 
+    variant('vec', default='ompif', description='Which method of vectorisation to use',
+            values=('ompif', 'sse', 'avx', 'simde'), multi=False)
+
+    variant('opt', default=False, description='Enable optimization flags for improved OpenMP')
+
     depends_on('mpi')
 
+    # Set up SIMD Everywhere config
+    depends_on('simde', when='vec=simde')
+    patch('simde.patch', when='vec=simde')
+
+    # Patch to add timer for Aarch64 rather than rdtsc
+    patch('aarch64_time.patch', when='target=aarch64:')
+
+    # Replaces inline with inline static, for correct syntax
+    patch('inline_static.patch')
+
     phases = ['build', 'install']
 
     def build(self, spec, prefix):
+
         cc = Executable(spec['mpi'].mpicc)
-        cc('-O3', self.compiler.openmp_flag, 'miniGMG.c',
-            'mg.c', 'box.c', 'solver.c', 'operators.ompif.c', 'timer.x86.c',
-            '-D__MPI', '-D__COLLABORATIVE_THREADING=6',
-            '-D__TEST_MG_CONVERGENCE', '-D__PRINT_NORM', '-D__USE_BICGSTAB',
-            '-o', 'run.miniGMG', '-lm')
+
+        args = []
+
+        # Default optimisation level
+        if spec.satisfies('+opt'):
+            if self.spec.satisfies('%nvhpc'):
+                args.append('-fast')
+            else:
+                args.append('-Ofast')
+        else:
+            args.append('-O3')
+
+        # Add OpenMP flag
+        args += [self.compiler.openmp_flag]
+
+        args += ['miniGMG.c', 'mg.c', 'box.c', 'solver.c']
+
+        # Set the correct operators file - using the vec variant
+        if spec.satisfies('vec=sse'):
+            args += ['operators.sse.c']
+        elif spec.satisfies('vec=avx'):
+            args += ['operators.avx.c']
+        elif spec.satisfies('vec=simde'):
+            args += ['operators.simde.c']
+        else:
+            args += ['operators.ompif.c']
+
+        # Switch out timer file (depends on patch)
+        if spec.satisfies('target=aarch64:'):
+            args += ['timer.aarch64.c']
+        else:
+            args += ['timer.x86.c']
+
+        args += ['-D__MPI']
+
+        if spec.satisfies('+opt'):
+            args += ['-D__PREFETCH_NEXT_PLANE_FROM_DRAM']
+            args += ['-D__FUSION_RESIDUAL_RESTRICTION']
+        else:
+            args += ['-D__COLLABORATIVE_THREADING=6']
+
+        args += ['-D__TEST_MG_CONVERGENCE', '-D__PRINT_NORM', '-D__USE_BICGSTAB']
+        args += ['-o', 'run.miniGMG', '-lm']
+
+        cc(*args)
 
     def install(self, spec, prefix):
         mkdir(prefix.bin)
diff --git a/var/spack/repos/builtin/packages/minigmg/simde.patch b/var/spack/repos/builtin/packages/minigmg/simde.patch
new file mode 100644
index 0000000000..ad570ddb9e
--- /dev/null
+++ b/var/spack/repos/builtin/packages/minigmg/simde.patch
@@ -0,0 +1,40 @@
+diff --git a/operators.simde.c b/operators.simde.c
+new file mode 100755
+index 0000000..22ab6fd
+--- /dev/null
++++ b/operators.simde.c
+@@ -0,0 +1,34 @@
++//------------------------------------------------------------------------------------------------------------------------------
++// Samuel Williams
++// SWWilliams@lbl.gov
++// Lawrence Berkeley National Lab
++//------------------------------------------------------------------------------------------------------------------------------
++#include <stdio.h>
++#include <stdlib.h>
++#include <stdint.h>
++#include <string.h>
++#include <math.h>
++// #include <immintrin.h>
++#define SIMDE_ENABLE_NATIVE_ALIASES
++#define SIMDE_X86_SSE_ENABLE_NATIVE_ALIASES
++#define _MM_HINT_T0 1
++#define _MM_HINT_T1 2
++#include "simde/x86/avx2.h"
++//------------------------------------------------------------------------------------------------------------------------------
++#include "timer.h"
++#include "defines.h"
++#include "box.h"
++#include "mg.h"
++#include "operators.h"
++//------------------------------------------------------------------------------------------------------------------------------
++#include "operators.ompif/exchange_boundary.c"
++#include "operators.ompif/lambda.c"
++#include "operators.avx/gsrb.c"
++#include "operators.ompif/apply_op.c"
++#include "operators.ompif/residual.c"
++#include "operators.ompif/restriction.c"
++#include "operators.ompif/interpolation.c"
++#include "operators.ompif/misc.c"
++#include "operators.ompif/matmul.c"
++#include "operators.ompif/problem1.c"
++//------------------------------------------------------------------------------------------------------------------------------