Add configurable number of cores

fftma_module/gen/include/Py_py-api.h

@@ -11,5 +11,5 @@
 #include <stdlib.h>
 #include <string.h>
 
-int Py_getvalues(PyObject*, long*, struct grid_mod*, struct vario_mod*, struct statistic_mod*);
+int Py_getvalues(PyObject*, long*, struct grid_mod*, struct vario_mod*, struct statistic_mod*, int*);
-void Py_kgeneration(long, struct grid_mod, struct statistic_mod, struct vario_mod, struct realization_mod*, struct realization_mod*, int[3]);
+void Py_kgeneration(long, struct grid_mod, struct statistic_mod, struct vario_mod, struct realization_mod*, struct realization_mod*, int[3], int);

fftma_module/gen/include/condor.h

@@ -102,7 +102,7 @@ void FFTMA_gradient(struct vario_mod variogram, struct grid_mod grid, int n[3],
 /* n: number of components in the vector          */
 /*output:                                         */
 /* realization: structure defining the realization*/
-void generate(long* seed, int n, struct realization_mod* realization);
+void generate(long* seed, int n, struct realization_mod* realization, int cores);
 
 void ikrig(int option, struct statfacies_mod facies, int kk, struct welldata_mod data, struct variotable_mod variogram, struct grid_mod grid, float* krigout);
 

fftma_module/gen/lib_src/Py_getvalues.c

@@ -42,7 +42,7 @@
 #endif
 #endif
 
-int Py_getvalues(PyObject* args, long* seed, struct grid_mod* grid, struct vario_mod* variogram, struct statistic_mod* stat) {
+int Py_getvalues(PyObject* args, long* seed, struct grid_mod* grid, struct vario_mod* variogram, struct statistic_mod* stat, int* cores) {
     clock_t t = clock();
 
     log_info("RESULT = in progress");
@@ -63,7 +63,7 @@ int Py_getvalues(PyObject* args, long* seed, struct grid_mod* grid, struct vario
         return 0;
     }
 
-    if (!PyArg_ParseTuple(args, "iiidddlO!ddi", /*"iiidddslO!ddi",*/
+    if (!PyArg_ParseTuple(args, "iiidddlO!ddii", /*"iiidddslO!ddi",*/
             &(grid->NX),
             &(grid->NY),
             &(grid->NZ),
@@ -72,10 +72,12 @@ int Py_getvalues(PyObject* args, long* seed, struct grid_mod* grid, struct vario
             &(grid->DZ),
             /*gwnfilename,*/
             seed,
-            &PyList_Type, &listvario,
+            &PyList_Type, 
+            &listvario,
             stat->mean + 0,
             stat->variance + 0,
-            &(stat->type))) {
+            &(stat->type),
+            cores)) {
         free(stat->mean);
         free(stat->variance);
         log_error("RESULT = failed");

fftma_module/gen/lib_src/Py_kgeneration.c

@@ -19,7 +19,7 @@
 /* Z is the GWN with 0-mean and 1-variance */
 /* Y is the realization with mean and variance wanted */
 
-void Py_kgeneration(long seed, struct grid_mod grid, struct statistic_mod stat, struct vario_mod variogram, struct realization_mod* Z, struct realization_mod* Y, int n[3]) { 
+void Py_kgeneration(long seed, struct grid_mod grid, struct statistic_mod stat, struct vario_mod variogram, struct realization_mod* Z, struct realization_mod* Y, int n[3], int cores) { 
     clock_t t = clock();
  
     int i, N;
@@ -33,14 +33,27 @@ void Py_kgeneration(long seed, struct grid_mod grid, struct statistic_mod stat,
 
     log_info("RESULT = in progress, N = %d", N);
 
-    struct cpustat st0_0, st0_1;
+    struct cpustat initial[cores];
-    get_stats(&st0_0, -1);
+    struct cpustat final[cores];
 
-    generate(&seed, N, Z);
+    for (int i = 0; i < cores; i++) {
+        get_stats(&initial[i], i - 1);
+    }
+
+    generate(&seed, N, Z, cores);
 
     /*FFTMA*/
     FFTMA2(variogram, grid, n, Z, Y);
 
+    /* make a log normal realization */
+    if (stat.type == 1 || stat.type == 2) {
+        typelog = stat.type + 2;
+        nor2log(Y, typelog, Y);
+    }
+
+    t = clock() - t;
+    double time_taken = ((double)t)/CLOCKS_PER_SEC; // calculate the elapsed time
+
     double* total_ram = malloc(sizeof(double));
     getTotalVirtualMem(total_ram);
 
@@ -50,17 +63,13 @@ void Py_kgeneration(long seed, struct grid_mod grid, struct statistic_mod stat,
     log_info("TOTAL VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM BY CURRENT PROCESS = %d MB", 
     *total_ram, *used_ram, getVirtualMemUsedByCurrentProcess());
 
-    get_stats(&st0_1, -1);
+    for (int i = 0; i < cores; i++) {
-    log_info("CPU: %lf%%\n", calculate_load(&st0_0, &st0_1));
+        get_stats(&final[i], i - 1);
-
-    /* make a log normal realization */
-    if (stat.type == 1 || stat.type == 2) {
-        typelog = stat.type + 2;
-        nor2log(Y, typelog, Y);
     }
 
-    t = clock() - t;
+    for (int i = 0; i < cores; i++) {
-    double time_taken = ((double)t)/CLOCKS_PER_SEC; // calculate the elapsed time
+        log_info("CPU %d: %lf%%\n", i, calculate_load(&initial[i], &final[i]));
+    }
     
     log_info("RESULT = success, ELAPSED = %f seconds", time_taken);
 }

fftma_module/gen/lib_src/generate.c

@@ -12,13 +12,17 @@
 /*output:                                         */
 /* realization: structure defining the realization*/
 
-void generate(long* seed, int n, struct realization_mod* realization) {
+void generate(long* seed, int n, struct realization_mod* realization, int cores) {
     clock_t t = clock();
 
     log_info("RESULT = in progress, n = %d", n);
 
-    struct cpustat st0_0, st0_1;
+    struct cpustat initial[cores];
-    get_stats(&st0_0, -1);
+    struct cpustat final[cores];
+
+    for (int i = 0; i < cores; i++) {
+        get_stats(&initial[i], i - 1);
+    }
 
     int i;
     long idum2 = 123456789, iy = 0;
@@ -56,8 +60,13 @@ void generate(long* seed, int n, struct realization_mod* realization) {
     log_info("TOTAL VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM BY CURRENT PROCESS = %d MB", 
     *total_ram, *used_ram, getVirtualMemUsedByCurrentProcess());
 
-    get_stats(&st0_1, -1);
+    for (int i = 0; i < cores; i++) {
-    log_info("CPU: %lf%%\n", calculate_load(&st0_0, &st0_1));
+        get_stats(&final[i], i - 1);
+    }
+
+    for (int i = 0; i < cores; i++) {
+        log_info("CPU %d: %lf%%\n", i, calculate_load(&initial[i], &final[i]));
+    }
 
     free(total_ram);
     free(used_ram);

fftma_module/gen/moduleFFTMA.c

@@ -31,10 +31,12 @@ static PyObject* genFunc(PyObject* self, PyObject* args) {
     struct statistic_mod stat;
     PyObject* out_array;
     npy_intp out_dims[NPY_MAXDIMS];
+    int cores;
 
-    if (!Py_getvalues(args, &seed, &grid, &variogram, &stat))
+    if (!Py_getvalues(args, &seed, &grid, &variogram, &stat, &cores))
         return NULL;
-    Py_kgeneration(seed, grid, stat, variogram, &Z, &Y, n);
+
+    Py_kgeneration(seed, grid, stat, variogram, &Z, &Y, n, cores);
 
     out_dims[0] = grid.NZ;
     out_dims[1] = grid.NY;

fftma_module/gen/save_logs.sh

@@ -1 +1 @@
-python3 test.py 2>&1 | tee log2.txt
+python3 test.py 2>&1 | tee log.txt

fftma_module/gen/test.py

@@ -27,10 +27,7 @@ mean=15.3245987
 variance=3.5682389
 typ=3
 
-for i in range(1):
+k=gen(nx, ny, nz, dx, dy, dz, seed, variograms, mean, variance, typ, 8)
-    k=gen(nx, ny, nz, dx, dy, dz, seed, variograms, mean, variance, typ)
+np.save(f"out_{0}.npy",k)
-
+del k
-    np.save(f"out_{i}.npy",k)
+gc.collect()
-
-    del k
-    gc.collect()

tools/generation/fftma_gen.py

@@ -207,7 +207,7 @@ def genGaussK(
         0,
     )  # coord des vecteurs de base (1 0 0) y (0 1 0)
     k = gen(
-        Nz, Ny, Nx, dx, dy, dz, seed, [v1], 0, 1, 0
+        Nx, Ny, Nz, dx, dy, dz, seed, [v1], 0, 1, 0, 8 #TODO: change this harcoded value
     )  # 0, 1, 0 = mean, variance, typ	  #Generation of a correlated standard dsitribution N(0,1)
 
     return k

tools/generation/test.py

@@ -28,7 +28,7 @@ def fftmaGenerator(seed):
         0,
     )  # coord des vecteurs de base (1 0 0) y (0 1 0)
     kkc = gen(
-        1, 100, 100, dx, dy, dz, seed, [v1], 0, 1, 0
+        1, 100, 100, dx, dy, dz, seed, [v1], 0, 1, 0, 8
     )  # 0, 1, 0 = mean, variance, typ	  #Generation of a correlated standard dsitribution N(0,1)
     print(np.mean(kkc), np.var(kkc))
     k = 0