Add improvements

fftma_module/gen/include/chunk_array.h

@@ -0,0 +1,36 @@
+#ifndef _CHUNKARRAY_H
+#define _CHUNKARRAY_H
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+
+#define MAX_CHUNK_SIZE 32
+
+typedef struct chunk_array {
+    size_t init_pos;
+    size_t chunk_size;
+    size_t total_size;
+    FILE* fp;
+    double* data;
+}chunk_array_t;
+
+chunk_array_t* chunk_array_create(char* filename, size_t total_size, size_t chunk_size);
+
+void chunk_array_read(chunk_array_t* chunk_array);
+
+//void chunk_array_write(chunk_array_t* chunk_array, char* filename);
+
+void chunk_array_free(chunk_array_t* chunk_array);
+
+bool chunk_array_get(chunk_array_t* chunk_array, size_t pos, double *value_ptr);
+
+bool chunk_array_save(chunk_array_t* chunk_array, size_t pos, double valor);
+
+void chunk_array_flush(chunk_array_t* chunk_array);
+
+size_t chunk_array_size(chunk_array_t* chunk_array);
+
+#endif

fftma_module/gen/include/geostat.h

@@ -2,6 +2,7 @@
 #include <stddef.h>
 #include <stdio.h>
 #include <string.h>
+#include "chunk_array.h"
 
 #ifndef _GEOSTAT_H
 #define _GEOSTAT_H
@@ -283,6 +284,7 @@ struct realization_mod {
     int n;
     int code;
     double* vector;
+    chunk_array_t* vector_2;
 };
 
 /*=====================================================*/

fftma_module/gen/lib_src/chunk_array.c

@@ -0,0 +1,72 @@
+#include "chunk_array.h"
+#include "stdbool.h"
+
+void chunk_array_free(chunk_array_t* chunk_array) {
+  	fclose(chunk_array->fp);
+	free(chunk_array->data);
+	free(chunk_array);  
+}
+
+bool chunk_array_update_read(chunk_array_t* chunk_array) {
+    size_t newLen = fread(chunk_array->data, sizeof(double), chunk_array->chunk_size, chunk_array->fp);
+    chunk_array->init_pos += newLen;
+}
+
+bool chunk_array_get(chunk_array_t* chunk_array, size_t pos, double *valor) {
+	if (pos>((chunk_array->init_pos + chunk_array->chunk_size)-1)) {
+        chunk_array_update_read(chunk_array);
+	}
+    *valor=chunk_array->data[pos%chunk_array->chunk_size];
+    return true;
+}
+
+bool chunk_array_save(chunk_array_t* chunk_array, size_t pos, double valor) {
+	if (pos>((chunk_array->init_pos + chunk_array->chunk_size)-1)) {
+		chunk_array_flush(chunk_array);
+	}
+	chunk_array->data[pos%chunk_array->chunk_size]=valor;
+	return true;
+}
+
+chunk_array_t* chunk_array_create(char* filename, size_t total_size, size_t chunk_size) {
+    chunk_array_t* chunk_array = (chunk_array_t*)malloc(sizeof(chunk_array_t));
+
+    chunk_array->fp = fopen(filename, "w+");
+
+    if (chunk_array == NULL || chunk_array->fp == NULL) {
+        return NULL;
+    }
+
+    chunk_array->data = malloc(chunk_size * sizeof(double));
+
+    if (chunk_size > 0 && chunk_array->data == NULL) {
+        free(chunk_array);
+        return NULL;
+    }
+
+    chunk_array->init_pos = 0;
+    chunk_array->chunk_size = chunk_size;
+    chunk_array->total_size = total_size;
+    return chunk_array;
+}
+
+void chunk_array_read(chunk_array_t* chunk_array) {
+    rewind(chunk_array->fp);
+    chunk_array->init_pos = 0;
+    size_t newLen = fread(chunk_array->data, sizeof(double), chunk_array->chunk_size, chunk_array->fp);
+}
+
+void chunk_array_write(chunk_array_t* chunk_array, char* filename) {
+    chunk_array->fp = fopen(filename, "w");
+    if (chunk_array->fp == NULL) {
+		printf("ke");
+        fclose(chunk_array->fp);
+        chunk_array->fp = fopen(filename, "w");
+    }
+    chunk_array->init_pos = 0;
+}
+
+void chunk_array_flush(chunk_array_t* chunk_array) {
+    size_t newLen = fwrite(chunk_array->data, sizeof(double), chunk_array->chunk_size, chunk_array->fp);
+    chunk_array->init_pos += newLen;
+}

fftma_module/gen/lib_src/generate.c

@@ -41,15 +41,19 @@ void generate(long* seed, int n, struct realization_mod* realization, int cores)
     /*realization definition*/
     (*realization).n = n;
     (*realization).code = 0;
-    (*realization).vector = (double*)malloc(n * sizeof(double));
+    (*realization).vector_2 = chunk_array_create("realization1.txt", n, 32);
-    if ((*realization).vector == NULL) {
+    if ((*realization).vector_2 == NULL) {
         log_error("RESULT = failed - No memory available in generate");
-        exit;
+        exit(1);
     }
 
     /*Gaussian white noise generation*/
-    for (i = 0; i < n; i++)
+    for (i = 0; i < n; i++) {
-        (*realization).vector[i] = gasdev(seed, &idum2, &iy, iv, cores);
+        double value = gasdev(seed, &idum2, &iy, iv, cores);
+        chunk_array_save((*realization).vector_2, i, value);
+    }
+
+    chunk_array_flush((*realization).vector_2);
 
     t = clock() - t;
     double time_taken = ((double)t)/CLOCKS_PER_SEC; // calculate the elapsed time

fftma_module/gen/lib_src/geostat.h

@@ -2,6 +2,7 @@
 #include <stddef.h>
 #include <stdio.h>
 #include <string.h>
+#include "chunk_array.h"
 
 #ifndef _GEOSTAT_H
 #define _GEOSTAT_H
@@ -284,6 +285,7 @@ struct realization_mod {
     int n;
     int code;
     double* vector;
+    chunk_array_t* vector_2;
 };
 
 /*=====================================================*/

fftma_module/gen/lib_src/prebuild_gwn.c

@@ -42,9 +42,10 @@ void prebuild_gwn(struct grid_mod grid, int n[3], struct realization_mod* realin
 
     ntot = n[0] * n[1] * n[2];
     realization[0] = 0.;
+    chunk_array_read((*realin).vector_2);
     if (solver == 1) {
         for (i = 0; i < ntot; i++) {
-            realization[i + 1] = (*realin).vector[i];
+            chunk_array_get((*realin).vector_2, i, &realization[i + 1]);
         }
     } else {
         for (k = 1; k <= n[2]; k++) {
@@ -53,7 +54,7 @@ void prebuild_gwn(struct grid_mod grid, int n[3], struct realization_mod* realin
                     maille1 = i + (j - 1 + (k - 1) * n[1]) * n[0];
                     if (i <= grid.NX && j <= grid.NY && k <= grid.NZ) {
                         maille0 = i - 1 + (j - 1 + (k - 1) * grid.NY) * grid.NX;
-                        realization[maille1] = (*realin).vector[maille0];
+                        chunk_array_get((*realin).vector_2, maille0, &realization[maille1]);
                     } else {
                         realization[maille1] = 0.;
                     }

fftma_module/gen/moduleFFTMA.c

@@ -8,6 +8,7 @@
 #include "toolsFFTPSIM.h"
 #include "toolsIO.h"
 #include "lib_src/log.h"
+#include "chunk_array.h"
 #include <Python.h>
 #include <math.h>
 #include <numpy/arrayobject.h>
@@ -55,7 +56,9 @@ static PyObject* genFunc(PyObject* self, PyObject* args) {
     free(variogram.alpha);
     free(variogram.scf);   
     free(variogram.ap); 
-    free(Z.vector);
+
+    chunk_array_free(Z.vector_2);
+    remove("realization1.txt");
 
     log_info("RESULT = success");
     return out_array;

fftma_module/gen/parse_array.py

@@ -0,0 +1,25 @@
+import collections
+
+def parse_array():
+  with open("log_16-aa") as log_file:
+
+    # rearrange data
+    lines_parsed = []
+    lines = log_file.readlines()
+    for line in lines:
+      if "iv[0]" in line:
+        line_splitted = line.split()[2::3]
+        lines_parsed.append(line_splitted)
+
+    # create histogram
+    histogram = {}
+    for i in range(len(lines_parsed)):
+      for j in range(len(lines_parsed[i])):
+        histogram[j] = histogram.get(j, [])
+        histogram[j].append(lines_parsed[i][j])
+
+    # iterate histogram
+    for position, values in histogram.items():
+      print(collections.Counter(values))
+
+parse_array()

fftma_module/gen/save_logs.sh

@@ -1,2 +1,2 @@
 python3 setup.py install --user
-ENV=analysis python3 test.py $1 2>&1 | split -l 5000000 - log_$1-
+ENV=analysis python3 test.py $1 2>&1 | split -l 5000000 - log_$1_generate_improve-

fftma_module/gen/setup.py

@@ -4,7 +4,7 @@ import numpy
 module_FFTMA = Extension(
     "FFTMA",
     include_dirs=["./include", numpy.get_include()],
-    extra_compile_args=["-g", "-std=c99", "-Wall", "-Wconversion", "-Wtype-limits", "-pedantic"],
+    extra_compile_args=["-g", "-std=gnu99", "-Wall", "-Wconversion", "-Wtype-limits", "-pedantic"],
     sources=[
         "moduleFFTMA.c",
         "./lib_src/Py_getvalues.c",
@@ -43,7 +43,8 @@ module_FFTMA = Extension(
         "./lib_src/testmemory.c",
         "./lib_src/genlib.c",
         "./lib_src/log.c",
-        "./lib_src/memory.c"
+        "./lib_src/memory.c",
+        "./lib_src/chunk_array.c"
     ],
 )