double chunk not working

fftma_module/gen/include/chunk_array.h

@@ -7,19 +7,23 @@
 #include <string.h>
 #include <stdbool.h>
 
-#define MAX_CHUNK_SIZE 1500
+#define MAX_CHUNK_SIZE 6000
 
 typedef struct chunk_array {
     size_t init_pos;
+    size_t init_read;
+    size_t init_write;
     size_t chunk_size;
     size_t total_size;
     FILE* fp;
     double* data;
+    double* read_data;
+    double* write_data;
 }chunk_array_t;
 
-chunk_array_t* chunk_array_create(char* filename, size_t total_size, size_t chunk_size);
+chunk_array_t* chunk_array_create(char* filename, size_t total_size);
 
-void chunk_array_read(chunk_array_t* chunk_array);
+//void chunk_array_read(chunk_array_t* chunk_array);
 
 //void chunk_array_write(chunk_array_t* chunk_array, char* filename);
 

fftma_module/gen/lib_src/build_real.c

@@ -24,9 +24,9 @@ void build_real(int n[3], int NTOT, chunk_array_t* covar, chunk_array_t* realiza
     int i, j, k, maille1;
     double temp;
 
-    chunk_array_read(realization);
+    //chunk_array_read(realization);
-    chunk_array_read(ireal);
+    //chunk_array_read(ireal);
-    chunk_array_read(covar);
+    //chunk_array_read(covar);
 
     /*decomposition and multiplication in the spectral domain*/
     for (k = 1; k <= n[2]; k++) {

fftma_module/gen/lib_src/chunk_array.c

@@ -1,39 +1,50 @@
 #include "chunk_array.h"
 #include "stdbool.h"
 
+/*
 void chunk_array_free(chunk_array_t* chunk_array) {
   	fclose(chunk_array->fp);
 	free(chunk_array->data);
 	free(chunk_array);  
 }
 
-bool chunk_array_update_read(chunk_array_t* chunk_array, size_t pos) {
+
+bool chunk_array_update(chunk_array_t* chunk_array, size_t pos) {
 
     int init_pos = pos/chunk_array->chunk_size;
 
     fseek(chunk_array->fp, init_pos * chunk_array->chunk_size * sizeof(double), SEEK_SET);
     
+    chunk_array->init_pos = init_pos;
+}
+
+bool chunk_array_update_read(chunk_array_t* chunk_array, size_t pos) {
+
+    chunk_array_update(chunk_array, pos);
+    
     size_t newLen = fread(chunk_array->data, sizeof(double), chunk_array->chunk_size, chunk_array->fp);
     chunk_array->init_pos += newLen;
 }
 
-/*
+
-bool chunk_array_get(chunk_array_t* chunk_array, size_t pos, double *valor) {
+/*bool chunk_array_get(chunk_array_t* chunk_array, size_t pos, double *valor) {
-	if (pos>((chunk_array->init_pos + chunk_array->chunk_size)-1)) {
+	if (pos<chunk_array->init_pos || pos>((chunk_array->init_pos + chunk_array->chunk_size)-1)) {
         chunk_array_update_read(chunk_array, pos);
 	}
     *valor=chunk_array->data[pos%chunk_array->chunk_size];
     return true;
 }
 
+
 bool chunk_array_save(chunk_array_t* chunk_array, size_t pos, double valor) {
-	if (pos>((chunk_array->init_pos + chunk_array->chunk_size)-1)) {
+	if (pos<chunk_array->init_pos || pos>((chunk_array->init_pos + chunk_array->chunk_size)-1)) {
 		chunk_array_flush(chunk_array);
+        chunk_array_update(chunk_array, pos);
 	}
 	chunk_array->data[pos%chunk_array->chunk_size]=valor;
 	return true;
 }
-*/
+
 
 chunk_array_t* chunk_array_create(char* filename, size_t total_size, size_t chunk_size) {
     chunk_array_t* chunk_array = (chunk_array_t*)malloc(sizeof(chunk_array_t));
@@ -63,7 +74,7 @@ void chunk_array_read(chunk_array_t* chunk_array) {
     size_t newLen = fread(chunk_array->data, sizeof(double), chunk_array->chunk_size, chunk_array->fp);
 }
 
-/*
+
 void chunk_array_write(chunk_array_t* chunk_array, char* filename) {
     chunk_array->fp = fopen(filename, "w");
     if (chunk_array->fp == NULL) {
@@ -71,7 +82,7 @@ void chunk_array_write(chunk_array_t* chunk_array, char* filename) {
         chunk_array->fp = fopen(filename, "w");
     }
     chunk_array->init_pos = 0;
-}*/
+}
 
 void chunk_array_flush(chunk_array_t* chunk_array) {
     size_t newLen = fwrite(chunk_array->data, sizeof(double), chunk_array->chunk_size, chunk_array->fp);
@@ -90,3 +101,101 @@ bool chunk_array_save(chunk_array_t* chunk_array, size_t pos, double valor) {
     fwrite(&valor, sizeof(double), 1, chunk_array->fp);
 	return true;
 }
+*/
+
+void chunk_array_free(chunk_array_t* chunk_array) {
+  	fclose(chunk_array->fp);
+	free(chunk_array->read_data);
+    free(chunk_array->write_data);
+	free(chunk_array);  
+}
+
+void chunk_array_update_write(chunk_array_t* chunk_array, size_t pos) {
+    //printf("entre a update_write con pos %d\n", pos);
+    int init_write = pos/chunk_array->chunk_size;
+
+    fseek(chunk_array->fp, init_write * chunk_array->chunk_size * sizeof(double), SEEK_SET);
+    
+    chunk_array->init_write = init_write * chunk_array->chunk_size;
+
+    fwrite(chunk_array->write_data, sizeof(double), chunk_array->chunk_size, chunk_array->fp);
+    fread(chunk_array->write_data, sizeof(double), MAX_CHUNK_SIZE, chunk_array->fp);
+
+    //size_t newLen = fread(chunk_array->read_data, sizeof(double), chunk_array->chunk_size, chunk_array->fp);
+    //chunk_array->init_write += newLen;
+}
+
+void chunk_array_flush(chunk_array_t* chunk_array) {
+    //printf("entre a flush\n");
+    fwrite(chunk_array->write_data, sizeof(double), chunk_array->chunk_size, chunk_array->fp);
+}
+
+bool chunk_array_update_read(chunk_array_t* chunk_array, size_t pos) {
+    //printf("entre a update_read %d\n", pos);
+    int init_read = pos/chunk_array->chunk_size;
+
+    fseek(chunk_array->fp, init_read * chunk_array->chunk_size * sizeof(double), SEEK_SET);
+    
+    chunk_array->init_read = init_read * chunk_array->chunk_size;
+    
+    fread(chunk_array->read_data, sizeof(double), chunk_array->chunk_size, chunk_array->fp);
+}
+
+chunk_array_t* chunk_array_create(char* filename, size_t total_size) {
+    chunk_array_t* chunk_array = (chunk_array_t*)malloc(sizeof(chunk_array_t));
+
+    chunk_array->fp = fopen(filename, "w+");
+
+    if (chunk_array == NULL || chunk_array->fp == NULL) {
+        return NULL;
+    }
+
+    chunk_array->read_data = malloc(MAX_CHUNK_SIZE * sizeof(double));
+
+    if (MAX_CHUNK_SIZE > 0 && chunk_array->read_data == NULL) {
+        free(chunk_array);
+        return NULL;
+    }
+
+    fread(chunk_array->read_data, sizeof(double), MAX_CHUNK_SIZE, chunk_array->fp);
+    fread(chunk_array->write_data, sizeof(double), MAX_CHUNK_SIZE, chunk_array->fp);
+
+    chunk_array->write_data = malloc(MAX_CHUNK_SIZE * sizeof(double));
+
+    if (MAX_CHUNK_SIZE > 0 && chunk_array->write_data == NULL) {
+        free(chunk_array);
+        free(chunk_array->read_data);
+        return NULL;
+    }
+
+    chunk_array->init_read = 0;
+    chunk_array->init_write = 0;
+    chunk_array->chunk_size = MAX_CHUNK_SIZE;
+    chunk_array->total_size = total_size;
+    return chunk_array;
+}
+
+bool chunk_array_get(chunk_array_t* chunk_array, size_t pos, double *valor) {
+    if (pos > chunk_array->init_write && pos < chunk_array->init_write + chunk_array->chunk_size) {
+        *valor=chunk_array->write_data[pos%chunk_array->chunk_size];
+        printf("[READ] Del write valor = %f\n", *valor);
+        return true;
+    }
+	if (pos<chunk_array->init_read || pos>chunk_array->init_read + chunk_array->chunk_size) {
+        printf("[READ] actualizo porq pos = %d y init_read = %d\n", pos, chunk_array->init_read);
+        chunk_array_update_read(chunk_array, pos);
+	}
+    *valor=chunk_array->read_data[pos%chunk_array->chunk_size];
+    printf("[READ] Del read valor = %f\n", *valor);
+    return true;
+}
+
+
+bool chunk_array_save(chunk_array_t* chunk_array, size_t pos, double valor) {
+	if (pos<chunk_array->init_write || pos>chunk_array->init_write + chunk_array->chunk_size) {
+		printf("[WRITE] actualizo porq pos = %d y init_write = %d\n", pos, chunk_array->init_write);
+        chunk_array_update_write(chunk_array,pos);
+	}
+	chunk_array->write_data[pos%chunk_array->chunk_size]=valor;
+	return true;
+}

fftma_module/gen/lib_src/clean_real.c

@@ -16,7 +16,7 @@ void clean_real(struct realization_mod* realin, int n[3], struct grid_mod grid,
     /*is the output realization already allocated?*/
     /*if not, memory allocation*/
 
-    chunk_array_read(vectorresult);
+    //chunk_array_read(vectorresult);
 
     if (realout->vector == NULL || realout->n != realin->n) {
         realout->vector = (double*)malloc(realin->n * sizeof(double));

fftma_module/gen/lib_src/covariance.c

@@ -11,7 +11,7 @@ void covariance(chunk_array_t* covar, struct vario_mod variogram, struct grid_mo
     for (i = 0; i < 3; i++)
         n2[i] = n[i] / 2;
 
-    chunk_array_read(covar);
+    //chunk_array_read(covar);
 
     for (i = 0; i <= n2[0]; i++) {
         for (j = 0; j <= n2[1]; j++) {

fftma_module/gen/lib_src/fftma2.c

@@ -51,10 +51,12 @@ void FFTMA2(struct vario_mod variogram, struct grid_mod grid, int n[3], struct r
     NXYZ = grid.NX * grid.NY * grid.NZ;
     nxyz = NXYZ + 1;
 
+    printf("NTOT %d\n", ntot);
+
     /*array initialization*/
-    covar = chunk_array_create("covar.txt", ntot, 1500);
+    covar = chunk_array_create("covar.txt", ntot);
-    ireal = chunk_array_create("ireal.txt", ntot, 1500);
+    ireal = chunk_array_create("ireal.txt", ntot);
-    realization = chunk_array_create("realization.txt", ntot, 1500);
+    realization = chunk_array_create("realization.txt", ntot);
 
     workr = (double*)malloc(nmax * sizeof(double));
     testmemory(workr);
@@ -65,25 +67,40 @@ void FFTMA2(struct vario_mod variogram, struct grid_mod grid, int n[3], struct r
     /*covariance function creation*/
     covariance(covar, variogram, grid, n, cores);
 
+    chunk_array_flush(covar);
+
     /*power spectrum*/
     fourt(covar, ireal, n, NDIM, 1, 0, workr, worki, cores);
 
+    chunk_array_flush(covar);
+    chunk_array_flush(ireal);
+
     /*organization of the input Gaussian white noise*/
     solver = 0;
     prebuild_gwn(grid, n, realin, realization, solver, cores, seed);
 
+    chunk_array_flush(realization);
+
     /*forward fourier transform of the GWN*/
     fourt(realization, ireal, n, NDIM, 1, 0, workr, worki, cores);
 
+    chunk_array_flush(realization);
+    chunk_array_flush(ireal);
+
     /* build realization in spectral domain   */
     build_real(n, NTOT, covar, realization, ireal, cores);
 
+    chunk_array_flush(realization);
+
     chunk_array_free(covar);
     remove("covar.txt");
 
     /*backward fourier transform*/
     fourt(realization, ireal, n, NDIM, 0, 1, workr, worki, cores);
 
+    chunk_array_flush(realization);
+    chunk_array_flush(ireal);
+
     chunk_array_free(ireal);
     remove("ireal.txt");
 
@@ -92,4 +109,5 @@ void FFTMA2(struct vario_mod variogram, struct grid_mod grid, int n[3], struct r
 
     /*output realization*/
     clean_real(realin, n, grid, realization, realout, cores);
+
 }

fftma_module/gen/lib_src/fourt.c

@@ -101,8 +101,8 @@ void fourt(chunk_array_t* datar, chunk_array_t* datai, int nn[3], int ndim, int
         ntot *= nn[idim];
     }
 
-    chunk_array_read(datar);
+    //chunk_array_read(datar);
-    chunk_array_read(datai);
+    //chunk_array_read(datai);
 
     /*main loop for each dimension*/
     np1 = 1;

fftma_module/gen/lib_src/fourt_covar.c

@@ -101,7 +101,7 @@ void fourt_covar(chunk_array_t* datar, double* datai, int nn[3], int ndim, int i
         ntot *= nn[idim];
     }
 
-    chunk_array_read(datar);
+    //chunk_array_read(datar);
 
     /*main loop for each dimension*/
     np1 = 1;

fftma_module/gen/test.py

@@ -29,8 +29,8 @@ variance=3.5682389
 typ=3
 
 k=gen(nx, ny, nz, dx, dy, dz, seed, variograms, mean, variance, typ, 8)
-np.save(f"out_{N}.npy",k)
+#np.save(f"out_{N}.npy",k)
 
+k2 = np.load(f"out_{N}_ceci.npy")
 
-del k
+print(np.mean(k-k2))
-gc.collect()