From bcbfaf9f750ab56036793e2a277af10860838979 Mon Sep 17 00:00:00 2001 From: Oli Date: Wed, 19 Jan 2022 18:52:02 -0300 Subject: [PATCH] chunk not working --- fftma_module/gen/include/chunk_array.h | 3 +- fftma_module/gen/lib_src/chunk_array.c | 100 ++++++++++++++++--------- fftma_module/gen/test.py | 1 + 3 files changed, 68 insertions(+), 36 deletions(-) diff --git a/fftma_module/gen/include/chunk_array.h b/fftma_module/gen/include/chunk_array.h index 2cb1944..f827f64 100644 --- a/fftma_module/gen/include/chunk_array.h +++ b/fftma_module/gen/include/chunk_array.h @@ -7,9 +7,10 @@ #include #include -#define MAX_CHUNK_SIZE 6000 +#define MAX_CHUNK_SIZE 2048 typedef struct chunk_array { + char* filename; size_t init_pos; size_t init_read; size_t init_write; diff --git a/fftma_module/gen/lib_src/chunk_array.c b/fftma_module/gen/lib_src/chunk_array.c index ca4a4a1..0502e86 100644 --- a/fftma_module/gen/lib_src/chunk_array.c +++ b/fftma_module/gen/lib_src/chunk_array.c @@ -110,64 +110,66 @@ void chunk_array_free(chunk_array_t* chunk_array) { free(chunk_array); } -void chunk_array_update_write(chunk_array_t* chunk_array, size_t pos) { - //printf("entre a update_write con pos %d\n", pos); - int init_write = pos/chunk_array->chunk_size; - - fseek(chunk_array->fp, init_write * chunk_array->chunk_size * sizeof(double), SEEK_SET); - - chunk_array->init_write = init_write * chunk_array->chunk_size; - - fwrite(chunk_array->write_data, sizeof(double), chunk_array->chunk_size, chunk_array->fp); - fread(chunk_array->write_data, sizeof(double), MAX_CHUNK_SIZE, chunk_array->fp); - - //size_t newLen = fread(chunk_array->read_data, sizeof(double), chunk_array->chunk_size, chunk_array->fp); - //chunk_array->init_write += newLen; -} - void chunk_array_flush(chunk_array_t* chunk_array) { - //printf("entre a flush\n"); - fwrite(chunk_array->write_data, sizeof(double), chunk_array->chunk_size, chunk_array->fp); + printf("[FLUSH][%s] init_write %d\n", chunk_array->filename, chunk_array->init_write); + size_t mod; + if (chunk_array->total_size - chunk_array->init_write > chunk_array->chunk_size ) { + mod = chunk_array->chunk_size; + } else { + mod = chunk_array->total_size - chunk_array->init_write; + printf("[FLUSH] ACA mod = %d\n", mod); + } + fseek(chunk_array->fp, chunk_array->init_write * chunk_array->chunk_size * sizeof(double), SEEK_SET); + fwrite(chunk_array->write_data, sizeof(double), mod, chunk_array->fp); } -bool chunk_array_update_read(chunk_array_t* chunk_array, size_t pos) { - //printf("entre a update_read %d\n", pos); - int init_read = pos/chunk_array->chunk_size; +bool chunk_array_update(chunk_array_t* chunk_array, size_t pos) { + chunk_array_flush(chunk_array); - fseek(chunk_array->fp, init_read * chunk_array->chunk_size * sizeof(double), SEEK_SET); + int new_init = pos/chunk_array->chunk_size; + + printf("[UPDATE] pos= %d new_init = %d\n", pos, new_init); + + fseek(chunk_array->fp, new_init * chunk_array->chunk_size * sizeof(double), SEEK_SET); - chunk_array->init_read = init_read * chunk_array->chunk_size; + chunk_array->init_write = new_init * chunk_array->chunk_size; - fread(chunk_array->read_data, sizeof(double), chunk_array->chunk_size, chunk_array->fp); + fread(chunk_array->write_data, sizeof(double), chunk_array->chunk_size, chunk_array->fp); } chunk_array_t* chunk_array_create(char* filename, size_t total_size) { chunk_array_t* chunk_array = (chunk_array_t*)malloc(sizeof(chunk_array_t)); - chunk_array->fp = fopen(filename, "w+"); + chunk_array->fp = fopen(filename, "r+"); if (chunk_array == NULL || chunk_array->fp == NULL) { - return NULL; + chunk_array->fp = fopen(filename, "w+"); + + if (chunk_array->fp == NULL) { + return NULL; + } } - chunk_array->read_data = malloc(MAX_CHUNK_SIZE * sizeof(double)); + /*chunk_array->read_data = malloc(MAX_CHUNK_SIZE * sizeof(double)); if (MAX_CHUNK_SIZE > 0 && chunk_array->read_data == NULL) { free(chunk_array); return NULL; } - - fread(chunk_array->read_data, sizeof(double), MAX_CHUNK_SIZE, chunk_array->fp); - fread(chunk_array->write_data, sizeof(double), MAX_CHUNK_SIZE, chunk_array->fp); + //fread(chunk_array->write_data, sizeof(double), MAX_CHUNK_SIZE, chunk_array->fp); + */ chunk_array->write_data = malloc(MAX_CHUNK_SIZE * sizeof(double)); if (MAX_CHUNK_SIZE > 0 && chunk_array->write_data == NULL) { free(chunk_array); - free(chunk_array->read_data); + //free(chunk_array->read_data); return NULL; } + fread(chunk_array->write_data, sizeof(double), MAX_CHUNK_SIZE, chunk_array->fp); + + chunk_array->filename = filename; chunk_array->init_read = 0; chunk_array->init_write = 0; chunk_array->chunk_size = MAX_CHUNK_SIZE; @@ -176,7 +178,8 @@ chunk_array_t* chunk_array_create(char* filename, size_t total_size) { } bool chunk_array_get(chunk_array_t* chunk_array, size_t pos, double *valor) { - if (pos > chunk_array->init_write && pos < chunk_array->init_write + chunk_array->chunk_size) { + //printf("[READ] pos = %d y init_write = %d\n", pos, chunk_array->init_write); + /*if (pos > chunk_array->init_write && pos < chunk_array->init_write + chunk_array->chunk_size) { *valor=chunk_array->write_data[pos%chunk_array->chunk_size]; printf("[READ] Del write valor = %f\n", *valor); return true; @@ -187,15 +190,42 @@ bool chunk_array_get(chunk_array_t* chunk_array, size_t pos, double *valor) { } *valor=chunk_array->read_data[pos%chunk_array->chunk_size]; printf("[READ] Del read valor = %f\n", *valor); + return true;*/ + + if (posinit_write || pos>chunk_array->init_write + chunk_array->chunk_size-1) { + printf("[READ][%s] actualizo pos = %d init_write = %d\n",chunk_array->filename, pos, chunk_array->init_write); + //chunk_array_flush(chunk_array); + chunk_array_update(chunk_array, pos); + } + size_t mod; + if (chunk_array->total_size - chunk_array->init_write > chunk_array->chunk_size ) { + mod = chunk_array->chunk_size; + } else { + mod = chunk_array->total_size - chunk_array->init_write; + } + int real_pos = pos%mod; + *valor=chunk_array->write_data[real_pos]; + printf("[READ] %s[%d] = %f = chunk[%d] init_write = %d\n", chunk_array-> filename, pos, *valor, real_pos, chunk_array->init_write); return true; } bool chunk_array_save(chunk_array_t* chunk_array, size_t pos, double valor) { - if (posinit_write || pos>chunk_array->init_write + chunk_array->chunk_size) { - printf("[WRITE] actualizo porq pos = %d y init_write = %d\n", pos, chunk_array->init_write); - chunk_array_update_write(chunk_array,pos); + //printf("[WRITE] pos = %d y init_write = %d\n", pos, chunk_array->init_write); + if (posinit_write || pos>chunk_array->init_write + chunk_array->chunk_size-1) { + printf("[WRITE][%s] actualizo pos = %d init_write = %d\n",chunk_array->filename, pos, chunk_array->init_write); + //chunk_array_flush(chunk_array); + chunk_array_update(chunk_array,pos); } - chunk_array->write_data[pos%chunk_array->chunk_size]=valor; + size_t mod; + if (chunk_array->total_size - chunk_array->init_write > chunk_array->chunk_size ) { + mod = chunk_array->chunk_size; + } else { + mod = chunk_array->total_size - chunk_array->init_write; + printf("[WRITE] ACA mod = %d\n", mod); + } + chunk_array->write_data[pos%mod]=valor; + + printf("[WRITE] %s[%d] = %f\n", chunk_array-> filename, pos, valor); return true; } \ No newline at end of file diff --git a/fftma_module/gen/test.py b/fftma_module/gen/test.py index dcde8d2..db27a9a 100644 --- a/fftma_module/gen/test.py +++ b/fftma_module/gen/test.py @@ -33,4 +33,5 @@ k=gen(nx, ny, nz, dx, dy, dz, seed, variograms, mean, variance, typ, 8) k2 = np.load(f"out_{N}_ceci.npy") +print(k==k2) print(np.mean(k-k2)) \ No newline at end of file