Add memory profiler

.gitignore

@@ -17,3 +17,4 @@ fftma_module/gen/build/
 tools/connec/conec2d
 tools/connec/conec3d
 utilities/__pycache__/
+fftma_module/gen/log.txt

fftma_module/gen/lib_src/Py_kgeneration.c

@@ -5,6 +5,7 @@
 #include "toolsFFTMA.h"
 #include "toolsIO.h"
 #include "log.h"
+#include "memory.h"
 #include <Python.h>
 #include <numpy/arrayobject.h>
 #include <stdarg.h>
@@ -32,11 +33,26 @@ void Py_kgeneration(long seed, struct grid_mod grid, struct statistic_mod stat,
 
     log_info("RESULT = in progress, N = %d", N);
 
+    struct cpustat st0_0, st0_1;
+    get_stats(&st0_0, -1);
+
     generate(&seed, N, Z);
 
     /*FFTMA*/
     FFTMA2(variogram, grid, n, Z, Y);
 
+    double* total_ram = malloc(sizeof(double));
+    getTotalVirtualMem(total_ram);
+
+    double* used_ram = malloc(sizeof(double));
+    getVirtualMemUsed(used_ram);
+
+    log_info("TOTAL VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM BY CURRENT PROCESS = %d MB", 
+    *total_ram, *used_ram, getVirtualMemUsedByCurrentProcess());
+
+    get_stats(&st0_1, -1);
+    log_info("CPU: %lf%%\n", calculate_load(&st0_0, &st0_1));
+
     /* make a log normal realization */
     if (stat.type == 1 || stat.type == 2) {
         typelog = stat.type + 2;

fftma_module/gen/lib_src/generate.c

@@ -3,6 +3,7 @@
 #include <stdlib.h>
 #include <time.h>
 #include "log.h"
+#include "memory.h"
 
 /* GENERATION OF A GAUSSIAN WHITE NOISE VECTOR    */
 /*input:                                          */
@@ -16,6 +17,9 @@ void generate(long* seed, int n, struct realization_mod* realization) {
 
     log_info("RESULT = in progress, n = %d", n);
 
+    struct cpustat st0_0, st0_1;
+    get_stats(&st0_0, -1);
+
     int i;
     long idum2 = 123456789, iy = 0;
     long* iv;
@@ -43,6 +47,21 @@ void generate(long* seed, int n, struct realization_mod* realization) {
     t = clock() - t;
     double time_taken = ((double)t)/CLOCKS_PER_SEC; // calculate the elapsed time
 
-    log_info("RESULT = success, ELAPSED = %f seconds", time_taken);
+    double* total_ram = malloc(sizeof(double));
+    getTotalVirtualMem(total_ram);
+
+    double* used_ram = malloc(sizeof(double));
+    getVirtualMemUsed(used_ram);
+
+    log_info("TOTAL VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM BY CURRENT PROCESS = %d MB", 
+    *total_ram, *used_ram, getVirtualMemUsedByCurrentProcess());
+
+    get_stats(&st0_1, -1);
+    log_info("CPU: %lf%%\n", calculate_load(&st0_0, &st0_1));
+
+    free(total_ram);
+    free(used_ram);
     free(iv);
+
+    log_info("RESULT = success, ELAPSED = %f seconds", time_taken);
 }

fftma_module/gen/lib_src/memory.c

@@ -0,0 +1,92 @@
+#include <sys/types.h>
+#include <sys/sysinfo.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/times.h>
+#include <sys/vtimes.h>
+#include <unistd.h>
+#include "log.h"
+#include "memory.h"
+
+static unsigned long long lastTotalUser, lastTotalUserLow, lastTotalSys, lastTotalIdle;
+static clock_t lastCPU, lastSysCPU, lastUserCPU;
+static int numProcessors;
+
+void getTotalVirtualMem(double* total_ram) {
+    const double megabyte = 1024 * 1024;
+    struct sysinfo si;
+    sysinfo(&si);
+    *total_ram = si.totalram / megabyte;
+}
+
+void getVirtualMemUsed(double* used_ram) {
+    const double megabyte = 1024 * 1024;
+    struct sysinfo si;
+    sysinfo(&si);
+    *used_ram = si.freeram / megabyte;
+}
+
+int parseLine(char* line) {
+    // This assumes that a digit will be found and the line ends in " Kb".
+    int i = strlen(line);
+    const char* p = line;
+    while (*p <'0' || *p > '9') p++;
+    line[i-3] = '\0';
+    i = atoi(p);
+    return i;
+}
+
+int getVirtualMemUsedByCurrentProcess() { 
+    FILE* file = fopen("/proc/self/status", "r");
+    int result = -1;
+    char line[128];
+
+    while (fgets(line, 128, file) != NULL){
+        if (strncmp(line, "VmSize:", 7) == 0){
+            result = parseLine(line);
+            break;
+        }
+    }
+    fclose(file);
+    return result / 1024;
+}
+
+void skip_lines(FILE *fp, int numlines) { 
+    int cnt = 0;
+    char ch;
+    while ((cnt < numlines) && ((ch = getc(fp)) != EOF)) {
+        if (ch == '\n') cnt++;
+    }
+}
+
+void get_stats(struct cpustat *st, int cpunum) {
+    FILE *fp = fopen("/proc/stat", "r");
+    int lskip = cpunum+1;
+    skip_lines(fp, lskip);
+    char cpun[255];
+    fscanf(fp, "%s %d %d %d %d %d %d %d", cpun, &(st->t_user), &(st->t_nice), 
+        &(st->t_system), &(st->t_idle), &(st->t_iowait), &(st->t_irq),
+        &(st->t_softirq));
+    fclose(fp);
+	return;
+}
+
+double calculate_load(struct cpustat *prev, struct cpustat *cur) {
+    int idle_prev = (prev->t_idle) + (prev->t_iowait);
+    int idle_cur = (cur->t_idle) + (cur->t_iowait);
+
+    int nidle_prev = (prev->t_user) + (prev->t_nice) + (prev->t_system) + (prev->t_irq) + (prev->t_softirq);
+    int nidle_cur = (cur->t_user) + (cur->t_nice) + (cur->t_system) + (cur->t_irq) + (cur->t_softirq);
+
+    int total_prev = idle_prev + nidle_prev;
+    int total_cur = idle_cur + nidle_cur;
+
+    double totald = (double) total_cur - (double) total_prev;
+
+    double idled = (double) idle_cur - (double) idle_prev;
+
+    double cpu_perc = (1000 * (totald - idled) / totald + 1) / 10;
+
+    return cpu_perc;
+}

fftma_module/gen/lib_src/memory.h

@@ -0,0 +1,24 @@
+#include "sys/types.h"
+#include "sys/sysinfo.h"
+#include "stdlib.h"
+#include "stdio.h"
+#include "string.h"
+#include "sys/times.h"
+#include "sys/vtimes.h"
+
+void getTotalVirtualMem();
+void getVirtualMemUsed();
+int getVirtualMemUsedByCurrentProcess();
+
+struct cpustat {
+    unsigned long t_user;
+    unsigned long t_nice;
+    unsigned long t_system;
+    unsigned long t_idle;
+    unsigned long t_iowait;
+    unsigned long t_irq;
+    unsigned long t_softirq;
+};
+
+void get_stats(struct cpustat *st, int cpunum);
+double calculate_load(struct cpustat *prev, struct cpustat *cur);

fftma_module/gen/save_logs.sh

@@ -1 +1 @@
-python3 test.py 2>&1 | tee log.txt
+python3 test.py 2>&1 | tee log2.txt

fftma_module/gen/setup.py

@@ -42,7 +42,8 @@ module_FFTMA = Extension(
         "./lib_src/clean_real.c",
         "./lib_src/testmemory.c",
         "./lib_src/genlib.c",
-        "./lib_src/log.c"
+        "./lib_src/log.c",
+        "./lib_src/memory.c"
     ],
 )
 

fftma_module/gen/test.py

@@ -2,7 +2,7 @@ from FFTMA import gen
 import numpy as np
 import gc
 
-N=16
+N=128
 
 nx, ny, nz = N,N,N
 dx, dy, dz = 1.0, 1.0, 1.0