generation performance tests. wip

Makefile

@@ -12,4 +12,7 @@ binaries:
 	./script_fortran.sh
 
 test: binaries
-	cd tests/integration && python3 -m unittest test.py
+	cd tests/integration && python3 -m unittest test.py
+
+perf: binaries
+	cd tests/performance && python3 generation.py

requirements.txt

@@ -6,4 +6,6 @@ jupyter
 pandas
 sympy
 nose
-mpi4py
+mpi4py
+Benchmarker==4.0.1
+memory_profiler

tests/integration/test.py

@@ -35,7 +35,7 @@ class TestIntegration(unittest.TestCase):
     def setUpClass(cls):
         os.chdir('../..')
         config_file = os.path.abspath("./tests/integration/conf_test.ini")
-        os.system("CONFIG_FILE_PATH="+ config_file + " mpirun python3 mpirunner.py")
+        os.system("CONFIG_FILE_PATH="+ config_file + " mpirun -np 1 python3 mpirunner.py")
 
         binary_results = {}
         for binary in BINARIES:

tests/performance/conf_gen_128.ini

@@ -0,0 +1,44 @@
+[General]
+simDir=tests/performance/tmp_gen_output
+startJob=0
+[Iterables]
+p=[10,39,15]
+seeds=[5462,2]
+lc=[4]
+connectivity=[1,2,3]
+variances=[1]
+
+[Generation]
+Nx = 128
+Ny = 128
+Nz = 128
+variogram_type=1
+binary = yes
+kh = 100
+kl = 0.01
+compute_lc = yes
+lcBin=yes
+genera=yes
+
+
+[Connectivity]
+keep_aspect= no
+block_size = 4
+indicators_MinBlockSize =4 
+Max_sample_size = 12
+compGconec= 1
+conec=no
+
+
+[Solver]
+num_of_cores = 1
+ref = 2
+solve = no
+rtol = 1e-4
+
+[K-Postprocess]
+MinBlockSize =1
+Max_sample_size = 4
+kperm=no
+postprocess=no
+SaveVfield=no

tests/performance/conf_gen_256.ini

@@ -0,0 +1,44 @@
+[General]
+simDir=tests/performance/tmp_gen_output
+startJob=0
+[Iterables]
+p=[10,39,15]
+seeds=[5462,2]
+lc=[4]
+connectivity=[1,2,3]
+variances=[1]
+
+[Generation]
+Nx = 128
+Ny = 128
+Nz = 128
+variogram_type=1
+binary = yes
+kh = 100
+kl = 0.01
+compute_lc = yes
+lcBin=yes
+genera=yes
+
+
+[Connectivity]
+keep_aspect= no
+block_size = 4
+indicators_MinBlockSize =4 
+Max_sample_size = 12
+compGconec= 1
+conec=no
+
+
+[Solver]
+num_of_cores = 1
+ref = 2
+solve = no
+rtol = 1e-4
+
+[K-Postprocess]
+MinBlockSize =1
+Max_sample_size = 4
+kperm=no
+postprocess=no
+SaveVfield=no

tests/performance/conf_gen_64.ini

@@ -0,0 +1,44 @@
+[General]
+simDir=tests/performance/tmp_gen_output
+startJob=0
+[Iterables]
+p=[10,39,15]
+seeds=[5462,2]
+lc=[4]
+connectivity=[1,2,3]
+variances=[1]
+
+[Generation]
+Nx = 64
+Ny = 64
+Nz = 64
+variogram_type=1
+binary = yes
+kh = 100
+kl = 0.01
+compute_lc = yes
+lcBin=yes
+genera=yes
+
+
+[Connectivity]
+keep_aspect= no
+block_size = 4
+indicators_MinBlockSize =4 
+Max_sample_size = 12
+compGconec= 1
+conec=no
+
+
+[Solver]
+num_of_cores = 1
+ref = 2
+solve = no
+rtol = 1e-4
+
+[K-Postprocess]
+MinBlockSize =1
+Max_sample_size = 4
+kperm=no
+postprocess=no
+SaveVfield=no

tests/performance/generation.py

@@ -0,0 +1,33 @@
+import os
+from benchmarker import Benchmarker
+
+os.chdir('../..')
+
+config_file_64 = os.path.abspath("./tests/performance/conf_gen_64.ini")
+
+config_file_128 = os.path.abspath("./tests/performance/conf_gen_128.ini")
+
+config_file_256 = os.path.abspath("./tests/performance/conf_gen_256.ini")
+
+CONFIG_FILES = [config_file_64, config_file_128, config_file_256]
+
+with Benchmarker() as bench:
+
+    for i,config_file in enumerate(CONFIG_FILES):
+        size = 2**(6+i)
+
+        @bench(f"generation 1 core {size} tamaño")
+        def _(bm):
+            with bm:
+                os.system("CONFIG_FILE_PATH="+ config_file + " mpirun -oversubscribe -np 1 python3 -m memory_profiler mpirunner.py")
+
+            ## teardown
+            os.system("rm -rf ./tests/performance/tmp_gen_output")
+
+        @bench(f"generation 8 core {size} tamaño")
+        def _(bm):
+            with bm:
+                os.system("CONFIG_FILE_PATH="+ config_file + " mpirun -oversubscribe -np 8 python3 -m memory_profiler mpirunner.py")
+
+            ## teardown
+            os.system("rm -rf ./tests/performance/tmp_gen_output")

tools/generation/fftma_gen.py

@@ -9,9 +9,9 @@ from scipy.interpolate import interp1d
 import sys
 import time
 import os
+from memory_profiler import profile
 
-
+# @profile
-
 def fftmaGenerator(datadir,job,conffile):
 
 	t0=time.time()
@@ -79,7 +79,7 @@ def obtainLctobinBack(p,con,vario):
 	return 1.0/f(p)
 
  
-
+#@profile
 def generate_K(Nx,Ny,Nz,con,lc,p,kh,kl,seed,logn,LogVariance,vario,datadir,compute_lc,uselc_bin):