Merge pull request #3 from medios-porosos-fiuba/milestone_4

Milestone 4 - Part 1

.gitignore

@@ -10,8 +10,10 @@ tools/__pycache__/
 tools/generation/__pycache__/
 output/
 tests/integration/tmp_output/
+tests/performance/tmp_gen_output/
 output/
 tests/integration/__pycache__/
 fftma_module/gen/build/
 tools/connec/conec2d
 tools/connec/conec3d
+utilities/__pycache__/

Makefile

@@ -13,3 +13,7 @@ binaries:
 
 test: binaries
 	cd tests/integration && python3 -m unittest test.py
+
+perf: binaries
+	cd tests/performance && python3 generation.py
+	cd tests/performance && python3 connectivity.py

README.md

@@ -24,6 +24,12 @@ make run
 make test
 ```
 
+## Correr las pruebas de performance
+
+```
+make perf
+```
+
 ## Instalar el binario FFTMA
 
 ```

mpirunner.py

@@ -5,8 +5,11 @@ from tools.generation.config import DotheLoop, get_config
 import os
 import sys
 from tools.Prealization import realization
+from utilities.conditional_decorator import *
+from memory_profiler import profile
 
 CONFIG_FILE_PATH = 'config.ini' if 'CONFIG_FILE_PATH' not in os.environ else os.environ['CONFIG_FILE_PATH']
+IS_TEST = False if 'TEST' not in os.environ else True
 
 def main():
 	comm = MPI.COMM_WORLD
@@ -22,8 +25,7 @@ def main():
 		worker()
 	return
 
-
+@conditional_decorator(profile, IS_TEST)
-
 def sequential():
 	comm = MPI.COMM_WORLD
 	conffile = CONFIG_FILE_PATH
@@ -49,6 +51,7 @@ def manager():
 
 	return
 
+@conditional_decorator(profile, IS_TEST)
 def worker():
 	comm = MPI.COMM_WORLD
 	rank = comm.Get_rank()

requirements.txt

@@ -7,3 +7,5 @@ pandas
 sympy
 nose
 mpi4py
+Benchmarker==4.0.1
+memory_profiler

tests/integration/test.py

@@ -35,7 +35,7 @@ class TestIntegration(unittest.TestCase):
     def setUpClass(cls):
         os.chdir('../..')
         config_file = os.path.abspath("./tests/integration/conf_test.ini")
-        os.system("CONFIG_FILE_PATH="+ config_file + " mpirun python3 mpirunner.py")
+        os.system(f"CONFIG_FILE_PATH={config_file} mpirun -np 1 python3 mpirunner.py")
 
         binary_results = {}
         for binary in BINARIES:

tests/performance/conf_conn_64.ini

@@ -0,0 +1,44 @@
+[General]
+simDir=tests/performance/tmp_gen_output
+startJob=0
+[Iterables]
+p=[10,39,10]
+seeds=[5462,2]
+lc=[4]
+connectivity=[1]
+variances=[1]
+
+[Generation]
+Nx = 64
+Ny = 64
+Nz = 64
+variogram_type=1
+binary = yes
+kh = 100
+kl = 0.01
+compute_lc = no
+lcBin=no
+genera=no
+
+
+[Connectivity]
+keep_aspect= yes
+block_size = 4
+indicators_MinBlockSize =4 
+Max_sample_size = 12
+compGconec= 1
+conec=yes
+
+
+[Solver]
+num_of_cores = 1
+ref = 2
+solve = no
+rtol = 1e-4
+
+[K-Postprocess]
+MinBlockSize =1
+Max_sample_size = 4
+kperm=no
+postprocess=no
+SaveVfield=no

tests/performance/conf_gen_128.ini

@@ -0,0 +1,44 @@
+[General]
+simDir=tests/performance/tmp_gen_output
+startJob=0
+[Iterables]
+p=[10,39,10]
+seeds=[5462,2]
+lc=[4]
+connectivity=[1]
+variances=[1]
+
+[Generation]
+Nx = 128
+Ny = 128
+Nz = 128
+variogram_type=1
+binary = yes
+kh = 100
+kl = 0.01
+compute_lc = yes
+lcBin=yes
+genera=yes
+
+
+[Connectivity]
+keep_aspect= no
+block_size = 4
+indicators_MinBlockSize =4 
+Max_sample_size = 12
+compGconec= 1
+conec=no
+
+
+[Solver]
+num_of_cores = 1
+ref = 2
+solve = no
+rtol = 1e-4
+
+[K-Postprocess]
+MinBlockSize =1
+Max_sample_size = 4
+kperm=no
+postprocess=no
+SaveVfield=no

tests/performance/conf_gen_256.ini

@@ -0,0 +1,44 @@
+[General]
+simDir=tests/performance/tmp_gen_output
+startJob=0
+[Iterables]
+p=[10,39,10]
+seeds=[5462,2]
+lc=[4]
+connectivity=[1]
+variances=[1]
+
+[Generation]
+Nx = 256
+Ny = 256
+Nz = 256
+variogram_type=1
+binary = yes
+kh = 100
+kl = 0.01
+compute_lc = yes
+lcBin=yes
+genera=yes
+
+
+[Connectivity]
+keep_aspect= no
+block_size = 4
+indicators_MinBlockSize =4 
+Max_sample_size = 12
+compGconec= 1
+conec=no
+
+
+[Solver]
+num_of_cores = 1
+ref = 2
+solve = no
+rtol = 1e-4
+
+[K-Postprocess]
+MinBlockSize =1
+Max_sample_size = 4
+kperm=no
+postprocess=no
+SaveVfield=no

tests/performance/conf_gen_512.ini

@@ -0,0 +1,44 @@
+[General]
+simDir=tests/performance/tmp_gen_output
+startJob=0
+[Iterables]
+p=[10,39,10]
+seeds=[5462,2]
+lc=[4]
+connectivity=[1]
+variances=[1]
+
+[Generation]
+Nx = 512
+Ny = 512
+Nz = 512
+variogram_type=1
+binary = yes
+kh = 100
+kl = 0.01
+compute_lc = yes
+lcBin=yes
+genera=yes
+
+
+[Connectivity]
+keep_aspect= no
+block_size = 4
+indicators_MinBlockSize =4 
+Max_sample_size = 12
+compGconec= 1
+conec=no
+
+
+[Solver]
+num_of_cores = 1
+ref = 2
+solve = no
+rtol = 1e-4
+
+[K-Postprocess]
+MinBlockSize =1
+Max_sample_size = 4
+kperm=no
+postprocess=no
+SaveVfield=no

tests/performance/conf_gen_64.ini

@@ -0,0 +1,44 @@
+[General]
+simDir=tests/performance/tmp_gen_output
+startJob=0
+[Iterables]
+p=[10,39,10]
+seeds=[5462,2]
+lc=[4]
+connectivity=[1]
+variances=[1]
+
+[Generation]
+Nx = 64
+Ny = 64
+Nz = 64
+variogram_type=1
+binary = yes
+kh = 100
+kl = 0.01
+compute_lc = yes
+lcBin=yes
+genera=yes
+
+
+[Connectivity]
+keep_aspect= no
+block_size = 4
+indicators_MinBlockSize =4 
+Max_sample_size = 12
+compGconec= 1
+conec=no
+
+
+[Solver]
+num_of_cores = 1
+ref = 2
+solve = no
+rtol = 1e-4
+
+[K-Postprocess]
+MinBlockSize =1
+Max_sample_size = 4
+kperm=no
+postprocess=no
+SaveVfield=no

tests/performance/connectivity.py

@@ -0,0 +1,57 @@
+import os
+from benchmarker import Benchmarker
+
+os.chdir('../..')
+
+config_gen_file_64 = os.path.abspath("./tests/performance/conf_gen_64.ini")
+config_conn_file_64 = os.path.abspath("./tests/performance/conf_conn_64.ini")
+
+GEN_CONFIG_FILES = [config_gen_file_64]
+CONN_CONFIG_FILES = [config_conn_file_64]
+
+index_1 = 0
+
+index_8 = 0
+
+'''
+Esta etapa tarda mucho tiempo y no es muy independiente de la generación de medios.
+Si se generan medios con los parámetros dados:
+[Iterables]
+p=[10,39,15]
+seeds=[5462,2]
+lc=[4]
+connectivity=[1,2,3]
+variances=[1]
+Se generan 90 medios: 15 (p[2]) * 2 (seeds[1]) * 3 (len(connectivity)) * 1 (len(lc))
+Pero si se toman esos medios generados y se aplica solo la etapa de conectividad
+Se calcula la conectividad sobre 6 medios: 2 (seeds[1]) * 3 (len(connectivity)) * 1 (len(variances))* 1 (len(lc))
+Solucion: marcar en la etapa de generacion binary = yes -> esta bien esto?
+'''
+
+with Benchmarker() as bench:
+
+    for i in range(len(CONN_CONFIG_FILES)):
+        size = 2**(6+i)
+
+        @bench(f"Connectivity 1 core with size {size}")
+        def _(bm):
+            global index_1
+            os.system(f"CONFIG_FILE_PATH={GEN_CONFIG_FILES[index_1]} TEST=True mpirun -oversubscribe -np 1 python3 mpirunner.py")
+            with bm:
+                os.system(f"CONFIG_FILE_PATH={CONN_CONFIG_FILES[index_1]} TEST=True mpirun -oversubscribe -np 1 python3 mpirunner.py")
+
+            ## teardown
+            os.system("rm -rf ./tests/performance/tmp_gen_output")
+            index_1 +=1
+
+        @bench(f"Connectivity 8 core with size {size}")
+        def _(bm):
+            global index_8
+            os.system(f"CONFIG_FILE_PATH={GEN_CONFIG_FILES[index_8]} TEST=True mpirun -oversubscribe -np 8 python3 mpirunner.py")
+            with bm:
+                os.system(f"CONFIG_FILE_PATH={CONN_CONFIG_FILES[index_8]} TEST=True mpirun -oversubscribe -np 8 python3 mpirunner.py")
+
+            ## teardown
+            os.system("rm -rf ./tests/performance/tmp_gen_output")
+            index_8 +=1
+

tests/performance/generation.py

@@ -0,0 +1,43 @@
+import os
+from benchmarker import Benchmarker
+
+os.chdir('../..')
+
+config_file_64 = os.path.abspath("./tests/performance/conf_gen_64.ini")
+
+config_file_128 = os.path.abspath("./tests/performance/conf_gen_128.ini")
+
+config_file_256 = os.path.abspath("./tests/performance/conf_gen_256.ini")
+
+CONFIG_FILES = [config_file_64, config_file_128, config_file_256]
+
+index_1 = 0
+
+index_8 = 0
+
+with Benchmarker() as bench:
+
+    for i in range(len(CONFIG_FILES)):
+        size = 2**(6+i)
+
+        @bench(f"generation 1 core {size} tamaño")
+        def _(bm):
+            global index_1
+            config_file = CONFIG_FILES[index_1]
+            with bm:
+                os.system(f"CONFIG_FILE_PATH={config_file} TEST=True mpirun -oversubscribe -np 1 python3 mpirunner.py")
+
+            ## teardown
+            os.system("rm -rf ./tests/performance/tmp_gen_output")
+            index_1 +=1
+
+        @bench(f"generation 8 core {size} tamaño")
+        def _(bm):
+            global index_8
+            config_file = CONFIG_FILES[index_8]
+            with bm:
+                os.system(f"CONFIG_FILE_PATH={config_file} TEST=True mpirun -oversubscribe -np 8 python3 mpirunner.py")
+
+            ## teardown
+            os.system("rm -rf ./tests/performance/tmp_gen_output")
+            index_8 +=1

tools/Prealization.py

@@ -8,6 +8,7 @@ from tools.postprocessK.comp_PostKeff import comp_postKeff
 from shutil import copyfile
 from tools.solver.comp_Kperm_scale import comp_kperm_sub
 from tools.solver.Ndar import PetscP
+from tools.generation.fftma_gen import fftmaGenerator
 
 CONFIG_FILE_PATH = 'config.ini' if 'CONFIG_FILE_PATH' not in os.environ else os.environ['CONFIG_FILE_PATH']
 
@@ -33,11 +34,10 @@ def realization(job):
 
 	genera=parser.get('Generation',"genera")
 	if genera!='no':
-		os.system('CONFIG_FILE_PATH=' + CONFIG_FILE_PATH + ' python3 ./tools/generation/fftma_gen.py ' + datadir +' ' + str(job))
+		fftmaGenerator(datadir, job, CONFIG_FILE_PATH)
-
+		#os.system('CONFIG_FILE_PATH=' + CONFIG_FILE_PATH + ' python3 ./tools/generation/fftma_gen.py ' + datadir +' ' + str(job))
 	
 	nr= DotheLoop(job,parser, iterables)[3] -iterables['seeds'][0]
-
 	Cconec=parser.get('Connectivity',"conec")
 	if Cconec!='no':
 		comp_connec(parser,datadir,nr)

tools/connec/comp_cmap.py

@@ -9,7 +9,6 @@ import json
 
 def comp_connec(parser,rundir,nr):
 
-
 	kc=np.load(rundir+'k.npy')
 	keep_aspect = parser.get('Connectivity','keep_aspect')
 	kh,sx = float(parser.get('Generation','kh')),int(parser.get('Connectivity','block_size'))

tools/generation/config.py

@@ -49,7 +49,6 @@ def DotheLoop(job,parser,iterables):
 				njobs=len(variances)*len(cons)*len(seeds)*len(lcs)
 			else:
 				njobs=len(variances)*(len(cons)-1)*len(seeds)*len(lcs)+len(variances)*len(seeds)
-
 		return njobs
 
 	i=0

tools/generation/fftma_gen.py

@@ -3,14 +3,13 @@ from scipy.special import erf, erfinv
 from scipy.optimize import curve_fit
 from scipy.stats import norm
 from FFTMA import gen
-from config import DotheLoop, get_config	
+from tools.generation.config import DotheLoop, get_config	
-from variograma import get_lc
+from tools.generation.variograma import get_lc
 from scipy.interpolate import interp1d
 import sys
 import time
 import os
-
+#from memory_profiler import profile
-
 
 def fftmaGenerator(datadir,job,conffile):
 
@@ -79,7 +78,7 @@ def obtainLctobinBack(p,con,vario):
 	return 1.0/f(p)
 
  
-
+#@profile
 def generate_K(Nx,Ny,Nz,con,lc,p,kh,kl,seed,logn,LogVariance,vario,datadir,compute_lc,uselc_bin):
 
 
@@ -154,6 +153,6 @@ def binarize(kc,kh,kl,p):
 		kc=np.where(kc<t1, kl,kh)
 	return kc
 
-CONFIG_FILE_PATH = 'config.ini' if 'CONFIG_FILE_PATH' not in os.environ else os.environ['CONFIG_FILE_PATH']
+#CONFIG_FILE_PATH = 'config.ini' if 'CONFIG_FILE_PATH' not in os.environ else os.environ['CONFIG_FILE_PATH']
 
-fftmaGenerator(sys.argv[1],int(sys.argv[2]),CONFIG_FILE_PATH)
+#fftmaGenerator(sys.argv[1],int(sys.argv[2]),CONFIG_FILE_PATH)

utilities/conditional_decorator.py

@@ -0,0 +1,7 @@
+def conditional_decorator(dec, condition):
+    def decorator(func):
+        if not condition:
+            # Return the function unchanged, not decorated.
+            return func
+        return dec(func)
+    return decorator