ssantisi
/
simulacion-permeabilidad
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add memory profiler to a file

Browse Source
milestone_5_without_improvements-logs
chortas 3 years ago
parent e5c70006f4
commit 6510ea7a88
6 changed files with 696 additions and 287 deletions
Show Stats Download Patch File Download Diff File

199
fftma_module/gen/.ipynb_checkpoints/analysis-checkpoint.ipynb
Unescape Escape View File

@ -0,0 +1,199 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd "
]
},
{
"cell_type": "code",
"execution_count": 33,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>elapsed_time (seconds)</th>\n",
" <th>function_name</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.150900</td>\n",
" <td>./lib_src/generate.c:75:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>0.000051</td>\n",
" <td>./lib_src/build_real.c:50:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>0.000091</td>\n",
" <td>./lib_src/cov_value.c:60:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>0.000021</td>\n",
" <td>./lib_src/length.c:47:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>0.000017</td>\n",
" <td>./lib_src/clean_real.c:48:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>0.000068</td>\n",
" <td>./lib_src/gasdev.c:43:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>0.082331</td>\n",
" <td>./lib_src/covariance.c:86:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>0.000713</td>\n",
" <td>./lib_src/gasdev.c:35:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.000101</td>\n",
" <td>./lib_src/ran2.c:68:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>0.000047</td>\n",
" <td>./lib_src/prebuild_gwn.c:57:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>0.236221</td>\n",
" <td>./lib_src/Py_kgeneration.c:74:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>0.084618</td>\n",
" <td>./lib_src/fftma2.c:106:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>0.000007</td>\n",
" <td>./lib_src/maxfactor.c:42:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>0.000835</td>\n",
" <td>./lib_src/fourt.c:593:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>0.000086</td>\n",
" <td>./lib_src/cgrid.c:50:</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>0.000026</td>\n",
" <td>./lib_src/Py_getvalues.c:157:</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" elapsed_time (seconds) function_name\n",
"0 0.150900 ./lib_src/generate.c:75:\n",
"1 0.000051 ./lib_src/build_real.c:50:\n",
"2 0.000091 ./lib_src/cov_value.c:60:\n",
"3 0.000021 ./lib_src/length.c:47:\n",
"4 0.000017 ./lib_src/clean_real.c:48:\n",
"5 0.000068 ./lib_src/gasdev.c:43:\n",
"6 0.082331 ./lib_src/covariance.c:86:\n",
"7 0.000713 ./lib_src/gasdev.c:35:\n",
"8 0.000101 ./lib_src/ran2.c:68:\n",
"9 0.000047 ./lib_src/prebuild_gwn.c:57:\n",
"10 0.236221 ./lib_src/Py_kgeneration.c:74:\n",
"11 0.084618 ./lib_src/fftma2.c:106:\n",
"12 0.000007 ./lib_src/maxfactor.c:42:\n",
"13 0.000835 ./lib_src/fourt.c:593:\n",
"14 0.000086 ./lib_src/cgrid.c:50:\n",
"15 0.000026 ./lib_src/Py_getvalues.c:157:"
]
},
"execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data = {}\n",
"with open(\"log_16.txt\") as log_file:\n",
" lines = log_file.readlines()\n",
" for line in lines:\n",
" if \"ELAPSED\" in line:\n",
" split_line = line.split()\n",
" idx_elapsed = split_line.index(\"ELAPSED\") + 2\n",
" function_name, elapsed = split_line[2], float(split_line[idx_elapsed])\n",
" if function_name not in data or (function_name in data and elapsed > data[function_name]):\n",
" data[function_name] = elapsed\n",
"new_data = {\"function_name\": data.keys(), \"elapsed_time (seconds)\": data.values()}\n",
" \n",
"df = pd.DataFrame(new_data) \n",
"\n",
"df"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.18"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

737
fftma_module/gen/analysis.ipynb
Unescape Escape View File

@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
@ -11,17 +11,21 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 45,
"metadata": {},
"outputs": [],
"source": [
"def get_function_name(function_name):\n",
" return function_name[10:].rsplit(\".c\")[0]"
]
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['12:30:24', 'INFO', './lib_src/generate.c:61:', 'TOTAL', 'VIRTUAL', 'MEM', '=', '7683.0', 'MB,', 'USED', 'VIRTUAL', 'MEM', '=', '304.2', 'MB,', 'USED', 'VIRTUAL', 'MEM', 'BY', 'CURRENT', 'PROCESS', '=', '1072', 'MB']\n",
"['12:30:24', 'INFO', './lib_src/Py_kgeneration.c:64:', 'TOTAL', 'VIRTUAL', 'MEM', '=', '7683.0', 'MB,', 'USED', 'VIRTUAL', 'MEM', '=', '303.6', 'MB,', 'USED', 'VIRTUAL', 'MEM', 'BY', 'CURRENT', 'PROCESS', '=', '1072', 'MB']\n"
]
},
{
"data": {
"text/html": [
@ -43,133 +47,142 @@
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>function_name</th>\n",
" <th>elapsed_time</th>\n",
" <th>executions</th>\n",
" <th>function_name</th>\n",
" <th>used_virtual_mem</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>./lib_src/Py_getvalues.c:157:</td>\n",
" <td>0.000007</td>\n",
" <td>0.000456</td>\n",
" <td>1</td>\n",
" <td>cgrid</td>\n",
" <td>2877.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>./lib_src/ran2.c:68:</td>\n",
" <td>0.000004</td>\n",
" <td>142</td>\n",
" <td>0.180264</td>\n",
" <td>1</td>\n",
" <td>Py_kgeneration</td>\n",
" <td>2877.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>./lib_src/gasdev.c:35:</td>\n",
" <td>0.000023</td>\n",
" <td>151</td>\n",
" <td>0.001184</td>\n",
" <td>512</td>\n",
" <td>gasdev</td>\n",
" <td>2882.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>./lib_src/gasdev.c:43:</td>\n",
" <td>0.000003</td>\n",
" <td>202</td>\n",
" <td>0.000081</td>\n",
" <td>700</td>\n",
" <td>cov_value</td>\n",
" <td>2877.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>./lib_src/generate.c:75:</td>\n",
" <td>0.010957</td>\n",
" <td>0.000015</td>\n",
" <td>1</td>\n",
" <td>clean_real</td>\n",
" <td>2877.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>./lib_src/maxfactor.c:42:</td>\n",
" <td>0.000004</td>\n",
" <td>0.077514</td>\n",
" <td>1</td>\n",
" <td>covariance</td>\n",
" <td>2877.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>./lib_src/length.c:47:</td>\n",
" <td>0.000013</td>\n",
" <td>0.079166</td>\n",
" <td>1</td>\n",
" <td>fftma2</td>\n",
" <td>2877.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>./lib_src/cgrid.c:50:</td>\n",
" <td>0.000060</td>\n",
" <td>0.100123</td>\n",
" <td>1</td>\n",
" <td>generate</td>\n",
" <td>2877.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>./lib_src/cov_value.c:60:</td>\n",
" <td>0.000011</td>\n",
" <td>76</td>\n",
" <td>0.000098</td>\n",
" <td>3</td>\n",
" <td>length</td>\n",
" <td>2877.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>./lib_src/covariance.c:86:</td>\n",
" <td>0.012162</td>\n",
" <td>0.000034</td>\n",
" <td>1</td>\n",
" <td>Py_getvalues</td>\n",
" <td>0.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>./lib_src/fourt.c:593:</td>\n",
" <td>0.000065</td>\n",
" <td>2</td>\n",
" <td>0.000037</td>\n",
" <td>702</td>\n",
" <td>ran2</td>\n",
" <td>2882.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>./lib_src/prebuild_gwn.c:57:</td>\n",
" <td>0.000009</td>\n",
" <td>1</td>\n",
" <td>0.000193</td>\n",
" <td>3</td>\n",
" <td>fourt</td>\n",
" <td>2877.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>./lib_src/build_real.c:50:</td>\n",
" <td>0.000019</td>\n",
" <td>0.000033</td>\n",
" <td>1</td>\n",
" <td>build_real</td>\n",
" <td>2877.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>./lib_src/clean_real.c:48:</td>\n",
" <td>0.000007</td>\n",
" <td>0.000034</td>\n",
" <td>1</td>\n",
" <td>prebuild_gwn</td>\n",
" <td>2877.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>./lib_src/fftma2.c:106:</td>\n",
" <td>0.012503</td>\n",
" <td>1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>./lib_src/Py_kgeneration.c:74:</td>\n",
" <td>0.023829</td>\n",
" <td>1</td>\n",
" <td>0.000010</td>\n",
" <td>3</td>\n",
" <td>maxfactor</td>\n",
" <td>2877.4</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" function_name elapsed_time executions\n",
"0 ./lib_src/Py_getvalues.c:157: 0.000007 1\n",
"1 ./lib_src/ran2.c:68: 0.000004 142\n",
"2 ./lib_src/gasdev.c:35: 0.000023 151\n",
"3 ./lib_src/gasdev.c:43: 0.000003 202\n",
"4 ./lib_src/generate.c:75: 0.010957 1\n",
"5 ./lib_src/maxfactor.c:42: 0.000004 1\n",
"6 ./lib_src/length.c:47: 0.000013 1\n",
"7 ./lib_src/cgrid.c:50: 0.000060 1\n",
"8 ./lib_src/cov_value.c:60: 0.000011 76\n",
"9 ./lib_src/covariance.c:86: 0.012162 1\n",
"10 ./lib_src/fourt.c:593: 0.000065 2\n",
"11 ./lib_src/prebuild_gwn.c:57: 0.000009 1\n",
"12 ./lib_src/build_real.c:50: 0.000019 1\n",
"13 ./lib_src/clean_real.c:48: 0.000007 1\n",
"14 ./lib_src/fftma2.c:106: 0.012503 1\n",
"15 ./lib_src/Py_kgeneration.c:74: 0.023829 1"
" elapsed_time executions function_name used_virtual_mem\n",
"0 0.000456 1 cgrid 2877.4\n",
"1 0.180264 1 Py_kgeneration 2877.9\n",
"2 0.001184 512 gasdev 2882.6\n",
"3 0.000081 700 cov_value 2877.9\n",
"4 0.000015 1 clean_real 2877.9\n",
"5 0.077514 1 covariance 2877.9\n",
"6 0.079166 1 fftma2 2877.9\n",
"7 0.100123 1 generate 2877.4\n",
"8 0.000098 3 length 2877.4\n",
"9 0.000034 1 Py_getvalues 0.0\n",
"10 0.000037 702 ran2 2882.6\n",
"11 0.000193 3 fourt 2877.9\n",
"12 0.000033 1 build_real 2877.9\n",
"13 0.000034 1 prebuild_gwn 2877.9\n",
"14 0.000010 3 maxfactor 2877.4"
]
},
"execution_count": 3,
"execution_count": 54,
"metadata": {},
"output_type": "execute_result"
}
@ -182,17 +195,19 @@
" for line in lines:\n",
" if \"MEM\" in line:\n",
" split_line = line.split()\n",
" print(split_line)\n",
" idx_used_mem = split_line.index(\"USED\") + 4\n",
" function_name, used_virtual_mem = get_function_name(split_line[2]), float(split_line[idx_used_mem])\n",
" val = data.get(function_name, (0, 0, used_virtual_mem))\n",
" data[function_name] = (val[0], val[1], max(used_virtual_mem, val[2])) \n",
" if \"ELAPSED\" in line:\n",
" split_line = line.split()\n",
" idx_elapsed = split_line.index(\"ELAPSED\") + 2\n",
" function_name, elapsed = split_line[2], float(split_line[idx_elapsed])\n",
" if function_name not in data or (function_name in data and elapsed > data[function_name][0]):\n",
" val = data.get(function_name, (elapsed, 0))\n",
" data[function_name] = (val[0], val[1] + 1)\n",
" function_name, elapsed = get_function_name(split_line[2]), float(split_line[idx_elapsed])\n",
" val = data.get(function_name, (elapsed, 0, 0))\n",
" data[function_name] = (max(elapsed, val[0]), val[1] + 1, val[2])\n",
"\n",
"values = data.values()\n",
"new_data = {\"function_name\": data.keys(), \"elapsed_time\": map(lambda x: x[0], values), \"executions\": map(lambda x: x[1], values)}\n",
"new_data = {\"function_name\": data.keys(), \"elapsed_time\": map(lambda x: x[0], values), \"executions\": map(lambda x: x[1], values), \"used_virtual_mem\": map(lambda x: x[2], values)}\n",
" \n",
"df = pd.DataFrame(new_data) \n",
"\n",
@ -201,7 +216,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 55,
"metadata": {},
"outputs": [],
"source": [
@ -210,7 +225,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 56,
"metadata": {},
"outputs": [
{
@ -234,150 +249,158 @@
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>function_name</th>\n",
" <th>elapsed_time</th>\n",
" <th>executions</th>\n",
" <th>function_name</th>\n",
" <th>used_virtual_mem</th>\n",
" <th>total_time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>./lib_src/Py_kgeneration.c:74:</td>\n",
" <td>0.023829</td>\n",
" <td>1</td>\n",
" <td>0.023829</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>./lib_src/fftma2.c:106:</td>\n",
" <td>0.012503</td>\n",
" <th>1</th>\n",
" <td>0.180264</td>\n",
" <td>1</td>\n",
" <td>0.012503</td>\n",
" <td>Py_kgeneration</td>\n",
" <td>2877.9</td>\n",
" <td>0.180264</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>./lib_src/covariance.c:86:</td>\n",
" <td>0.012162</td>\n",
" <th>7</th>\n",
" <td>0.100123</td>\n",
" <td>1</td>\n",
" <td>0.012162</td>\n",
" <td>generate</td>\n",
" <td>2877.4</td>\n",
" <td>0.100123</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>./lib_src/generate.c:75:</td>\n",
" <td>0.010957</td>\n",
" <th>6</th>\n",
" <td>0.079166</td>\n",
" <td>1</td>\n",
" <td>0.010957</td>\n",
" <td>fftma2</td>\n",
" <td>2877.9</td>\n",
" <td>0.079166</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>./lib_src/fourt.c:593:</td>\n",
" <td>0.000065</td>\n",
" <td>2</td>\n",
" <td>0.000130</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>./lib_src/cgrid.c:50:</td>\n",
" <td>0.000060</td>\n",
" <th>5</th>\n",
" <td>0.077514</td>\n",
" <td>1</td>\n",
" <td>0.000060</td>\n",
" <td>covariance</td>\n",
" <td>2877.9</td>\n",
" <td>0.077514</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>./lib_src/gasdev.c:35:</td>\n",
" <td>0.000023</td>\n",
" <td>151</td>\n",
" <td>0.003473</td>\n",
" <td>0.001184</td>\n",
" <td>512</td>\n",
" <td>gasdev</td>\n",
" <td>2882.6</td>\n",
" <td>0.606208</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>./lib_src/build_real.c:50:</td>\n",
" <td>0.000019</td>\n",
" <th>0</th>\n",
" <td>0.000456</td>\n",
" <td>1</td>\n",
" <td>0.000019</td>\n",
" <td>cgrid</td>\n",
" <td>2877.4</td>\n",
" <td>0.000456</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>./lib_src/length.c:47:</td>\n",
" <td>0.000013</td>\n",
" <td>1</td>\n",
" <td>0.000013</td>\n",
" <th>11</th>\n",
" <td>0.000193</td>\n",
" <td>3</td>\n",
" <td>fourt</td>\n",
" <td>2877.9</td>\n",
" <td>0.000579</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>./lib_src/cov_value.c:60:</td>\n",
" <td>0.000011</td>\n",
" <td>76</td>\n",
" <td>0.000836</td>\n",
" <td>0.000098</td>\n",
" <td>3</td>\n",
" <td>length</td>\n",
" <td>2877.4</td>\n",
" <td>0.000294</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>./lib_src/prebuild_gwn.c:57:</td>\n",
" <td>0.000009</td>\n",
" <td>1</td>\n",
" <td>0.000009</td>\n",
" <th>3</th>\n",
" <td>0.000081</td>\n",
" <td>700</td>\n",
" <td>cov_value</td>\n",
" <td>2877.9</td>\n",
" <td>0.056700</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>./lib_src/Py_getvalues.c:157:</td>\n",
" <td>0.000007</td>\n",
" <th>10</th>\n",
" <td>0.000037</td>\n",
" <td>702</td>\n",
" <td>ran2</td>\n",
" <td>2882.6</td>\n",
" <td>0.025974</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>0.000034</td>\n",
" <td>1</td>\n",
" <td>0.000007</td>\n",
" <td>Py_getvalues</td>\n",
" <td>0.0</td>\n",
" <td>0.000034</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>./lib_src/clean_real.c:48:</td>\n",
" <td>0.000007</td>\n",
" <td>0.000034</td>\n",
" <td>1</td>\n",
" <td>0.000007</td>\n",
" <td>prebuild_gwn</td>\n",
" <td>2877.9</td>\n",
" <td>0.000034</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>./lib_src/ran2.c:68:</td>\n",
" <td>0.000004</td>\n",
" <td>142</td>\n",
" <td>0.000568</td>\n",
" <th>12</th>\n",
" <td>0.000033</td>\n",
" <td>1</td>\n",
" <td>build_real</td>\n",
" <td>2877.9</td>\n",
" <td>0.000033</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>./lib_src/maxfactor.c:42:</td>\n",
" <td>0.000004</td>\n",
" <th>4</th>\n",
" <td>0.000015</td>\n",
" <td>1</td>\n",
" <td>0.000004</td>\n",
" <td>clean_real</td>\n",
" <td>2877.9</td>\n",
" <td>0.000015</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>./lib_src/gasdev.c:43:</td>\n",
" <td>0.000003</td>\n",
" <td>202</td>\n",
" <td>0.000606</td>\n",
" <th>14</th>\n",
" <td>0.000010</td>\n",
" <td>3</td>\n",
" <td>maxfactor</td>\n",
" <td>2877.4</td>\n",
" <td>0.000030</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" function_name elapsed_time executions total_time\n",
"15 ./lib_src/Py_kgeneration.c:74: 0.023829 1 0.023829\n",
"14 ./lib_src/fftma2.c:106: 0.012503 1 0.012503\n",
"9 ./lib_src/covariance.c:86: 0.012162 1 0.012162\n",
"4 ./lib_src/generate.c:75: 0.010957 1 0.010957\n",
"10 ./lib_src/fourt.c:593: 0.000065 2 0.000130\n",
"7 ./lib_src/cgrid.c:50: 0.000060 1 0.000060\n",
"2 ./lib_src/gasdev.c:35: 0.000023 151 0.003473\n",
"12 ./lib_src/build_real.c:50: 0.000019 1 0.000019\n",
"6 ./lib_src/length.c:47: 0.000013 1 0.000013\n",
"8 ./lib_src/cov_value.c:60: 0.000011 76 0.000836\n",
"11 ./lib_src/prebuild_gwn.c:57: 0.000009 1 0.000009\n",
"0 ./lib_src/Py_getvalues.c:157: 0.000007 1 0.000007\n",
"13 ./lib_src/clean_real.c:48: 0.000007 1 0.000007\n",
"1 ./lib_src/ran2.c:68: 0.000004 142 0.000568\n",
"5 ./lib_src/maxfactor.c:42: 0.000004 1 0.000004\n",
"3 ./lib_src/gasdev.c:43: 0.000003 202 0.000606"
" elapsed_time executions function_name used_virtual_mem total_time\n",
"1 0.180264 1 Py_kgeneration 2877.9 0.180264\n",
"7 0.100123 1 generate 2877.4 0.100123\n",
"6 0.079166 1 fftma2 2877.9 0.079166\n",
"5 0.077514 1 covariance 2877.9 0.077514\n",
"2 0.001184 512 gasdev 2882.6 0.606208\n",
"0 0.000456 1 cgrid 2877.4 0.000456\n",
"11 0.000193 3 fourt 2877.9 0.000579\n",
"8 0.000098 3 length 2877.4 0.000294\n",
"3 0.000081 700 cov_value 2877.9 0.056700\n",
"10 0.000037 702 ran2 2882.6 0.025974\n",
"9 0.000034 1 Py_getvalues 0.0 0.000034\n",
"13 0.000034 1 prebuild_gwn 2877.9 0.000034\n",
"12 0.000033 1 build_real 2877.9 0.000033\n",
"4 0.000015 1 clean_real 2877.9 0.000015\n",
"14 0.000010 3 maxfactor 2877.4 0.000030"
]
},
"execution_count": 5,
"execution_count": 56,
"metadata": {},
"output_type": "execute_result"
}
@ -388,7 +411,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 57,
"metadata": {},
"outputs": [
{
@ -412,156 +435,350 @@
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>function_name</th>\n",
" <th>elapsed_time</th>\n",
" <th>executions</th>\n",
" <th>function_name</th>\n",
" <th>used_virtual_mem</th>\n",
" <th>total_time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>./lib_src/Py_kgeneration.c:74:</td>\n",
" <td>0.023829</td>\n",
" <th>2</th>\n",
" <td>0.001184</td>\n",
" <td>512</td>\n",
" <td>gasdev</td>\n",
" <td>2882.6</td>\n",
" <td>0.606208</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>0.180264</td>\n",
" <td>1</td>\n",
" <td>0.023829</td>\n",
" <td>Py_kgeneration</td>\n",
" <td>2877.9</td>\n",
" <td>0.180264</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>./lib_src/fftma2.c:106:</td>\n",
" <td>0.012503</td>\n",
" <th>7</th>\n",
" <td>0.100123</td>\n",
" <td>1</td>\n",
" <td>0.012503</td>\n",
" <td>generate</td>\n",
" <td>2877.4</td>\n",
" <td>0.100123</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>./lib_src/covariance.c:86:</td>\n",
" <td>0.012162</td>\n",
" <th>6</th>\n",
" <td>0.079166</td>\n",
" <td>1</td>\n",
" <td>0.012162</td>\n",
" <td>fftma2</td>\n",
" <td>2877.9</td>\n",
" <td>0.079166</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>./lib_src/generate.c:75:</td>\n",
" <td>0.010957</td>\n",
" <th>5</th>\n",
" <td>0.077514</td>\n",
" <td>1</td>\n",
" <td>0.010957</td>\n",
" <td>covariance</td>\n",
" <td>2877.9</td>\n",
" <td>0.077514</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>./lib_src/gasdev.c:35:</td>\n",
" <td>0.000023</td>\n",
" <td>151</td>\n",
" <td>0.003473</td>\n",
" <th>3</th>\n",
" <td>0.000081</td>\n",
" <td>700</td>\n",
" <td>cov_value</td>\n",
" <td>2877.9</td>\n",
" <td>0.056700</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>0.000037</td>\n",
" <td>702</td>\n",
" <td>ran2</td>\n",
" <td>2882.6</td>\n",
" <td>0.025974</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>0.000193</td>\n",
" <td>3</td>\n",
" <td>fourt</td>\n",
" <td>2877.9</td>\n",
" <td>0.000579</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.000456</td>\n",
" <td>1</td>\n",
" <td>cgrid</td>\n",
" <td>2877.4</td>\n",
" <td>0.000456</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>./lib_src/cov_value.c:60:</td>\n",
" <td>0.000011</td>\n",
" <td>76</td>\n",
" <td>0.000836</td>\n",
" <td>0.000098</td>\n",
" <td>3</td>\n",
" <td>length</td>\n",
" <td>2877.4</td>\n",
" <td>0.000294</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>./lib_src/gasdev.c:43:</td>\n",
" <td>0.000003</td>\n",
" <td>202</td>\n",
" <td>0.000606</td>\n",
" <th>9</th>\n",
" <td>0.000034</td>\n",
" <td>1</td>\n",
" <td>Py_getvalues</td>\n",
" <td>0.0</td>\n",
" <td>0.000034</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>./lib_src/ran2.c:68:</td>\n",
" <td>0.000004</td>\n",
" <td>142</td>\n",
" <td>0.000568</td>\n",
" <th>13</th>\n",
" <td>0.000034</td>\n",
" <td>1</td>\n",
" <td>prebuild_gwn</td>\n",
" <td>2877.9</td>\n",
" <td>0.000034</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>0.000033</td>\n",
" <td>1</td>\n",
" <td>build_real</td>\n",
" <td>2877.9</td>\n",
" <td>0.000033</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>0.000010</td>\n",
" <td>3</td>\n",
" <td>maxfactor</td>\n",
" <td>2877.4</td>\n",
" <td>0.000030</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>0.000015</td>\n",
" <td>1</td>\n",
" <td>clean_real</td>\n",
" <td>2877.9</td>\n",
" <td>0.000015</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" elapsed_time executions function_name used_virtual_mem total_time\n",
"2 0.001184 512 gasdev 2882.6 0.606208\n",
"1 0.180264 1 Py_kgeneration 2877.9 0.180264\n",
"7 0.100123 1 generate 2877.4 0.100123\n",
"6 0.079166 1 fftma2 2877.9 0.079166\n",
"5 0.077514 1 covariance 2877.9 0.077514\n",
"3 0.000081 700 cov_value 2877.9 0.056700\n",
"10 0.000037 702 ran2 2882.6 0.025974\n",
"11 0.000193 3 fourt 2877.9 0.000579\n",
"0 0.000456 1 cgrid 2877.4 0.000456\n",
"8 0.000098 3 length 2877.4 0.000294\n",
"9 0.000034 1 Py_getvalues 0.0 0.000034\n",
"13 0.000034 1 prebuild_gwn 2877.9 0.000034\n",
"12 0.000033 1 build_real 2877.9 0.000033\n",
"14 0.000010 3 maxfactor 2877.4 0.000030\n",
"4 0.000015 1 clean_real 2877.9 0.000015"
]
},
"execution_count": 57,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.sort_values(by=[\"total_time\"], ascending=False)"
]
},
{
"cell_type": "code",
"execution_count": 58,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>elapsed_time</th>\n",
" <th>executions</th>\n",
" <th>function_name</th>\n",
" <th>used_virtual_mem</th>\n",
" <th>total_time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>0.001184</td>\n",
" <td>512</td>\n",
" <td>gasdev</td>\n",
" <td>2882.6</td>\n",
" <td>0.606208</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>./lib_src/fourt.c:593:</td>\n",
" <td>0.000065</td>\n",
" <td>2</td>\n",
" <td>0.000130</td>\n",
" <td>0.000037</td>\n",
" <td>702</td>\n",
" <td>ran2</td>\n",
" <td>2882.6</td>\n",
" <td>0.025974</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>./lib_src/cgrid.c:50:</td>\n",
" <td>0.000060</td>\n",
" <th>1</th>\n",
" <td>0.180264</td>\n",
" <td>1</td>\n",
" <td>0.000060</td>\n",
" <td>Py_kgeneration</td>\n",
" <td>2877.9</td>\n",
" <td>0.180264</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>./lib_src/build_real.c:50:</td>\n",
" <td>0.000019</td>\n",
" <th>3</th>\n",
" <td>0.000081</td>\n",
" <td>700</td>\n",
" <td>cov_value</td>\n",
" <td>2877.9</td>\n",
" <td>0.056700</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>0.000015</td>\n",
" <td>1</td>\n",
" <td>0.000019</td>\n",
" <td>clean_real</td>\n",
" <td>2877.9</td>\n",
" <td>0.000015</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>0.077514</td>\n",
" <td>1</td>\n",
" <td>covariance</td>\n",
" <td>2877.9</td>\n",
" <td>0.077514</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>./lib_src/length.c:47:</td>\n",
" <td>0.000013</td>\n",
" <td>0.079166</td>\n",
" <td>1</td>\n",
" <td>0.000013</td>\n",
" <td>fftma2</td>\n",
" <td>2877.9</td>\n",
" <td>0.079166</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>./lib_src/prebuild_gwn.c:57:</td>\n",
" <td>0.000009</td>\n",
" <td>0.000193</td>\n",
" <td>3</td>\n",
" <td>fourt</td>\n",
" <td>2877.9</td>\n",
" <td>0.000579</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>0.000033</td>\n",
" <td>1</td>\n",
" <td>0.000009</td>\n",
" <td>build_real</td>\n",
" <td>2877.9</td>\n",
" <td>0.000033</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>0.000034</td>\n",
" <td>1</td>\n",
" <td>prebuild_gwn</td>\n",
" <td>2877.9</td>\n",
" <td>0.000034</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>./lib_src/Py_getvalues.c:157:</td>\n",
" <td>0.000007</td>\n",
" <td>0.000456</td>\n",
" <td>1</td>\n",
" <td>0.000007</td>\n",
" <td>cgrid</td>\n",
" <td>2877.4</td>\n",
" <td>0.000456</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>./lib_src/clean_real.c:48:</td>\n",
" <td>0.000007</td>\n",
" <th>7</th>\n",
" <td>0.100123</td>\n",
" <td>1</td>\n",
" <td>0.000007</td>\n",
" <td>generate</td>\n",
" <td>2877.4</td>\n",
" <td>0.100123</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>./lib_src/maxfactor.c:42:</td>\n",
" <td>0.000004</td>\n",
" <th>8</th>\n",
" <td>0.000098</td>\n",
" <td>3</td>\n",
" <td>length</td>\n",
" <td>2877.4</td>\n",
" <td>0.000294</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>0.000010</td>\n",
" <td>3</td>\n",
" <td>maxfactor</td>\n",
" <td>2877.4</td>\n",
" <td>0.000030</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>0.000034</td>\n",
" <td>1</td>\n",
" <td>0.000004</td>\n",
" <td>Py_getvalues</td>\n",
" <td>0.0</td>\n",
" <td>0.000034</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" function_name elapsed_time executions total_time\n",
"15 ./lib_src/Py_kgeneration.c:74: 0.023829 1 0.023829\n",
"14 ./lib_src/fftma2.c:106: 0.012503 1 0.012503\n",
"9 ./lib_src/covariance.c:86: 0.012162 1 0.012162\n",
"4 ./lib_src/generate.c:75: 0.010957 1 0.010957\n",
"2 ./lib_src/gasdev.c:35: 0.000023 151 0.003473\n",
"8 ./lib_src/cov_value.c:60: 0.000011 76 0.000836\n",
"3 ./lib_src/gasdev.c:43: 0.000003 202 0.000606\n",
"1 ./lib_src/ran2.c:68: 0.000004 142 0.000568\n",
"10 ./lib_src/fourt.c:593: 0.000065 2 0.000130\n",
"7 ./lib_src/cgrid.c:50: 0.000060 1 0.000060\n",
"12 ./lib_src/build_real.c:50: 0.000019 1 0.000019\n",
"6 ./lib_src/length.c:47: 0.000013 1 0.000013\n",
"11 ./lib_src/prebuild_gwn.c:57: 0.000009 1 0.000009\n",
"0 ./lib_src/Py_getvalues.c:157: 0.000007 1 0.000007\n",
"13 ./lib_src/clean_real.c:48: 0.000007 1 0.000007\n",
"5 ./lib_src/maxfactor.c:42: 0.000004 1 0.000004"
" elapsed_time executions function_name used_virtual_mem total_time\n",
"2 0.001184 512 gasdev 2882.6 0.606208\n",
"10 0.000037 702 ran2 2882.6 0.025974\n",
"1 0.180264 1 Py_kgeneration 2877.9 0.180264\n",
"3 0.000081 700 cov_value 2877.9 0.056700\n",
"4 0.000015 1 clean_real 2877.9 0.000015\n",
"5 0.077514 1 covariance 2877.9 0.077514\n",
"6 0.079166 1 fftma2 2877.9 0.079166\n",
"11 0.000193 3 fourt 2877.9 0.000579\n",
"12 0.000033 1 build_real 2877.9 0.000033\n",
"13 0.000034 1 prebuild_gwn 2877.9 0.000034\n",
"0 0.000456 1 cgrid 2877.4 0.000456\n",
"7 0.100123 1 generate 2877.4 0.100123\n",
"8 0.000098 3 length 2877.4 0.000294\n",
"14 0.000010 3 maxfactor 2877.4 0.000030\n",
"9 0.000034 1 Py_getvalues 0.0 0.000034"
]
},
"execution_count": 7,
"execution_count": 58,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.sort_values(by=[\"total_time\"], ascending=False)"
"df.sort_values(by=[\"used_virtual_mem\"], ascending=False)"
]
},
{
@ -716,14 +933,14 @@
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.9"
"pygments_lexer": "ipython2",
"version": "2.7.18"
}
},
"nbformat": 4,

19
fftma_module/gen/lib_src/build_real.c
Unescape Escape View File

@ -23,6 +23,9 @@
void build_real(int n[3], int NTOT, double* covar, double* realization, double* ireal) {
clock_t t = clock();
double* used_ram_t0 = malloc(sizeof(double));
getVirtualMemUsed(used_ram_t0);
int i, j, k, maille1;
double temp;
log_info("RESULT = in progress, NTOT = %d, covar = %f, n[0] = %d, n[1] = %d, n[2] = %d", NTOT, *covar, n[0], n[1], n[2]);
@ -47,17 +50,11 @@ void build_real(int n[3], int NTOT, double* covar, double* realization, double*
t = clock() - t;
double time_taken = ((double)t)/CLOCKS_PER_SEC; // calculate the elapsed time
double* total_ram = malloc(sizeof(double));
getTotalVirtualMem(total_ram);
double* used_ram = malloc(sizeof(double));
getVirtualMemUsed(used_ram);
log_info("TOTAL VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM BY CURRENT PROCESS = %d MB",
*total_ram, *used_ram, getVirtualMemUsedByCurrentProcess());
double* used_ram_tf = malloc(sizeof(double));
getVirtualMemUsed(used_ram_tf);
free(total_ram);
free(used_ram);
log_info("RESULT = success, realization = %f, ireal = %f, ELAPSED = %f seconds, USED VIRTUAL MEM = %5.1f MB, DIF USED VIRTUAL MEM = %5.1f MB", *realization, *ireal, time_taken, *used_ram_tf, *used_ram_tf - *used_ram_t0);
log_info("RESULT = success, realization = %f, ireal = %f, ELAPSED = %f seconds", *realization, *ireal, time_taken);
free(used_ram_t0);
free(used_ram_tf);
}

23
fftma_module/gen/lib_src/fftma2.c
Unescape Escape View File

@ -25,6 +25,9 @@
void FFTMA2(struct vario_mod variogram, struct grid_mod grid, int n[3], struct realization_mod* realin, struct realization_mod* realout) {
clock_t t = clock();
double* used_ram_t0 = malloc(sizeof(double));
getVirtualMemUsed(used_ram_t0);
log_info("RESULT = in progress");
@ -86,6 +89,9 @@ void FFTMA2(struct vario_mod variogram, struct grid_mod grid, int n[3], struct r
/* build realization in spectral domain */
build_real(n, NTOT, covar, realization, ireal);
double* used_ram_tf = malloc(sizeof(double));
getVirtualMemUsed(used_ram_tf);
free(covar);
/*backward fourier transform*/
@ -98,22 +104,11 @@ void FFTMA2(struct vario_mod variogram, struct grid_mod grid, int n[3], struct r
/*output realization*/
clean_real(realin, n, grid, realization, realout);
free(realization);
t = clock() - t;
double time_taken = ((double)t)/CLOCKS_PER_SEC; // calculate the elapsed time
double* total_ram = malloc(sizeof(double));
getTotalVirtualMem(total_ram);
double* used_ram = malloc(sizeof(double));
getVirtualMemUsed(used_ram);
log_info("TOTAL VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM BY CURRENT PROCESS = %d MB",
*total_ram, *used_ram, getVirtualMemUsedByCurrentProcess());
free(total_ram);
free(used_ram);
log_info("RESULT = success, NTOT = %d, NMAX = %d, NDIM = %d, ntot = %d, nmax = %d, NXYZ = %d, nxyz = %d, ELAPSED = %f seconds, USED VIRTUAL MEM = %5.1f MB, DIF USED VIRTUAL MEM = %5.1f MB", NTOT, NMAX, NDIM, ntot, nmax, NXYZ, nxyz, time_taken, *used_ram_tf, *used_ram_tf - *used_ram_t0);
log_info("RESULT = success, NTOT = %d, NMAX = %d, NDIM = %d, ntot = %d, nmax = %d, NXYZ = %d, nxyz = %d, ELAPSED = %f seconds", NTOT, NMAX, NDIM, ntot, nmax, NXYZ, nxyz, time_taken);
free(used_ram_t0);
free(used_ram_tf);
}

2
fftma_module/gen/save_logs.sh
Unescape Escape View File

@ -1 +1 @@
ENV=true python3 test.py 2>&1 | tee log_$1.txt
ENV=true python3 test.py $1 2>&1 | tee log_$1.txt

3
fftma_module/gen/test.py
Unescape Escape View File

@ -1,8 +1,9 @@
from FFTMA import gen
import numpy as np
import gc
import sys
N=64
N=int(sys.argv[1])
nx, ny, nz = N,N,N
dx, dy, dz = 1.0, 1.0, 1.0

Write Preview
Loading…
Cancel
Save