diff --git a/fftma_module/gen/.ipynb_checkpoints/analysis-checkpoint.ipynb b/fftma_module/gen/.ipynb_checkpoints/analysis-checkpoint.ipynb new file mode 100644 index 0000000..c6547c0 --- /dev/null +++ b/fftma_module/gen/.ipynb_checkpoints/analysis-checkpoint.ipynb @@ -0,0 +1,199 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd " + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
elapsed_time (seconds)function_name
00.150900./lib_src/generate.c:75:
10.000051./lib_src/build_real.c:50:
20.000091./lib_src/cov_value.c:60:
30.000021./lib_src/length.c:47:
40.000017./lib_src/clean_real.c:48:
50.000068./lib_src/gasdev.c:43:
60.082331./lib_src/covariance.c:86:
70.000713./lib_src/gasdev.c:35:
80.000101./lib_src/ran2.c:68:
90.000047./lib_src/prebuild_gwn.c:57:
100.236221./lib_src/Py_kgeneration.c:74:
110.084618./lib_src/fftma2.c:106:
120.000007./lib_src/maxfactor.c:42:
130.000835./lib_src/fourt.c:593:
140.000086./lib_src/cgrid.c:50:
150.000026./lib_src/Py_getvalues.c:157:
\n", + "
" + ], + "text/plain": [ + " elapsed_time (seconds) function_name\n", + "0 0.150900 ./lib_src/generate.c:75:\n", + "1 0.000051 ./lib_src/build_real.c:50:\n", + "2 0.000091 ./lib_src/cov_value.c:60:\n", + "3 0.000021 ./lib_src/length.c:47:\n", + "4 0.000017 ./lib_src/clean_real.c:48:\n", + "5 0.000068 ./lib_src/gasdev.c:43:\n", + "6 0.082331 ./lib_src/covariance.c:86:\n", + "7 0.000713 ./lib_src/gasdev.c:35:\n", + "8 0.000101 ./lib_src/ran2.c:68:\n", + "9 0.000047 ./lib_src/prebuild_gwn.c:57:\n", + "10 0.236221 ./lib_src/Py_kgeneration.c:74:\n", + "11 0.084618 ./lib_src/fftma2.c:106:\n", + "12 0.000007 ./lib_src/maxfactor.c:42:\n", + "13 0.000835 ./lib_src/fourt.c:593:\n", + "14 0.000086 ./lib_src/cgrid.c:50:\n", + "15 0.000026 ./lib_src/Py_getvalues.c:157:" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data = {}\n", + "with open(\"log_16.txt\") as log_file:\n", + " lines = log_file.readlines()\n", + " for line in lines:\n", + " if \"ELAPSED\" in line:\n", + " split_line = line.split()\n", + " idx_elapsed = split_line.index(\"ELAPSED\") + 2\n", + " function_name, elapsed = split_line[2], float(split_line[idx_elapsed])\n", + " if function_name not in data or (function_name in data and elapsed > data[function_name]):\n", + " data[function_name] = elapsed\n", + "new_data = {\"function_name\": data.keys(), \"elapsed_time (seconds)\": data.values()}\n", + " \n", + "df = pd.DataFrame(new_data) \n", + "\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.18" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/fftma_module/gen/analysis.ipynb b/fftma_module/gen/analysis.ipynb index 1182e74..1daea73 100644 --- a/fftma_module/gen/analysis.ipynb +++ b/fftma_module/gen/analysis.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -11,17 +11,21 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 45, "metadata": {}, + "outputs": [], + "source": [ + "def get_function_name(function_name):\n", + " return function_name[10:].rsplit(\".c\")[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": { + "scrolled": true + }, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['12:30:24', 'INFO', './lib_src/generate.c:61:', 'TOTAL', 'VIRTUAL', 'MEM', '=', '7683.0', 'MB,', 'USED', 'VIRTUAL', 'MEM', '=', '304.2', 'MB,', 'USED', 'VIRTUAL', 'MEM', 'BY', 'CURRENT', 'PROCESS', '=', '1072', 'MB']\n", - "['12:30:24', 'INFO', './lib_src/Py_kgeneration.c:64:', 'TOTAL', 'VIRTUAL', 'MEM', '=', '7683.0', 'MB,', 'USED', 'VIRTUAL', 'MEM', '=', '303.6', 'MB,', 'USED', 'VIRTUAL', 'MEM', 'BY', 'CURRENT', 'PROCESS', '=', '1072', 'MB']\n" - ] - }, { "data": { "text/html": [ @@ -43,133 +47,142 @@ " \n", " \n", " \n", - " function_name\n", " elapsed_time\n", " executions\n", + " function_name\n", + " used_virtual_mem\n", " \n", " \n", " \n", " \n", " 0\n", - " ./lib_src/Py_getvalues.c:157:\n", - " 0.000007\n", + " 0.000456\n", " 1\n", + " cgrid\n", + " 2877.4\n", " \n", " \n", " 1\n", - " ./lib_src/ran2.c:68:\n", - " 0.000004\n", - " 142\n", + " 0.180264\n", + " 1\n", + " Py_kgeneration\n", + " 2877.9\n", " \n", " \n", " 2\n", - " ./lib_src/gasdev.c:35:\n", - " 0.000023\n", - " 151\n", + " 0.001184\n", + " 512\n", + " gasdev\n", + " 2882.6\n", " \n", " \n", " 3\n", - " ./lib_src/gasdev.c:43:\n", - " 0.000003\n", - " 202\n", + " 0.000081\n", + " 700\n", + " cov_value\n", + " 2877.9\n", " \n", " \n", " 4\n", - " ./lib_src/generate.c:75:\n", - " 0.010957\n", + " 0.000015\n", " 1\n", + " clean_real\n", + " 2877.9\n", " \n", " \n", " 5\n", - " ./lib_src/maxfactor.c:42:\n", - " 0.000004\n", + " 0.077514\n", " 1\n", + " covariance\n", + " 2877.9\n", " \n", " \n", " 6\n", - " ./lib_src/length.c:47:\n", - " 0.000013\n", + " 0.079166\n", " 1\n", + " fftma2\n", + " 2877.9\n", " \n", " \n", " 7\n", - " ./lib_src/cgrid.c:50:\n", - " 0.000060\n", + " 0.100123\n", " 1\n", + " generate\n", + " 2877.4\n", " \n", " \n", " 8\n", - " ./lib_src/cov_value.c:60:\n", - " 0.000011\n", - " 76\n", + " 0.000098\n", + " 3\n", + " length\n", + " 2877.4\n", " \n", " \n", " 9\n", - " ./lib_src/covariance.c:86:\n", - " 0.012162\n", + " 0.000034\n", " 1\n", + " Py_getvalues\n", + " 0.0\n", " \n", " \n", " 10\n", - " ./lib_src/fourt.c:593:\n", - " 0.000065\n", - " 2\n", + " 0.000037\n", + " 702\n", + " ran2\n", + " 2882.6\n", " \n", " \n", " 11\n", - " ./lib_src/prebuild_gwn.c:57:\n", - " 0.000009\n", - " 1\n", + " 0.000193\n", + " 3\n", + " fourt\n", + " 2877.9\n", " \n", " \n", " 12\n", - " ./lib_src/build_real.c:50:\n", - " 0.000019\n", + " 0.000033\n", " 1\n", + " build_real\n", + " 2877.9\n", " \n", " \n", " 13\n", - " ./lib_src/clean_real.c:48:\n", - " 0.000007\n", + " 0.000034\n", " 1\n", + " prebuild_gwn\n", + " 2877.9\n", " \n", " \n", " 14\n", - " ./lib_src/fftma2.c:106:\n", - " 0.012503\n", - " 1\n", - " \n", - " \n", - " 15\n", - " ./lib_src/Py_kgeneration.c:74:\n", - " 0.023829\n", - " 1\n", + " 0.000010\n", + " 3\n", + " maxfactor\n", + " 2877.4\n", " \n", " \n", "\n", "" ], "text/plain": [ - " function_name elapsed_time executions\n", - "0 ./lib_src/Py_getvalues.c:157: 0.000007 1\n", - "1 ./lib_src/ran2.c:68: 0.000004 142\n", - "2 ./lib_src/gasdev.c:35: 0.000023 151\n", - "3 ./lib_src/gasdev.c:43: 0.000003 202\n", - "4 ./lib_src/generate.c:75: 0.010957 1\n", - "5 ./lib_src/maxfactor.c:42: 0.000004 1\n", - "6 ./lib_src/length.c:47: 0.000013 1\n", - "7 ./lib_src/cgrid.c:50: 0.000060 1\n", - "8 ./lib_src/cov_value.c:60: 0.000011 76\n", - "9 ./lib_src/covariance.c:86: 0.012162 1\n", - "10 ./lib_src/fourt.c:593: 0.000065 2\n", - "11 ./lib_src/prebuild_gwn.c:57: 0.000009 1\n", - "12 ./lib_src/build_real.c:50: 0.000019 1\n", - "13 ./lib_src/clean_real.c:48: 0.000007 1\n", - "14 ./lib_src/fftma2.c:106: 0.012503 1\n", - "15 ./lib_src/Py_kgeneration.c:74: 0.023829 1" + " elapsed_time executions function_name used_virtual_mem\n", + "0 0.000456 1 cgrid 2877.4\n", + "1 0.180264 1 Py_kgeneration 2877.9\n", + "2 0.001184 512 gasdev 2882.6\n", + "3 0.000081 700 cov_value 2877.9\n", + "4 0.000015 1 clean_real 2877.9\n", + "5 0.077514 1 covariance 2877.9\n", + "6 0.079166 1 fftma2 2877.9\n", + "7 0.100123 1 generate 2877.4\n", + "8 0.000098 3 length 2877.4\n", + "9 0.000034 1 Py_getvalues 0.0\n", + "10 0.000037 702 ran2 2882.6\n", + "11 0.000193 3 fourt 2877.9\n", + "12 0.000033 1 build_real 2877.9\n", + "13 0.000034 1 prebuild_gwn 2877.9\n", + "14 0.000010 3 maxfactor 2877.4" ] }, - "execution_count": 3, + "execution_count": 54, "metadata": {}, "output_type": "execute_result" } @@ -182,17 +195,19 @@ " for line in lines:\n", " if \"MEM\" in line:\n", " split_line = line.split()\n", - " print(split_line)\n", + " idx_used_mem = split_line.index(\"USED\") + 4\n", + " function_name, used_virtual_mem = get_function_name(split_line[2]), float(split_line[idx_used_mem])\n", + " val = data.get(function_name, (0, 0, used_virtual_mem))\n", + " data[function_name] = (val[0], val[1], max(used_virtual_mem, val[2])) \n", " if \"ELAPSED\" in line:\n", " split_line = line.split()\n", " idx_elapsed = split_line.index(\"ELAPSED\") + 2\n", - " function_name, elapsed = split_line[2], float(split_line[idx_elapsed])\n", - " if function_name not in data or (function_name in data and elapsed > data[function_name][0]):\n", - " val = data.get(function_name, (elapsed, 0))\n", - " data[function_name] = (val[0], val[1] + 1)\n", + " function_name, elapsed = get_function_name(split_line[2]), float(split_line[idx_elapsed])\n", + " val = data.get(function_name, (elapsed, 0, 0))\n", + " data[function_name] = (max(elapsed, val[0]), val[1] + 1, val[2])\n", "\n", "values = data.values()\n", - "new_data = {\"function_name\": data.keys(), \"elapsed_time\": map(lambda x: x[0], values), \"executions\": map(lambda x: x[1], values)}\n", + "new_data = {\"function_name\": data.keys(), \"elapsed_time\": map(lambda x: x[0], values), \"executions\": map(lambda x: x[1], values), \"used_virtual_mem\": map(lambda x: x[2], values)}\n", " \n", "df = pd.DataFrame(new_data) \n", "\n", @@ -201,7 +216,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 55, "metadata": {}, "outputs": [], "source": [ @@ -210,7 +225,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 56, "metadata": {}, "outputs": [ { @@ -234,150 +249,158 @@ " \n", " \n", " \n", - " function_name\n", " elapsed_time\n", " executions\n", + " function_name\n", + " used_virtual_mem\n", " total_time\n", " \n", " \n", " \n", " \n", - " 15\n", - " ./lib_src/Py_kgeneration.c:74:\n", - " 0.023829\n", - " 1\n", - " 0.023829\n", - " \n", - " \n", - " 14\n", - " ./lib_src/fftma2.c:106:\n", - " 0.012503\n", + " 1\n", + " 0.180264\n", " 1\n", - " 0.012503\n", + " Py_kgeneration\n", + " 2877.9\n", + " 0.180264\n", " \n", " \n", - " 9\n", - " ./lib_src/covariance.c:86:\n", - " 0.012162\n", + " 7\n", + " 0.100123\n", " 1\n", - " 0.012162\n", + " generate\n", + " 2877.4\n", + " 0.100123\n", " \n", " \n", - " 4\n", - " ./lib_src/generate.c:75:\n", - " 0.010957\n", + " 6\n", + " 0.079166\n", " 1\n", - " 0.010957\n", + " fftma2\n", + " 2877.9\n", + " 0.079166\n", " \n", " \n", - " 10\n", - " ./lib_src/fourt.c:593:\n", - " 0.000065\n", - " 2\n", - " 0.000130\n", - " \n", - " \n", - " 7\n", - " ./lib_src/cgrid.c:50:\n", - " 0.000060\n", + " 5\n", + " 0.077514\n", " 1\n", - " 0.000060\n", + " covariance\n", + " 2877.9\n", + " 0.077514\n", " \n", " \n", " 2\n", - " ./lib_src/gasdev.c:35:\n", - " 0.000023\n", - " 151\n", - " 0.003473\n", + " 0.001184\n", + " 512\n", + " gasdev\n", + " 2882.6\n", + " 0.606208\n", " \n", " \n", - " 12\n", - " ./lib_src/build_real.c:50:\n", - " 0.000019\n", + " 0\n", + " 0.000456\n", " 1\n", - " 0.000019\n", + " cgrid\n", + " 2877.4\n", + " 0.000456\n", " \n", " \n", - " 6\n", - " ./lib_src/length.c:47:\n", - " 0.000013\n", - " 1\n", - " 0.000013\n", + " 11\n", + " 0.000193\n", + " 3\n", + " fourt\n", + " 2877.9\n", + " 0.000579\n", " \n", " \n", " 8\n", - " ./lib_src/cov_value.c:60:\n", - " 0.000011\n", - " 76\n", - " 0.000836\n", + " 0.000098\n", + " 3\n", + " length\n", + " 2877.4\n", + " 0.000294\n", " \n", " \n", - " 11\n", - " ./lib_src/prebuild_gwn.c:57:\n", - " 0.000009\n", - " 1\n", - " 0.000009\n", + " 3\n", + " 0.000081\n", + " 700\n", + " cov_value\n", + " 2877.9\n", + " 0.056700\n", " \n", " \n", - " 0\n", - " ./lib_src/Py_getvalues.c:157:\n", - " 0.000007\n", + " 10\n", + " 0.000037\n", + " 702\n", + " ran2\n", + " 2882.6\n", + " 0.025974\n", + " \n", + " \n", + " 9\n", + " 0.000034\n", " 1\n", - " 0.000007\n", + " Py_getvalues\n", + " 0.0\n", + " 0.000034\n", " \n", " \n", " 13\n", - " ./lib_src/clean_real.c:48:\n", - " 0.000007\n", + " 0.000034\n", " 1\n", - " 0.000007\n", + " prebuild_gwn\n", + " 2877.9\n", + " 0.000034\n", " \n", " \n", - " 1\n", - " ./lib_src/ran2.c:68:\n", - " 0.000004\n", - " 142\n", - " 0.000568\n", + " 12\n", + " 0.000033\n", + " 1\n", + " build_real\n", + " 2877.9\n", + " 0.000033\n", " \n", " \n", - " 5\n", - " ./lib_src/maxfactor.c:42:\n", - " 0.000004\n", + " 4\n", + " 0.000015\n", " 1\n", - " 0.000004\n", + " clean_real\n", + " 2877.9\n", + " 0.000015\n", " \n", " \n", - " 3\n", - " ./lib_src/gasdev.c:43:\n", - " 0.000003\n", - " 202\n", - " 0.000606\n", + " 14\n", + " 0.000010\n", + " 3\n", + " maxfactor\n", + " 2877.4\n", + " 0.000030\n", " \n", " \n", "\n", "" ], "text/plain": [ - " function_name elapsed_time executions total_time\n", - "15 ./lib_src/Py_kgeneration.c:74: 0.023829 1 0.023829\n", - "14 ./lib_src/fftma2.c:106: 0.012503 1 0.012503\n", - "9 ./lib_src/covariance.c:86: 0.012162 1 0.012162\n", - "4 ./lib_src/generate.c:75: 0.010957 1 0.010957\n", - "10 ./lib_src/fourt.c:593: 0.000065 2 0.000130\n", - "7 ./lib_src/cgrid.c:50: 0.000060 1 0.000060\n", - "2 ./lib_src/gasdev.c:35: 0.000023 151 0.003473\n", - "12 ./lib_src/build_real.c:50: 0.000019 1 0.000019\n", - "6 ./lib_src/length.c:47: 0.000013 1 0.000013\n", - "8 ./lib_src/cov_value.c:60: 0.000011 76 0.000836\n", - "11 ./lib_src/prebuild_gwn.c:57: 0.000009 1 0.000009\n", - "0 ./lib_src/Py_getvalues.c:157: 0.000007 1 0.000007\n", - "13 ./lib_src/clean_real.c:48: 0.000007 1 0.000007\n", - "1 ./lib_src/ran2.c:68: 0.000004 142 0.000568\n", - "5 ./lib_src/maxfactor.c:42: 0.000004 1 0.000004\n", - "3 ./lib_src/gasdev.c:43: 0.000003 202 0.000606" + " elapsed_time executions function_name used_virtual_mem total_time\n", + "1 0.180264 1 Py_kgeneration 2877.9 0.180264\n", + "7 0.100123 1 generate 2877.4 0.100123\n", + "6 0.079166 1 fftma2 2877.9 0.079166\n", + "5 0.077514 1 covariance 2877.9 0.077514\n", + "2 0.001184 512 gasdev 2882.6 0.606208\n", + "0 0.000456 1 cgrid 2877.4 0.000456\n", + "11 0.000193 3 fourt 2877.9 0.000579\n", + "8 0.000098 3 length 2877.4 0.000294\n", + "3 0.000081 700 cov_value 2877.9 0.056700\n", + "10 0.000037 702 ran2 2882.6 0.025974\n", + "9 0.000034 1 Py_getvalues 0.0 0.000034\n", + "13 0.000034 1 prebuild_gwn 2877.9 0.000034\n", + "12 0.000033 1 build_real 2877.9 0.000033\n", + "4 0.000015 1 clean_real 2877.9 0.000015\n", + "14 0.000010 3 maxfactor 2877.4 0.000030" ] }, - "execution_count": 5, + "execution_count": 56, "metadata": {}, "output_type": "execute_result" } @@ -388,7 +411,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 57, "metadata": {}, "outputs": [ { @@ -412,156 +435,350 @@ " \n", " \n", " \n", - " function_name\n", " elapsed_time\n", " executions\n", + " function_name\n", + " used_virtual_mem\n", " total_time\n", " \n", " \n", " \n", " \n", - " 15\n", - " ./lib_src/Py_kgeneration.c:74:\n", - " 0.023829\n", + " 2\n", + " 0.001184\n", + " 512\n", + " gasdev\n", + " 2882.6\n", + " 0.606208\n", + " \n", + " \n", + " 1\n", + " 0.180264\n", " 1\n", - " 0.023829\n", + " Py_kgeneration\n", + " 2877.9\n", + " 0.180264\n", " \n", " \n", - " 14\n", - " ./lib_src/fftma2.c:106:\n", - " 0.012503\n", + " 7\n", + " 0.100123\n", " 1\n", - " 0.012503\n", + " generate\n", + " 2877.4\n", + " 0.100123\n", " \n", " \n", - " 9\n", - " ./lib_src/covariance.c:86:\n", - " 0.012162\n", + " 6\n", + " 0.079166\n", " 1\n", - " 0.012162\n", + " fftma2\n", + " 2877.9\n", + " 0.079166\n", " \n", " \n", - " 4\n", - " ./lib_src/generate.c:75:\n", - " 0.010957\n", + " 5\n", + " 0.077514\n", " 1\n", - " 0.010957\n", + " covariance\n", + " 2877.9\n", + " 0.077514\n", " \n", " \n", - " 2\n", - " ./lib_src/gasdev.c:35:\n", - " 0.000023\n", - " 151\n", - " 0.003473\n", + " 3\n", + " 0.000081\n", + " 700\n", + " cov_value\n", + " 2877.9\n", + " 0.056700\n", + " \n", + " \n", + " 10\n", + " 0.000037\n", + " 702\n", + " ran2\n", + " 2882.6\n", + " 0.025974\n", + " \n", + " \n", + " 11\n", + " 0.000193\n", + " 3\n", + " fourt\n", + " 2877.9\n", + " 0.000579\n", + " \n", + " \n", + " 0\n", + " 0.000456\n", + " 1\n", + " cgrid\n", + " 2877.4\n", + " 0.000456\n", " \n", " \n", " 8\n", - " ./lib_src/cov_value.c:60:\n", - " 0.000011\n", - " 76\n", - " 0.000836\n", + " 0.000098\n", + " 3\n", + " length\n", + " 2877.4\n", + " 0.000294\n", " \n", " \n", - " 3\n", - " ./lib_src/gasdev.c:43:\n", - " 0.000003\n", - " 202\n", - " 0.000606\n", + " 9\n", + " 0.000034\n", + " 1\n", + " Py_getvalues\n", + " 0.0\n", + " 0.000034\n", " \n", " \n", - " 1\n", - " ./lib_src/ran2.c:68:\n", - " 0.000004\n", - " 142\n", - " 0.000568\n", + " 13\n", + " 0.000034\n", + " 1\n", + " prebuild_gwn\n", + " 2877.9\n", + " 0.000034\n", + " \n", + " \n", + " 12\n", + " 0.000033\n", + " 1\n", + " build_real\n", + " 2877.9\n", + " 0.000033\n", + " \n", + " \n", + " 14\n", + " 0.000010\n", + " 3\n", + " maxfactor\n", + " 2877.4\n", + " 0.000030\n", + " \n", + " \n", + " 4\n", + " 0.000015\n", + " 1\n", + " clean_real\n", + " 2877.9\n", + " 0.000015\n", + " \n", + " \n", + "\n", + "" + ], + "text/plain": [ + " elapsed_time executions function_name used_virtual_mem total_time\n", + "2 0.001184 512 gasdev 2882.6 0.606208\n", + "1 0.180264 1 Py_kgeneration 2877.9 0.180264\n", + "7 0.100123 1 generate 2877.4 0.100123\n", + "6 0.079166 1 fftma2 2877.9 0.079166\n", + "5 0.077514 1 covariance 2877.9 0.077514\n", + "3 0.000081 700 cov_value 2877.9 0.056700\n", + "10 0.000037 702 ran2 2882.6 0.025974\n", + "11 0.000193 3 fourt 2877.9 0.000579\n", + "0 0.000456 1 cgrid 2877.4 0.000456\n", + "8 0.000098 3 length 2877.4 0.000294\n", + "9 0.000034 1 Py_getvalues 0.0 0.000034\n", + "13 0.000034 1 prebuild_gwn 2877.9 0.000034\n", + "12 0.000033 1 build_real 2877.9 0.000033\n", + "14 0.000010 3 maxfactor 2877.4 0.000030\n", + "4 0.000015 1 clean_real 2877.9 0.000015" + ] + }, + "execution_count": 57, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.sort_values(by=[\"total_time\"], ascending=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", "
elapsed_timeexecutionsfunction_nameused_virtual_memtotal_time
20.001184512gasdev2882.60.606208
10./lib_src/fourt.c:593:0.00006520.0001300.000037702ran22882.60.025974
7./lib_src/cgrid.c:50:0.00006010.18026410.000060Py_kgeneration2877.90.180264
12./lib_src/build_real.c:50:0.00001930.000081700cov_value2877.90.056700
40.00001510.000019clean_real2877.90.000015
50.0775141covariance2877.90.077514
6./lib_src/length.c:47:0.0000130.07916610.000013fftma22877.90.079166
11./lib_src/prebuild_gwn.c:57:0.0000090.0001933fourt2877.90.000579
120.00003310.000009build_real2877.90.000033
130.0000341prebuild_gwn2877.90.000034
0./lib_src/Py_getvalues.c:157:0.0000070.00045610.000007cgrid2877.40.000456
13./lib_src/clean_real.c:48:0.00000770.10012310.000007generate2877.40.100123
5./lib_src/maxfactor.c:42:0.00000480.0000983length2877.40.000294
140.0000103maxfactor2877.40.000030
90.00003410.000004Py_getvalues0.00.000034
\n", "
" ], "text/plain": [ - " function_name elapsed_time executions total_time\n", - "15 ./lib_src/Py_kgeneration.c:74: 0.023829 1 0.023829\n", - "14 ./lib_src/fftma2.c:106: 0.012503 1 0.012503\n", - "9 ./lib_src/covariance.c:86: 0.012162 1 0.012162\n", - "4 ./lib_src/generate.c:75: 0.010957 1 0.010957\n", - "2 ./lib_src/gasdev.c:35: 0.000023 151 0.003473\n", - "8 ./lib_src/cov_value.c:60: 0.000011 76 0.000836\n", - "3 ./lib_src/gasdev.c:43: 0.000003 202 0.000606\n", - "1 ./lib_src/ran2.c:68: 0.000004 142 0.000568\n", - "10 ./lib_src/fourt.c:593: 0.000065 2 0.000130\n", - "7 ./lib_src/cgrid.c:50: 0.000060 1 0.000060\n", - "12 ./lib_src/build_real.c:50: 0.000019 1 0.000019\n", - "6 ./lib_src/length.c:47: 0.000013 1 0.000013\n", - "11 ./lib_src/prebuild_gwn.c:57: 0.000009 1 0.000009\n", - "0 ./lib_src/Py_getvalues.c:157: 0.000007 1 0.000007\n", - "13 ./lib_src/clean_real.c:48: 0.000007 1 0.000007\n", - "5 ./lib_src/maxfactor.c:42: 0.000004 1 0.000004" + " elapsed_time executions function_name used_virtual_mem total_time\n", + "2 0.001184 512 gasdev 2882.6 0.606208\n", + "10 0.000037 702 ran2 2882.6 0.025974\n", + "1 0.180264 1 Py_kgeneration 2877.9 0.180264\n", + "3 0.000081 700 cov_value 2877.9 0.056700\n", + "4 0.000015 1 clean_real 2877.9 0.000015\n", + "5 0.077514 1 covariance 2877.9 0.077514\n", + "6 0.079166 1 fftma2 2877.9 0.079166\n", + "11 0.000193 3 fourt 2877.9 0.000579\n", + "12 0.000033 1 build_real 2877.9 0.000033\n", + "13 0.000034 1 prebuild_gwn 2877.9 0.000034\n", + "0 0.000456 1 cgrid 2877.4 0.000456\n", + "7 0.100123 1 generate 2877.4 0.100123\n", + "8 0.000098 3 length 2877.4 0.000294\n", + "14 0.000010 3 maxfactor 2877.4 0.000030\n", + "9 0.000034 1 Py_getvalues 0.0 0.000034" ] }, - "execution_count": 7, + "execution_count": 58, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df.sort_values(by=[\"total_time\"], ascending=False)" + "df.sort_values(by=[\"used_virtual_mem\"], ascending=False)" ] }, { @@ -716,14 +933,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 3 + "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.9" + "pygments_lexer": "ipython2", + "version": "2.7.18" } }, "nbformat": 4, diff --git a/fftma_module/gen/lib_src/build_real.c b/fftma_module/gen/lib_src/build_real.c index 35c83df..8e541c2 100644 --- a/fftma_module/gen/lib_src/build_real.c +++ b/fftma_module/gen/lib_src/build_real.c @@ -23,6 +23,9 @@ void build_real(int n[3], int NTOT, double* covar, double* realization, double* ireal) { clock_t t = clock(); + double* used_ram_t0 = malloc(sizeof(double)); + getVirtualMemUsed(used_ram_t0); + int i, j, k, maille1; double temp; log_info("RESULT = in progress, NTOT = %d, covar = %f, n[0] = %d, n[1] = %d, n[2] = %d", NTOT, *covar, n[0], n[1], n[2]); @@ -47,17 +50,11 @@ void build_real(int n[3], int NTOT, double* covar, double* realization, double* t = clock() - t; double time_taken = ((double)t)/CLOCKS_PER_SEC; // calculate the elapsed time - double* total_ram = malloc(sizeof(double)); - getTotalVirtualMem(total_ram); - - double* used_ram = malloc(sizeof(double)); - getVirtualMemUsed(used_ram); - - log_info("TOTAL VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM BY CURRENT PROCESS = %d MB", - *total_ram, *used_ram, getVirtualMemUsedByCurrentProcess()); + double* used_ram_tf = malloc(sizeof(double)); + getVirtualMemUsed(used_ram_tf); - free(total_ram); - free(used_ram); + log_info("RESULT = success, realization = %f, ireal = %f, ELAPSED = %f seconds, USED VIRTUAL MEM = %5.1f MB, DIF USED VIRTUAL MEM = %5.1f MB", *realization, *ireal, time_taken, *used_ram_tf, *used_ram_tf - *used_ram_t0); - log_info("RESULT = success, realization = %f, ireal = %f, ELAPSED = %f seconds", *realization, *ireal, time_taken); + free(used_ram_t0); + free(used_ram_tf); } diff --git a/fftma_module/gen/lib_src/fftma2.c b/fftma_module/gen/lib_src/fftma2.c index 03b06d5..c35ac00 100644 --- a/fftma_module/gen/lib_src/fftma2.c +++ b/fftma_module/gen/lib_src/fftma2.c @@ -25,6 +25,9 @@ void FFTMA2(struct vario_mod variogram, struct grid_mod grid, int n[3], struct realization_mod* realin, struct realization_mod* realout) { clock_t t = clock(); + + double* used_ram_t0 = malloc(sizeof(double)); + getVirtualMemUsed(used_ram_t0); log_info("RESULT = in progress"); @@ -86,6 +89,9 @@ void FFTMA2(struct vario_mod variogram, struct grid_mod grid, int n[3], struct r /* build realization in spectral domain */ build_real(n, NTOT, covar, realization, ireal); + double* used_ram_tf = malloc(sizeof(double)); + getVirtualMemUsed(used_ram_tf); + free(covar); /*backward fourier transform*/ @@ -98,22 +104,11 @@ void FFTMA2(struct vario_mod variogram, struct grid_mod grid, int n[3], struct r /*output realization*/ clean_real(realin, n, grid, realization, realout); - free(realization); - t = clock() - t; double time_taken = ((double)t)/CLOCKS_PER_SEC; // calculate the elapsed time - double* total_ram = malloc(sizeof(double)); - getTotalVirtualMem(total_ram); - - double* used_ram = malloc(sizeof(double)); - getVirtualMemUsed(used_ram); - - log_info("TOTAL VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM = %5.1f MB, USED VIRTUAL MEM BY CURRENT PROCESS = %d MB", - *total_ram, *used_ram, getVirtualMemUsedByCurrentProcess()); - - free(total_ram); - free(used_ram); + log_info("RESULT = success, NTOT = %d, NMAX = %d, NDIM = %d, ntot = %d, nmax = %d, NXYZ = %d, nxyz = %d, ELAPSED = %f seconds, USED VIRTUAL MEM = %5.1f MB, DIF USED VIRTUAL MEM = %5.1f MB", NTOT, NMAX, NDIM, ntot, nmax, NXYZ, nxyz, time_taken, *used_ram_tf, *used_ram_tf - *used_ram_t0); - log_info("RESULT = success, NTOT = %d, NMAX = %d, NDIM = %d, ntot = %d, nmax = %d, NXYZ = %d, nxyz = %d, ELAPSED = %f seconds", NTOT, NMAX, NDIM, ntot, nmax, NXYZ, nxyz, time_taken); + free(used_ram_t0); + free(used_ram_tf); } diff --git a/fftma_module/gen/save_logs.sh b/fftma_module/gen/save_logs.sh index 32651ff..45f6d53 100755 --- a/fftma_module/gen/save_logs.sh +++ b/fftma_module/gen/save_logs.sh @@ -1 +1 @@ -ENV=true python3 test.py 2>&1 | tee log_$1.txt \ No newline at end of file +ENV=true python3 test.py $1 2>&1 | tee log_$1.txt \ No newline at end of file diff --git a/fftma_module/gen/test.py b/fftma_module/gen/test.py index 2799ff3..7ca3b6d 100644 --- a/fftma_module/gen/test.py +++ b/fftma_module/gen/test.py @@ -1,8 +1,9 @@ from FFTMA import gen import numpy as np import gc +import sys -N=64 +N=int(sys.argv[1]) nx, ny, nz = N,N,N dx, dy, dz = 1.0, 1.0, 1.0