From 0f8cbc686b80859652f28bc94f6b94e2f361844f Mon Sep 17 00:00:00 2001 From: chortas Date: Sat, 18 Dec 2021 18:29:25 -0300 Subject: [PATCH] Save logs with information and reorganize notebook --- fftma_module/gen/analysis.ipynb | 1680 ++++++++++++++++++------------- fftma_module/gen/lib_src/log.c | 4 + fftma_module/gen/save_logs.sh | 3 +- 3 files changed, 1008 insertions(+), 679 deletions(-) diff --git a/fftma_module/gen/analysis.ipynb b/fftma_module/gen/analysis.ipynb index 60ed061..a1f0cb1 100644 --- a/fftma_module/gen/analysis.ipynb +++ b/fftma_module/gen/analysis.ipynb @@ -1,8 +1,15 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Análisis de la etapa de generación de medios" + ] + }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -11,9 +18,23 @@ "import numpy as np " ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Armado del dataset\n", + "\n", + "En este paso parsearemos los archivos para obtener estadísticas sobre el tiempo que tarda cada ejecución de una función, sobre la memoria usada, el uso de CPU (TODO). Con esto buscamos identificar:\n", + "- Qué funciones son las que consumen mayor cantidad de memoria\n", + "- Qué funciones son las que tienen un mayor tiempo de procesamiento\n", + "- Qué funciones son las que son invocadas una mayor cantidad de veces\n", + "\n", + "Una vez identificados estos puntos de análisis podemos proponer soluciones para mejorar estas estadísticas." + ] + }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -21,12 +42,64 @@ " return function_name[10:].rsplit(\".c\")[0]" ] }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "relations = {\n", + " \"generate\": [\"gasdev\"],\n", + " \"fftma2\": [\"covariance\", \"fourt\", \"prebuild_gwn\"]\n", + "}" + ] + }, { "cell_type": "code", "execution_count": 28, - "metadata": { - "scrolled": true - }, + "metadata": {}, + "outputs": [], + "source": [ + "def analyze(file_name):\n", + " data = []\n", + "\n", + " with open(file_name) as log_file: \n", + " lines = log_file.readlines()\n", + " print(len(lines))\n", + " for line in lines:\n", + " row = {}\n", + " split_line = line.split()\n", + " if \"USED\" not in split_line or \"ELAPSED\" not in split_line:\n", + " continue\n", + " idx_used_mem = split_line.index(\"USED\") + 4\n", + " idx_elapsed = split_line.index(\"ELAPSED\") + 2\n", + " \n", + " function_name = get_function_name(split_line[2])\n", + " used_virtual_mem = float(split_line[idx_used_mem])\n", + " elapsed = float(split_line[idx_elapsed].rsplit(\",\")[0])\n", + "\n", + " # TODO: add CPU\n", + " row[\"function\"] = function_name\n", + " row[\"memory\"] = used_virtual_mem \n", + " row[\"time\"] = elapsed\n", + " data.append(row)\n", + " \n", + " df = pd.DataFrame(data)\n", + " df_grouped = df.groupby(['function']).agg({'time': ['min', 'max', 'mean', 'sum', 'count'], 'memory': ['min', 'max', 'median']})\n", + " return df_grouped.sort_values(by=('time', 'sum'), ascending=False) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## N = 8" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, "outputs": [ { "data": { @@ -41,193 +114,260 @@ " vertical-align: top;\n", " }\n", "\n", - " .dataframe thead th {\n", + " .dataframe thead tr th {\n", + " text-align: left;\n", + " }\n", + "\n", + " .dataframe thead tr:last-of-type th {\n", " text-align: right;\n", " }\n", "\n", "\n", " \n", - " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", " \n", " \n", "
function_nameelapsed_timeexecutionsused_virtual_memmemorytime
0Py_getvalues0.00000710.0minmaxmedianminmaxmeansumcount
1ran20.00006541552240.5function
2gasdev0.00038332768240.5Py_kgeneration2.92.92.90.0918050.0918050.0918050.0918051
3generate2.055015generate2.92.92.90.0817070.0817070.0817070.0817071219.2
4maxfactor0.0000055219.2gasdev0.00.50.00.0000000.0056740.0000930.047828512
5length0.0000713219.2fftma20.00.00.00.0079420.0079420.0079420.0079421
6cgrid0.000265covariance0.00.00.00.0074920.0074920.0074920.0074921219.2
7cov_value0.00015624624234.7ran20.00.50.00.0000000.0000180.0000020.001708702
8covariance0.8397371228.0cov_value0.00.00.00.0000000.0000300.0000010.000707700
9fourt0.003451fourt0.00.00.00.0000790.0001070.0000910.0002743227.9
10prebuild_gwn0.000138cgrid0.00.00.00.0000670.0000670.0000670.0000671227.4
11build_real0.0004241227.4length0.00.00.00.0000080.0000080.0000080.0000243
12clean_real0.000158build_real0.00.00.00.0000090.0000090.0000090.0000091227.4
13fftma20.849271prebuild_gwn0.00.00.00.0000070.0000070.0000070.0000071227.4
14Py_kgeneration2.904626maxfactor0.00.00.00.0000010.0000010.0000010.0000033
clean_real0.00.00.00.0000020.0000020.0000020.0000021227.4
\n", "" ], "text/plain": [ - " function_name elapsed_time executions used_virtual_mem\n", - "0 Py_getvalues 0.000007 1 0.0\n", - "1 ran2 0.000065 41552 240.5\n", - "2 gasdev 0.000383 32768 240.5\n", - "3 generate 2.055015 1 219.2\n", - "4 maxfactor 0.000005 5 219.2\n", - "5 length 0.000071 3 219.2\n", - "6 cgrid 0.000265 1 219.2\n", - "7 cov_value 0.000156 24624 234.7\n", - "8 covariance 0.839737 1 228.0\n", - "9 fourt 0.003451 3 227.9\n", - "10 prebuild_gwn 0.000138 1 227.4\n", - "11 build_real 0.000424 1 227.4\n", - "12 clean_real 0.000158 1 227.4\n", - "13 fftma2 0.849271 1 227.4\n", - "14 Py_kgeneration 2.904626 1 227.4" + " memory time \\\n", + " min max median min max mean sum \n", + "function \n", + "Py_kgeneration 2.9 2.9 2.9 0.091805 0.091805 0.091805 0.091805 \n", + "generate 2.9 2.9 2.9 0.081707 0.081707 0.081707 0.081707 \n", + "gasdev 0.0 0.5 0.0 0.000000 0.005674 0.000093 0.047828 \n", + "fftma2 0.0 0.0 0.0 0.007942 0.007942 0.007942 0.007942 \n", + "covariance 0.0 0.0 0.0 0.007492 0.007492 0.007492 0.007492 \n", + "ran2 0.0 0.5 0.0 0.000000 0.000018 0.000002 0.001708 \n", + "cov_value 0.0 0.0 0.0 0.000000 0.000030 0.000001 0.000707 \n", + "fourt 0.0 0.0 0.0 0.000079 0.000107 0.000091 0.000274 \n", + "cgrid 0.0 0.0 0.0 0.000067 0.000067 0.000067 0.000067 \n", + "length 0.0 0.0 0.0 0.000008 0.000008 0.000008 0.000024 \n", + "build_real 0.0 0.0 0.0 0.000009 0.000009 0.000009 0.000009 \n", + "prebuild_gwn 0.0 0.0 0.0 0.000007 0.000007 0.000007 0.000007 \n", + "maxfactor 0.0 0.0 0.0 0.000001 0.000001 0.000001 0.000003 \n", + "clean_real 0.0 0.0 0.0 0.000002 0.000002 0.000002 0.000002 \n", + "\n", + " \n", + " count \n", + "function \n", + "Py_kgeneration 1 \n", + "generate 1 \n", + "gasdev 512 \n", + "fftma2 1 \n", + "covariance 1 \n", + "ran2 702 \n", + "cov_value 700 \n", + "fourt 3 \n", + "cgrid 1 \n", + "length 3 \n", + "build_real 1 \n", + "prebuild_gwn 1 \n", + "maxfactor 3 \n", + "clean_real 1 " ] }, - "execution_count": 28, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "data = {}\n", - "with open(\"log_32.txt\") as log_file:\n", - " lines = log_file.readlines()\n", - " \n", - " for line in lines:\n", - " if \"MEM\" in line:\n", - " split_line = line.split()\n", - " idx_used_mem = split_line.index(\"USED\") + 4\n", - " function_name, used_virtual_mem = get_function_name(split_line[2]), float(split_line[idx_used_mem])\n", - " val = data.get(function_name, (0, 0, used_virtual_mem))\n", - " data[function_name] = (val[0], val[1], max(used_virtual_mem, val[2])) \n", - " if \"ELAPSED\" in line:\n", - " split_line = line.split()\n", - " idx_elapsed = split_line.index(\"ELAPSED\") + 2\n", - " function_name, elapsed = get_function_name(split_line[2]), float(split_line[idx_elapsed])\n", - " val = data.get(function_name, (elapsed, 0, 0))\n", - " data[function_name] = (max(elapsed, val[0]), val[1] + 1, val[2])\n", - "\n", - "values = data.values()\n", - "new_data = {\"function_name\": data.keys(), \"elapsed_time\": map(lambda x: x[0], values), \"executions\": map(lambda x: x[1], values), \"used_virtual_mem\": map(lambda x: x[2], values)}\n", - " \n", - "df = pd.DataFrame(new_data) \n", - "\n", - "df" + "analyze('log_8.txt')" ] }, { - "cell_type": "code", - "execution_count": 27, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "df[\"total_time\"] = df[\"elapsed_time\"] * df[\"executions\"]" + "## N = 16" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -243,172 +383,255 @@ " vertical-align: top;\n", " }\n", "\n", - " .dataframe thead th {\n", + " .dataframe thead tr th {\n", + " text-align: left;\n", + " }\n", + "\n", + " .dataframe thead tr:last-of-type th {\n", " text-align: right;\n", " }\n", "\n", "\n", " \n", - " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
function_nameelapsed_timeexecutionsused_virtual_memtotal_timememorytime
2gasdev0.00038332768240.512.550144minmaxmedianminmaxmeansumcount
7cov_value0.00015624624234.73.841344function
14Py_kgeneration2.904626Py_kgeneration4.94.94.90.3529310.3529313.529310e-010.3529311227.42.904626
1ran20.00006541552240.52.700880generate4.24.24.20.3195650.3195653.195650e-010.3195651
3generate2.0550151219.22.055015gasdev0.00.50.00.0000000.0146543.830811e-050.1569104096
13fftma20.849271fftma21.01.01.00.0312170.0312173.121700e-020.0312171227.40.849271
8covariance0.839737covariance1.01.01.00.0301040.0301043.010400e-020.0301041228.00.839737
9fourt0.0034513227.90.010353ran20.00.50.00.0000000.0000221.032080e-060.0054375268
11build_real0.0004241227.40.000424cov_value0.00.50.00.0000000.0000138.762626e-070.0031233564
6cgrid0.000265fourt0.00.00.00.0002690.0003713.073333e-040.0009223
cgrid0.00.00.00.0000470.0000474.700000e-050.0000471219.20.000265
5length0.000071length0.00.00.00.0000080.0000108.666667e-060.0000263219.20.000213
12clean_real0.000158prebuild_gwn0.00.00.00.0000230.0000232.300000e-050.0000231227.40.000158
10prebuild_gwn0.000138build_real0.00.00.00.0000210.0000212.100000e-050.0000211227.40.000138
4maxfactor0.0000055219.20.000025clean_real0.00.00.00.0000040.0000044.000000e-060.0000041
0Py_getvalues0.0000071maxfactor0.00.0000070.00.00.0000000.0000016.666667e-070.0000023
\n", "" ], "text/plain": [ - " function_name elapsed_time executions used_virtual_mem total_time\n", - "2 gasdev 0.000383 32768 240.5 12.550144\n", - "7 cov_value 0.000156 24624 234.7 3.841344\n", - "14 Py_kgeneration 2.904626 1 227.4 2.904626\n", - "1 ran2 0.000065 41552 240.5 2.700880\n", - "3 generate 2.055015 1 219.2 2.055015\n", - "13 fftma2 0.849271 1 227.4 0.849271\n", - "8 covariance 0.839737 1 228.0 0.839737\n", - "9 fourt 0.003451 3 227.9 0.010353\n", - "11 build_real 0.000424 1 227.4 0.000424\n", - "6 cgrid 0.000265 1 219.2 0.000265\n", - "5 length 0.000071 3 219.2 0.000213\n", - "12 clean_real 0.000158 1 227.4 0.000158\n", - "10 prebuild_gwn 0.000138 1 227.4 0.000138\n", - "4 maxfactor 0.000005 5 219.2 0.000025\n", - "0 Py_getvalues 0.000007 1 0.0 0.000007" + " memory time \\\n", + " min max median min max mean sum \n", + "function \n", + "Py_kgeneration 4.9 4.9 4.9 0.352931 0.352931 3.529310e-01 0.352931 \n", + "generate 4.2 4.2 4.2 0.319565 0.319565 3.195650e-01 0.319565 \n", + "gasdev 0.0 0.5 0.0 0.000000 0.014654 3.830811e-05 0.156910 \n", + "fftma2 1.0 1.0 1.0 0.031217 0.031217 3.121700e-02 0.031217 \n", + "covariance 1.0 1.0 1.0 0.030104 0.030104 3.010400e-02 0.030104 \n", + "ran2 0.0 0.5 0.0 0.000000 0.000022 1.032080e-06 0.005437 \n", + "cov_value 0.0 0.5 0.0 0.000000 0.000013 8.762626e-07 0.003123 \n", + "fourt 0.0 0.0 0.0 0.000269 0.000371 3.073333e-04 0.000922 \n", + "cgrid 0.0 0.0 0.0 0.000047 0.000047 4.700000e-05 0.000047 \n", + "length 0.0 0.0 0.0 0.000008 0.000010 8.666667e-06 0.000026 \n", + "prebuild_gwn 0.0 0.0 0.0 0.000023 0.000023 2.300000e-05 0.000023 \n", + "build_real 0.0 0.0 0.0 0.000021 0.000021 2.100000e-05 0.000021 \n", + "clean_real 0.0 0.0 0.0 0.000004 0.000004 4.000000e-06 0.000004 \n", + "maxfactor 0.0 0.0 0.0 0.000000 0.000001 6.666667e-07 0.000002 \n", + "\n", + " \n", + " count \n", + "function \n", + "Py_kgeneration 1 \n", + "generate 1 \n", + "gasdev 4096 \n", + "fftma2 1 \n", + "covariance 1 \n", + "ran2 5268 \n", + "cov_value 3564 \n", + "fourt 3 \n", + "cgrid 1 \n", + "length 3 \n", + "prebuild_gwn 1 \n", + "build_real 1 \n", + "clean_real 1 \n", + "maxfactor 3 " ] }, - "execution_count": 25, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df.sort_values(by=[\"total_time\", \"used_virtual_mem\"], ascending=False)" + "analyze('log_16.txt')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## N = 32" ] }, { @@ -429,163 +652,239 @@ " vertical-align: top;\n", " }\n", "\n", - " .dataframe thead th {\n", + " .dataframe thead tr th {\n", + " text-align: left;\n", + " }\n", + "\n", + " .dataframe thead tr:last-of-type th {\n", " text-align: right;\n", " }\n", "\n", "\n", " \n", - " \n", + " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
function_nameelapsed_timeexecutionsused_virtual_memtotal_timememorytime
1ran20.000220333450548.273.359000minmaxmedianminmaxmeansumcount
2gasdev0.000707262144548.2185.335808function
3generate20.687490Py_kgeneration2.72.72.71.1808461.1808461.180846e+001.1808461425.620.687490
4maxfactor0.0000054425.60.000020generate4.94.94.90.8400750.8400758.400750e-010.8400751
5length0.0000593425.60.000177gasdev-1.30.50.00.0000000.0032431.513522e-050.49595132768
6cgrid0.000199fftma2-2.2-2.2-2.20.3388820.3388823.388820e-010.3388821425.60.000199
7cov_value0.000158156816425.624.776928covariance-2.5-2.5-2.50.3302820.3302823.302820e-010.3302821
8covariance7.8137951313.07.813795ran2-0.30.50.00.0000000.0001039.740325e-070.04047341552
12clean_real0.0009661311.30.000966cov_value0.00.50.00.0000000.0000581.230344e-060.03029624624
14Py_kgeneration28.676072fourt0.00.30.20.0022120.0033462.646333e-030.0079393
prebuild_gwn0.00.00.00.0001780.0001781.780000e-040.0001781311.328.676072
9fourt0.0690883311.10.207264build_real0.00.00.00.0001510.0001511.510000e-040.0001511
10prebuild_gwn0.001624clean_real0.00.00.00.0001040.0001041.040000e-040.0001041308.70.001624
11build_real0.004294cgrid0.00.00.00.0000570.0000575.700000e-050.0000571308.40.004294
13fftma27.9882121308.47.988212length0.00.00.00.0000080.0000161.300000e-050.0000393
0Py_getvalues0.0000091maxfactor0.00.0000090.00.00.0000000.0000014.000000e-070.0000025
\n", "" ], "text/plain": [ - " function_name elapsed_time executions used_virtual_mem total_time\n", - "1 ran2 0.000220 333450 548.2 73.359000\n", - "2 gasdev 0.000707 262144 548.2 185.335808\n", - "3 generate 20.687490 1 425.6 20.687490\n", - "4 maxfactor 0.000005 4 425.6 0.000020\n", - "5 length 0.000059 3 425.6 0.000177\n", - "6 cgrid 0.000199 1 425.6 0.000199\n", - "7 cov_value 0.000158 156816 425.6 24.776928\n", - "8 covariance 7.813795 1 313.0 7.813795\n", - "12 clean_real 0.000966 1 311.3 0.000966\n", - "14 Py_kgeneration 28.676072 1 311.3 28.676072\n", - "9 fourt 0.069088 3 311.1 0.207264\n", - "10 prebuild_gwn 0.001624 1 308.7 0.001624\n", - "11 build_real 0.004294 1 308.4 0.004294\n", - "13 fftma2 7.988212 1 308.4 7.988212\n", - "0 Py_getvalues 0.000009 1 0.0 0.000009" + " memory time \\\n", + " min max median min max mean sum \n", + "function \n", + "Py_kgeneration 2.7 2.7 2.7 1.180846 1.180846 1.180846e+00 1.180846 \n", + "generate 4.9 4.9 4.9 0.840075 0.840075 8.400750e-01 0.840075 \n", + "gasdev -1.3 0.5 0.0 0.000000 0.003243 1.513522e-05 0.495951 \n", + "fftma2 -2.2 -2.2 -2.2 0.338882 0.338882 3.388820e-01 0.338882 \n", + "covariance -2.5 -2.5 -2.5 0.330282 0.330282 3.302820e-01 0.330282 \n", + "ran2 -0.3 0.5 0.0 0.000000 0.000103 9.740325e-07 0.040473 \n", + "cov_value 0.0 0.5 0.0 0.000000 0.000058 1.230344e-06 0.030296 \n", + "fourt 0.0 0.3 0.2 0.002212 0.003346 2.646333e-03 0.007939 \n", + "prebuild_gwn 0.0 0.0 0.0 0.000178 0.000178 1.780000e-04 0.000178 \n", + "build_real 0.0 0.0 0.0 0.000151 0.000151 1.510000e-04 0.000151 \n", + "clean_real 0.0 0.0 0.0 0.000104 0.000104 1.040000e-04 0.000104 \n", + "cgrid 0.0 0.0 0.0 0.000057 0.000057 5.700000e-05 0.000057 \n", + "length 0.0 0.0 0.0 0.000008 0.000016 1.300000e-05 0.000039 \n", + "maxfactor 0.0 0.0 0.0 0.000000 0.000001 4.000000e-07 0.000002 \n", + "\n", + " \n", + " count \n", + "function \n", + "Py_kgeneration 1 \n", + "generate 1 \n", + "gasdev 32768 \n", + "fftma2 1 \n", + "covariance 1 \n", + "ran2 41552 \n", + "cov_value 24624 \n", + "fourt 3 \n", + "prebuild_gwn 1 \n", + "build_real 1 \n", + "clean_real 1 \n", + "cgrid 1 \n", + "length 3 \n", + "maxfactor 5 " ] }, "execution_count": 21, @@ -594,171 +893,19 @@ } ], "source": [ - "df.sort_values(by=[\"used_virtual_mem\"], ascending=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Import libraries\n", - "from matplotlib_venn import venn3\n", - "from matplotlib import pyplot as plt\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "relations = {\n", - " \"generate\": [\"gasdev\"],\n", - " \"fftma2\": [\"covariance\", \"fourt\", \"prebuild_gwn\"]\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "plt.figure(dpi=125)\n", - "plt.title('Py_kgeneration')\n", - "plt.pie([0.010957, 0.012503], labels=[\"generate\", \"fftma2\"], normalize=True)\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import libraries\n", - "from matplotlib import pyplot as plt\n", - "import numpy as np\n", - "\n", - "\n", - "# Creating dataset\n", - "size = 6\n", - "cars = ['AUDI', 'BMW', 'FORD',\n", - "\t\t'TESLA', 'JAGUAR', 'MERCEDES']\n", - "\n", - "data = np.array([[23, 16], [17, 23],\n", - "\t\t\t\t[35, 11], [29, 33],\n", - "\t\t\t\t[12, 27], [41, 42]])\n", - "\n", - "# normalizing data to 2 pi\n", - "norm = data / np.sum(data)*2 * np.pi\n", - "\n", - "# obtaining ordinates of bar edges\n", - "left = np.cumsum(np.append(0,\n", - "\t\t\t\t\t\tnorm.flatten()[:-1])).reshape(data.shape)\n", - "\n", - "# Creating color scale\n", - "cmap = plt.get_cmap(\"tab20c\")\n", - "outer_colors = cmap(np.arange(6)*4)\n", - "inner_colors = cmap(np.array([1, 2, 5, 6, 9,\n", - "\t\t\t\t\t\t\t10, 12, 13, 15,\n", - "\t\t\t\t\t\t\t17, 18, 20 ]))\n", - "\n", - "# Creating plot\n", - "fig, ax = plt.subplots(figsize =(10, 7),\n", - "\t\t\t\t\tsubplot_kw = dict(polar = True))\n", - "\n", - "ax.bar(x = left[:, 0],\n", - "\twidth = norm.sum(axis = 1),\n", - "\tbottom = 1-size,\n", - "\theight = size,\n", - "\tcolor = outer_colors,\n", - "\tedgecolor ='w',\n", - "\tlinewidth = 1,\n", - "\talign =\"edge\")\n", - "\n", - "ax.bar(x = left.flatten(),\n", - "\twidth = norm.flatten(),\n", - "\tbottom = 1-2 * size,\n", - "\theight = size,\n", - "\tcolor = inner_colors,\n", - "\tedgecolor ='w',\n", - "\tlinewidth = 1,\n", - "\talign =\"edge\")\n", - "\n", - "ax.set(title =\"Nested pie chart\")\n", - "ax.set_axis_off()\n", - "\n", - "# show plot\n", - "plt.show()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "metadata": {}, - "outputs": [], - "source": [ - "data = []\n", - "with open(\"log_64.txt\") as log_file:\n", - " lines = log_file.readlines()\n", - " \n", - " for line in lines:\n", - " row = {}\n", - " if \"MEM\" in line:\n", - " split_line = line.split()\n", - " idx_used_mem = split_line.index(\"USED\") + 4\n", - " function_name, used_virtual_mem = get_function_name(split_line[2]), float(split_line[idx_used_mem])\n", - " #val = data.get(function_name, (0, 0, used_virtual_mem))\n", - " #data[function_name] = (val[0], val[1], max(used_virtual_mem, val[2]))\n", - "\n", - " row[\"function\"] = function_name\n", - " row[\"memory\"] = used_virtual_mem \n", - "\n", - " if \"ELAPSED\" in line:\n", - " split_line = line.split()\n", - " idx_elapsed = split_line.index(\"ELAPSED\") + 2\n", - " function_name, elapsed = get_function_name(split_line[2]), float(split_line[idx_elapsed].rsplit(\",\")[0])\n", - " #val = data.get(function_name, (elapsed, 0, 0))\n", - " #data[function_name] = (max(elapsed, val[0]), val[1] + 1, val[2])\n", - " #data.append({\"function\": function_name, \"time\": elapsed, \"memory\": 0})\n", - " row[\"function\"] = function_name\n", - " row[\"time\"] = elapsed\n", - "\n", - " if row: data.append(row)" - ] - }, - { - "cell_type": "code", - "execution_count": 88, - "metadata": {}, - "outputs": [], - "source": [ - "df2 = pd.DataFrame(data) " + "analyze('log_32.txt')" ] }, { - "cell_type": "code", - "execution_count": 89, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "df2 = df2.fillna(np.nan)" + "## N = 64" ] }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -774,86 +921,269 @@ " vertical-align: top;\n", " }\n", "\n", - " .dataframe thead th {\n", + " .dataframe thead tr th {\n", + " text-align: left;\n", + " }\n", + "\n", + " .dataframe thead tr:last-of-type th {\n", " text-align: right;\n", " }\n", "\n", "\n", " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
memorytime
minmaxmedianminmaxmeansumcount
functiontimememory
0Py_getvalues0.000008NaNPy_kgeneration233.5233.5233.530.35427030.35427030.35427030.3542701
1ran20.000003generate168.4168.4168.421.73952121.73952121.73952121.7395211
gasdev-2.94.40.00.0000090.0007420.00006116.076987262144
fftma268.068.068.08.6138548.6138548.6138548.6138541
covariance63.863.863.88.4123358.4123358.4123358.4123351
ran2-0.60.80.00.0000070.0002810.0000144.680977333450
2ran20.000023cov_value-0.71.00.00.0000180.0003200.0000304.646094156816
fourt0.31.70.40.0518300.0785160.0645630.1936893
prebuild_gwn2.52.52.50.0029390.0029390.0029390.0029391
build_real-0.3-0.3-0.30.0020890.0020890.0020890.0020891
clean_real1.01.01.00.0016770.0016770.0016770.0016771
cgrid0.00.00.00.0002710.0002710.0002710.0002711
3gasdev0.000046length0.00.00.00.0000480.0000810.0000650.0001963
4gasdev0.000006maxfactor0.00.00.00.0000130.0000150.0000140.0000564
\n", "" ], "text/plain": [ - " function time memory\n", - "0 Py_getvalues 0.000008 NaN\n", - "1 ran2 0.000003 0.0\n", - "2 ran2 0.000023 0.0\n", - "3 gasdev 0.000046 0.0\n", - "4 gasdev 0.000006 0.0" + " memory time \\\n", + " min max median min max mean \n", + "function \n", + "Py_kgeneration 233.5 233.5 233.5 30.354270 30.354270 30.354270 \n", + "generate 168.4 168.4 168.4 21.739521 21.739521 21.739521 \n", + "gasdev -2.9 4.4 0.0 0.000009 0.000742 0.000061 \n", + "fftma2 68.0 68.0 68.0 8.613854 8.613854 8.613854 \n", + "covariance 63.8 63.8 63.8 8.412335 8.412335 8.412335 \n", + "ran2 -0.6 0.8 0.0 0.000007 0.000281 0.000014 \n", + "cov_value -0.7 1.0 0.0 0.000018 0.000320 0.000030 \n", + "fourt 0.3 1.7 0.4 0.051830 0.078516 0.064563 \n", + "prebuild_gwn 2.5 2.5 2.5 0.002939 0.002939 0.002939 \n", + "build_real -0.3 -0.3 -0.3 0.002089 0.002089 0.002089 \n", + "clean_real 1.0 1.0 1.0 0.001677 0.001677 0.001677 \n", + "cgrid 0.0 0.0 0.0 0.000271 0.000271 0.000271 \n", + "length 0.0 0.0 0.0 0.000048 0.000081 0.000065 \n", + "maxfactor 0.0 0.0 0.0 0.000013 0.000015 0.000014 \n", + "\n", + " \n", + " sum count \n", + "function \n", + "Py_kgeneration 30.354270 1 \n", + "generate 21.739521 1 \n", + "gasdev 16.076987 262144 \n", + "fftma2 8.613854 1 \n", + "covariance 8.412335 1 \n", + "ran2 4.680977 333450 \n", + "cov_value 4.646094 156816 \n", + "fourt 0.193689 3 \n", + "prebuild_gwn 0.002939 1 \n", + "build_real 0.002089 1 \n", + "clean_real 0.001677 1 \n", + "cgrid 0.000271 1 \n", + "length 0.000196 3 \n", + "maxfactor 0.000056 4 " ] }, - "execution_count": 90, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df2.head()" + "analyze('log_64.txt')" ] }, { - "cell_type": "code", - "execution_count": 91, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "df2_grouped = df2.groupby(['function']).agg({'time': ['min', 'max', 'mean', 'sum', 'count'], 'memory': ['min', 'max', 'median']})" + "## N = 128" ] }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 29, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "5897863\n" + ] + }, { "data": { "text/html": [ @@ -879,19 +1209,19 @@ " \n", " \n", " \n", - " time\n", " memory\n", + " time\n", " \n", " \n", " \n", " min\n", " max\n", + " median\n", + " min\n", + " max\n", " mean\n", " sum\n", " count\n", - " min\n", - " max\n", - " median\n", " \n", " \n", " function\n", @@ -908,220 +1238,214 @@ " \n", " \n", " Py_kgeneration\n", - " 11.316621\n", - " 11.316621\n", - " 11.316621\n", - " 11.316621\n", + " 1864.7\n", + " 1864.7\n", + " 1864.7\n", + " 126.767549\n", + " 126.767549\n", + " 126.767549\n", + " 126.767549\n", " 1\n", - " -293.7\n", - " -293.7\n", - " -293.7\n", " \n", " \n", " generate\n", - " 7.424831\n", - " 7.424831\n", - " 7.424831\n", - " 7.424831\n", + " 1759.0\n", + " 1759.0\n", + " 1759.0\n", + " 97.731527\n", + " 97.731527\n", + " 97.731527\n", + " 97.731527\n", " 1\n", - " -226.3\n", - " -226.3\n", - " -226.3\n", " \n", " \n", " gasdev\n", - " 0.000003\n", - " 0.000277\n", - " 0.000021\n", - " 5.484581\n", - " 262144\n", - " -20.0\n", - " 5.3\n", + " -134.6\n", + " 18.5\n", " 0.0\n", + " 0.000000\n", + " 0.000772\n", + " 0.000027\n", + " 57.591066\n", + " 2097152\n", " \n", " \n", " fftma2\n", - " 3.891431\n", - " 3.891431\n", - " 3.891431\n", - " 3.891431\n", + " 129.3\n", + " 129.3\n", + " 129.3\n", + " 29.034783\n", + " 29.034783\n", + " 29.034783\n", + " 29.034783\n", " 1\n", - " -69.5\n", - " -69.5\n", - " -69.5\n", " \n", " \n", " covariance\n", - " 3.765731\n", - " 3.765731\n", - " 3.765731\n", - " 3.765731\n", + " 93.1\n", + " 93.1\n", + " 93.1\n", + " 27.493769\n", + " 27.493769\n", + " 27.493769\n", + " 27.493769\n", " 1\n", - " -65.0\n", - " -65.0\n", - " -65.0\n", " \n", " \n", - " cov_value\n", - " 0.000007\n", - " 0.000108\n", - " 0.000013\n", - " 2.062640\n", - " 156816\n", - " -1.9\n", - " 1.5\n", + " ran2\n", + " -4.8\n", + " 2.2\n", " 0.0\n", + " 0.000000\n", + " 0.000065\n", + " 0.000002\n", + " 4.661066\n", + " 2668394\n", " \n", " \n", - " ran2\n", - " 0.000002\n", - " 0.000114\n", - " 0.000005\n", - " 1.534732\n", - " 333450\n", - " -19.7\n", - " 2.6\n", + " cov_value\n", + " -0.6\n", + " 0.7\n", " 0.0\n", + " 0.000001\n", + " 0.000281\n", + " 0.000002\n", + " 2.443765\n", + " 1132300\n", " \n", " \n", " fourt\n", - " 0.032778\n", - " 0.051545\n", - " 0.039664\n", - " 0.118991\n", + " -5.2\n", + " 18.7\n", + " 0.5\n", + " 0.390409\n", + " 0.591145\n", + " 0.499473\n", + " 1.498418\n", " 3\n", - " -1.9\n", - " -0.0\n", - " -0.2\n", " \n", " \n", " build_real\n", - " 0.004053\n", - " 0.004053\n", - " 0.004053\n", - " 0.004053\n", - " 1\n", " 0.0\n", " 0.0\n", " 0.0\n", + " 0.017085\n", + " 0.017085\n", + " 0.017085\n", + " 0.017085\n", + " 1\n", " \n", " \n", " prebuild_gwn\n", - " 0.001132\n", - " 0.001132\n", - " 0.001132\n", - " 0.001132\n", + " 17.0\n", + " 17.0\n", + " 17.0\n", + " 0.014117\n", + " 0.014117\n", + " 0.014117\n", + " 0.014117\n", " 1\n", - " -2.5\n", - " -2.5\n", - " -2.5\n", " \n", " \n", " clean_real\n", - " 0.000830\n", - " 0.000830\n", - " 0.000830\n", - " 0.000830\n", + " 14.1\n", + " 14.1\n", + " 14.1\n", + " 0.009017\n", + " 0.009017\n", + " 0.009017\n", + " 0.009017\n", " 1\n", - " -0.7\n", - " -0.7\n", - " -0.7\n", " \n", " \n", " cgrid\n", - " 0.000148\n", - " 0.000148\n", - " 0.000148\n", - " 0.000148\n", - " 1\n", " 0.0\n", " 0.0\n", " 0.0\n", + " 0.000187\n", + " 0.000187\n", + " 0.000187\n", + " 0.000187\n", + " 1\n", " \n", " \n", " length\n", - " 0.000015\n", - " 0.000064\n", - " 0.000039\n", - " 0.000118\n", - " 3\n", " 0.0\n", " 0.0\n", " 0.0\n", + " 0.000032\n", + " 0.000035\n", + " 0.000033\n", + " 0.000099\n", + " 3\n", " \n", " \n", " maxfactor\n", - " 0.000004\n", - " 0.000025\n", - " 0.000011\n", - " 0.000043\n", - " 4\n", " 0.0\n", " 0.0\n", " 0.0\n", - " \n", - " \n", - " Py_getvalues\n", - " 0.000008\n", - " 0.000008\n", - " 0.000008\n", + " 0.000002\n", + " 0.000003\n", + " 0.000003\n", " 0.000008\n", - " 1\n", - " NaN\n", - " NaN\n", - " NaN\n", + " 3\n", " \n", " \n", "\n", "" ], "text/plain": [ - " time memory \\\n", - " min max mean sum count min \n", - "function \n", - "Py_kgeneration 11.316621 11.316621 11.316621 11.316621 1 -293.7 \n", - "generate 7.424831 7.424831 7.424831 7.424831 1 -226.3 \n", - "gasdev 0.000003 0.000277 0.000021 5.484581 262144 -20.0 \n", - "fftma2 3.891431 3.891431 3.891431 3.891431 1 -69.5 \n", - "covariance 3.765731 3.765731 3.765731 3.765731 1 -65.0 \n", - "cov_value 0.000007 0.000108 0.000013 2.062640 156816 -1.9 \n", - "ran2 0.000002 0.000114 0.000005 1.534732 333450 -19.7 \n", - "fourt 0.032778 0.051545 0.039664 0.118991 3 -1.9 \n", - "build_real 0.004053 0.004053 0.004053 0.004053 1 0.0 \n", - "prebuild_gwn 0.001132 0.001132 0.001132 0.001132 1 -2.5 \n", - "clean_real 0.000830 0.000830 0.000830 0.000830 1 -0.7 \n", - "cgrid 0.000148 0.000148 0.000148 0.000148 1 0.0 \n", - "length 0.000015 0.000064 0.000039 0.000118 3 0.0 \n", - "maxfactor 0.000004 0.000025 0.000011 0.000043 4 0.0 \n", - "Py_getvalues 0.000008 0.000008 0.000008 0.000008 1 NaN \n", + " memory time \\\n", + " min max median min max mean \n", + "function \n", + "Py_kgeneration 1864.7 1864.7 1864.7 126.767549 126.767549 126.767549 \n", + "generate 1759.0 1759.0 1759.0 97.731527 97.731527 97.731527 \n", + "gasdev -134.6 18.5 0.0 0.000000 0.000772 0.000027 \n", + "fftma2 129.3 129.3 129.3 29.034783 29.034783 29.034783 \n", + "covariance 93.1 93.1 93.1 27.493769 27.493769 27.493769 \n", + "ran2 -4.8 2.2 0.0 0.000000 0.000065 0.000002 \n", + "cov_value -0.6 0.7 0.0 0.000001 0.000281 0.000002 \n", + "fourt -5.2 18.7 0.5 0.390409 0.591145 0.499473 \n", + "build_real 0.0 0.0 0.0 0.017085 0.017085 0.017085 \n", + "prebuild_gwn 17.0 17.0 17.0 0.014117 0.014117 0.014117 \n", + "clean_real 14.1 14.1 14.1 0.009017 0.009017 0.009017 \n", + "cgrid 0.0 0.0 0.0 0.000187 0.000187 0.000187 \n", + "length 0.0 0.0 0.0 0.000032 0.000035 0.000033 \n", + "maxfactor 0.0 0.0 0.0 0.000002 0.000003 0.000003 \n", "\n", - " \n", - " max median \n", - "function \n", - "Py_kgeneration -293.7 -293.7 \n", - "generate -226.3 -226.3 \n", - "gasdev 5.3 0.0 \n", - "fftma2 -69.5 -69.5 \n", - "covariance -65.0 -65.0 \n", - "cov_value 1.5 0.0 \n", - "ran2 2.6 0.0 \n", - "fourt -0.0 -0.2 \n", - "build_real 0.0 0.0 \n", - "prebuild_gwn -2.5 -2.5 \n", - "clean_real -0.7 -0.7 \n", - "cgrid 0.0 0.0 \n", - "length 0.0 0.0 \n", - "maxfactor 0.0 0.0 \n", - "Py_getvalues NaN NaN " + " \n", + " sum count \n", + "function \n", + "Py_kgeneration 126.767549 1 \n", + "generate 97.731527 1 \n", + "gasdev 57.591066 2097152 \n", + "fftma2 29.034783 1 \n", + "covariance 27.493769 1 \n", + "ran2 4.661066 2668394 \n", + "cov_value 2.443765 1132300 \n", + "fourt 1.498418 3 \n", + "build_real 0.017085 1 \n", + "prebuild_gwn 0.014117 1 \n", + "clean_real 0.009017 1 \n", + "cgrid 0.000187 1 \n", + "length 0.000099 3 \n", + "maxfactor 0.000008 3 " ] }, - "execution_count": 92, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "df2_grouped.sort_values(by=('time', 'sum'), ascending=False)" + "analyze('log_128.txt')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## N = 256" ] }, { diff --git a/fftma_module/gen/lib_src/log.c b/fftma_module/gen/lib_src/log.c index 288e272..e3d12b7 100644 --- a/fftma_module/gen/lib_src/log.c +++ b/fftma_module/gen/lib_src/log.c @@ -22,6 +22,7 @@ #include #include "log.h" +#include #define MAX_CALLBACKS 32 @@ -149,6 +150,9 @@ void log_log(int level, const char *file, int line, const char *fmt, ...) { char* env_var = getenv("ENV"); if (strcmp("false", env_var) == 0) return; + char* substr = strstr(fmt, "MEM"); + if (strcmp("analysis", env_var) == 0 && substr == NULL) return; + lock(); if (!L.quiet && level >= L.level) { diff --git a/fftma_module/gen/save_logs.sh b/fftma_module/gen/save_logs.sh index 45f6d53..f12fd6a 100755 --- a/fftma_module/gen/save_logs.sh +++ b/fftma_module/gen/save_logs.sh @@ -1 +1,2 @@ -ENV=true python3 test.py $1 2>&1 | tee log_$1.txt \ No newline at end of file +python3 setup.py install --user +ENV=analysis python3 test.py $1 2>&1 | split -l 5000000 - log_$1- \ No newline at end of file