From e69ada1ccc54826e9393e437d231a8aa8e7f620d Mon Sep 17 00:00:00 2001 From: Michael McKinsey Date: Mon, 10 Jun 2024 14:40:37 -0500 Subject: [PATCH 1/5] Add optional argument to turn off _fill_perfdata --- thicket/ensemble.py | 8 ++++++-- thicket/thicket.py | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/thicket/ensemble.py b/thicket/ensemble.py index 3e229357..ef10fdea 100644 --- a/thicket/ensemble.py +++ b/thicket/ensemble.py @@ -369,12 +369,15 @@ def _handle_statsframe(): return combined_th @staticmethod - def _index(thickets, from_statsframes=False, disable_tqdm=False): + def _index( + thickets, from_statsframes=False, fill_perfdata=True, disable_tqdm=False + ): """Unify a list of thickets into a single thicket Arguments: thickets (list): list of Thicket objects from_statsframes (bool): Whether this method was invoked from from_statsframes + fill_perfdata (bool): whether to fill missing rows in performance data table disable_tqdm (bool): whether to disable tqdm progress bar Returns: @@ -456,7 +459,8 @@ def _fill_perfdata(df, numerical_fill_value=np.nan): validate_dataframe(unify_df) # Insert missing rows in dataframe - unify_df = _fill_perfdata(unify_df) + if fill_perfdata: + unify_df = _fill_perfdata(unify_df) # Sort PerfData unify_df.sort_index(inplace=True) diff --git a/thicket/thicket.py b/thicket/thicket.py index 439e1324..a272c7bc 100644 --- a/thicket/thicket.py +++ b/thicket/thicket.py @@ -203,7 +203,11 @@ def thicketize_graphframe(gf, prf): @staticmethod def from_caliper( - filename_or_stream, query=None, intersection=False, disable_tqdm=False + filename_or_stream, + query=None, + intersection=False, + fill_perfdata=True, + disable_tqdm=False, ): """Read in a Caliper .cali or .json file. @@ -212,36 +216,48 @@ def from_caliper( `.cali` or JSON-split format, or an open file object to read one query (str): cali-query in CalQL format intersection (bool): whether to perform intersection or union (default) + fill_perfdata (bool): whether to fill missing performance data with NaNs disable_tqdm (bool): whether to display tqdm progress bar """ return Thicket.reader_dispatch( GraphFrame.from_caliper, intersection, + fill_perfdata, disable_tqdm, filename_or_stream, query, ) @staticmethod - def from_hpctoolkit(dirname, intersection=False, disable_tqdm=False): + def from_hpctoolkit( + dirname, intersection=False, fill_perfdata=True, disable_tqdm=False + ): """Create a GraphFrame using hatchet's HPCToolkit reader and use its attributes to make a new thicket. Arguments: dirname (str): parent directory of an HPCToolkit experiment.xml file intersection (bool): whether to perform intersection or union (default) + fill_perfdata (bool): whether to fill missing performance data with NaNs disable_tqdm (bool): whether to display tqdm progress bar Returns: (thicket): new thicket containing HPCToolkit profile data """ return Thicket.reader_dispatch( - GraphFrame.from_hpctoolkit, intersection, disable_tqdm, dirname + GraphFrame.from_hpctoolkit, + intersection, + fill_perfdata, + disable_tqdm, + dirname, ) @staticmethod def from_caliperreader( - filename_or_caliperreader, intersection=False, disable_tqdm=False + filename_or_caliperreader, + intersection=False, + fill_perfdata=True, + disable_tqdm=False, ): """Helper function to read one caliper file. @@ -249,11 +265,13 @@ def from_caliperreader( filename_or_caliperreader (str or CaliperReader): name of a Caliper output file in `.cali` format, or a CaliperReader object intersection (bool): whether to perform intersection or union (default) + fill_perfdata (bool): whether to fill missing performance data with NaNs disable_tqdm (bool): whether to display tqdm progress bar """ return Thicket.reader_dispatch( GraphFrame.from_caliperreader, intersection, + fill_perfdata, disable_tqdm, filename_or_caliperreader, ) @@ -295,7 +313,9 @@ def from_literal(graph_dict): return tk @staticmethod - def reader_dispatch(func, intersection, disable_tqdm, *args, **kwargs): + def reader_dispatch( + func, intersection, fill_perfdata, disable_tqdm, *args, **kwargs + ): """Create a thicket from a list, directory of files, or a single file. Arguments: @@ -353,6 +373,7 @@ def reader_dispatch(func, intersection, disable_tqdm, *args, **kwargs): thickets=ens_list, axis="index", calltree=calltree, + fill_perfdata=fill_perfdata, disable_tqdm=disable_tqdm, ) @@ -372,6 +393,7 @@ def concat_thickets( calltree (str): calltree to use -> "union" or "intersection" Keyword Arguments: + fill_perfdata (bool): (if axis="index") Whether to fill missing performance data with NaNs headers (list): (if axis="columns") List of headers to use for the new columnar multi-index metadata_key (str): (if axis="columns") Name of the column from the metadata tables to replace the 'profile' index. If no argument is provided, it is assumed that there is no profile-wise @@ -381,10 +403,16 @@ def concat_thickets( (thicket): concatenated thicket """ - def _index(thickets, from_statsframes=False, disable_tqdm=disable_tqdm): + def _index( + thickets, + from_statsframes=False, + fill_perfdata=True, + disable_tqdm=disable_tqdm, + ): thicket_parts = Ensemble._index( thickets=thickets, from_statsframes=from_statsframes, + fill_perfdata=fill_perfdata, disable_tqdm=disable_tqdm, ) From 89a8a15d235dc1d52d36ca8e98edea03596d9604 Mon Sep 17 00:00:00 2001 From: Michael McKinsey Date: Mon, 10 Jun 2024 14:42:01 -0500 Subject: [PATCH 2/5] Update docstring --- thicket/ensemble.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thicket/ensemble.py b/thicket/ensemble.py index ef10fdea..b0d6d80e 100644 --- a/thicket/ensemble.py +++ b/thicket/ensemble.py @@ -377,7 +377,7 @@ def _index( Arguments: thickets (list): list of Thicket objects from_statsframes (bool): Whether this method was invoked from from_statsframes - fill_perfdata (bool): whether to fill missing rows in performance data table + fill_perfdata (bool): whether to fill missing performance data with NaNs disable_tqdm (bool): whether to disable tqdm progress bar Returns: From 3e87b2efd92c391f733fbb1e81158de2a914da80 Mon Sep 17 00:00:00 2001 From: Michael McKinsey Date: Tue, 11 Jun 2024 17:07:30 -0500 Subject: [PATCH 3/5] Add squash to filter_metadata if fill perfdata is off --- thicket/thicket.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/thicket/thicket.py b/thicket/thicket.py index a272c7bc..af0136a1 100644 --- a/thicket/thicket.py +++ b/thicket/thicket.py @@ -1140,6 +1140,12 @@ def filter_metadata(self, select_function): else: raise InvalidFilter("The argument passed to filter must be a callable.") + # If fill_perfdata is False, may need to squash + unique_perf_indices = set(new_thicket.dataframe.index.droplevel("node")) + full_idx = all([idx in unique_perf_indices for idx in new_thicket.profile]) + if not full_idx: + new_thicket = new_thicket.squash() + return new_thicket def filter(self, filter_func): From af09ae984652e54c6b1e9540fe968a772492af4d Mon Sep 17 00:00:00 2001 From: Michael McKinsey Date: Tue, 11 Jun 2024 18:01:06 -0500 Subject: [PATCH 4/5] Fix check --- thicket/thicket.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/thicket/thicket.py b/thicket/thicket.py index af0136a1..db44ec96 100644 --- a/thicket/thicket.py +++ b/thicket/thicket.py @@ -1141,9 +1141,7 @@ def filter_metadata(self, select_function): raise InvalidFilter("The argument passed to filter must be a callable.") # If fill_perfdata is False, may need to squash - unique_perf_indices = set(new_thicket.dataframe.index.droplevel("node")) - full_idx = all([idx in unique_perf_indices for idx in new_thicket.profile]) - if not full_idx: + if len(new_thicket.graph) != len(new_thicket.dataframe.index.get_level_values("node").unique()): new_thicket = new_thicket.squash() return new_thicket From e66ba50bb37401f14e3c43324119b16c55b6ca8f Mon Sep 17 00:00:00 2001 From: Michael McKinsey Date: Tue, 11 Jun 2024 18:06:39 -0500 Subject: [PATCH 5/5] black --- thicket/thicket.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/thicket/thicket.py b/thicket/thicket.py index db44ec96..3ae84645 100644 --- a/thicket/thicket.py +++ b/thicket/thicket.py @@ -1141,7 +1141,9 @@ def filter_metadata(self, select_function): raise InvalidFilter("The argument passed to filter must be a callable.") # If fill_perfdata is False, may need to squash - if len(new_thicket.graph) != len(new_thicket.dataframe.index.get_level_values("node").unique()): + if len(new_thicket.graph) != len( + new_thicket.dataframe.index.get_level_values("node").unique() + ): new_thicket = new_thicket.squash() return new_thicket