76 cloudy aggregation

ppcpy/cloudmask/profilesegment.py

@@ -22,11 +22,9 @@ def segment(data_cube) -> np.ndarray:
     """
 
     config_dict = data_cube.polly_config_dict
-    flagValPrf = data_cube.flagCloudFree & (~data_cube.retrievals_highres['depCalMask']) \
-            & (~data_cube.retrievals_highres['shutterOnMask']) & (~data_cube.retrievals_highres['fogMask'])
 
     print('intNProfiles', config_dict['intNProfiles'], 'minIntNProfiles', config_dict['minIntNProfiles'])
-    clFreeGrps = clFreeSeg(flagValPrf, config_dict['intNProfiles'], config_dict['minIntNProfiles'])
+    clFreeGrps = clFreeSeg(data_cube.flagValPrf, config_dict['intNProfiles'], config_dict['minIntNProfiles'])
 
     return clFreeGrps
 

ppcpy/interface/picassoProc.py

@@ -384,6 +384,14 @@ def cloudScreen(self, collect_debug:bool=False):
         self.flagCloudFree = cloudscreen.cloudscreen(self, collect_debug=collect_debug)
 
 
+    def genFlagValPrf(self):
+        """combine the cloudFree, depCal, shutterOn and fog masks to obtain valid profiles
+
+        """
+        self.flagValPrf = self.flagCloudFree & (~self.retrievals_highres['depCalMask']) \
+            & (~self.retrievals_highres['shutterOnMask']) & (~self.retrievals_highres['fogMask'])
+
+
     def cloudFreeSeg(self):
         """Cloud free profile segmentation.
 
@@ -397,29 +405,38 @@ def cloudFreeSeg(self):
             ]
 
         """
+        self.genFlagValPrf()
         self.clFreeGrps = profilesegment.segment(self)
 
-    def aggregate_profiles(self, var:str=None):
+    def aggregate_profiles(self, var:str=None, flagVal=None):
         """Aggregate highres profiles over cloud free segments.
 
         Parameters
         ----------
-        var : str
+        var : None | str
             Name of variables to aggregate.
             Default is `RCS`, `sigBGCor`, and `BG`.
+        flagVal : None | bool | np.ndarray
+            cut out single invalid profiles form `clFreeGrps`. If `True` the `data_cube.flagValPrf` is used. Individual array can also be supplied.
 
         Notes
         -----
         .. TODO:: Decide on a consistent way for doing the aggregation, do not mix mean and sum
         """
 
+        if isinstance(flagVal, (np.ndarray, list)):
+            flagVal = np.asarray(flagVal)
+            self.flagValPrf = flagVal
+        elif flagVal and not isinstance(flagVal, (np.ndarray, list)):
+            flagVal = self.flagValPrf
+
         if var == None:
             self.retrievals_profile['RCS'] = \
-                preprocprofiles.aggregate_clFreeGrps(self, 'RCS', func=np.nanmean)
+                preprocprofiles.aggregate_clFreeGrps(self, 'RCS', func=np.nanmean, flagVal=flagVal)
             self.retrievals_profile['sigBGCor'] = \
-                preprocprofiles.aggregate_clFreeGrps(self, 'sigBGCor')
+                preprocprofiles.aggregate_clFreeGrps(self, 'sigBGCor', flagVal=flagVal)
             self.retrievals_profile['BG'] = \
-                preprocprofiles.aggregate_clFreeGrps(self, 'BG')
+                preprocprofiles.aggregate_clFreeGrps(self, 'BG', flagVal=flagVal)
 
             # Remove empty dict keys (temporarly solution)
             for v in ['RCS', 'sigBGCor', 'BG']:
@@ -428,7 +445,7 @@ def aggregate_profiles(self, var:str=None):
 
         else:
             self.retrievals_profile[var] = \
-                preprocprofiles.aggregate_clFreeGrps(self, var)
+                preprocprofiles.aggregate_clFreeGrps(self, var, flagVal=flagVal)
 
             # Remove empty dict keys (temporarly solution)
             if self.retrievals_profile[var] is None:

ppcpy/preprocess/profiles.py

@@ -1,17 +1,24 @@
 
 import numpy as np
 
-def aggregate_clFreeGrps(data_cube, var:str, func=np.nansum):
-    """
-    Aggregate the highres signal over the periods of the cloud free signal.
+def aggregate_clFreeGrps(data_cube, var:str, func=np.nansum, flagVal=None):
+    """aggregate the highres signal over the periods of the cloud free signal.
 
-    Input:
-        - data_cube (object): Main PicassoProc object.
-        - var (string): name of variable to be aggregated.
-        - func (function): function to do the aggregateion (mean, sum, median, etc), defult: np.nansum.
-
-    Output:
-        - out (np.ndarray): Aggregated highres signal for each cloud free segment.
+    Parameters
+    ----------
+    data_cube : object
+        Main PicassoProc object.
+    var : string
+        name of variable to be aggregated.
+    func : function
+        function to do the aggregateion (mean, sum, median, etc), defult: np.nansum.
+    flagVal : None | np.ndarray
+        boolean flag for valid profiles
+
+    Returns
+    -------
+    out : np.ndarray
+        Aggregated highres signal for each cloud free segment.
     """
     # Check if variable exists, if not return.
     if var not in data_cube.retrievals_highres:
@@ -24,6 +31,12 @@ def aggregate_clFreeGrps(data_cube, var:str, func=np.nansum):
 
     for i, cldFree in enumerate(data_cube.clFreeGrps):
         cldFree = cldFree[0], cldFree[1] + 1
-        out[i, ...] = func(data_cube.retrievals_highres[var][slice(*cldFree), ...], axis=0)
+        data_chunk = data_cube.retrievals_highres[var][slice(*cldFree), ...]
+        print(data_chunk.shape)
+        if isinstance(flagVal, np.ndarray):
+            data_chunk = data_chunk[flagVal[slice(*cldFree)], ...] 
+        print(data_chunk.shape)
+        out[i, ...] = func(data_chunk, axis=0)
+
 
     return out