Merge remote-tracking branch 'upstream/hotfixes' into release

Author: fit-alessandro-berti

pm4py/algo/discovery/inductive/fall_through/activity_once_per_trace.py

@@ -56,4 +56,5 @@ def _get_candidate(
                 return None
 
         # Return any one of the remaining candidates
-        return next(iter(candidates)) if candidates else None
+        candidates = sorted(list(candidates))
+        return candidates[0] if candidates else None

pm4py/algo/filtering/pandas/attributes/attributes_filter.py

@@ -43,6 +43,7 @@ class Parameters(Enum):
     STREAM_FILTER_KEY2 = "stream_filter_key2"
     STREAM_FILTER_VALUE2 = "stream_filter_value2"
     KEEP_ONCE_PER_CASE = "keep_once_per_case"
+    KEEP_NAN_VALUES = "keep_nan_values"
 
 
 def apply_numeric_events(
@@ -66,6 +67,7 @@ def apply_numeric_events(
         Possible parameters of the algorithm:
             Parameters.ATTRIBUTE_KEY => indicates which attribute to filter
             positive => keep or remove events?
+            Parameters.KEEP_NAN_VALUES -> Specifies if the filter should keep NAN values for the selected attribute. Default is False
 
     Returns
     --------------
@@ -81,11 +83,19 @@ def apply_numeric_events(
     positive = exec_utils.get_param_value(
         Parameters.POSITIVE, parameters, True
     )
+    keep_nan = exec_utils.get_param_value(
+        Parameters.KEEP_NAN_VALUES, parameters, False
+    )
 
     if positive:
-        ret = df[(df[attribute_key] >= int1) & (df[attribute_key] <= int2)]
+        event_filter = (df[attribute_key] >= int1) & (df[attribute_key] <= int2)
     else:
-        ret = df[(df[attribute_key] < int1) | (df[attribute_key] > int2)]
+        event_filter = (df[attribute_key] < int1) | (df[attribute_key] > int2)
+
+    if keep_nan:
+        event_filter = event_filter | df[attribute_key].isna()
+
+    ret = df[event_filter]
 
     ret.attrs = copy(df.attrs) if hasattr(df, "attrs") else {}
     return ret
@@ -112,6 +122,7 @@ def apply_numeric(
         Possible parameters of the algorithm:
             Parameters.ATTRIBUTE_KEY => indicates which attribute to filter
             Parameters.POSITIVE => keep or remove traces with such events?
+            Parameters.KEEP_NAN_VALUES -> Specifies if the filter should keep NAN values for the selected attribute. Default is False
 
     Returns
     --------------
@@ -130,6 +141,9 @@ def apply_numeric(
     positive = exec_utils.get_param_value(
         Parameters.POSITIVE, parameters, True
     )
+    keep_nan = exec_utils.get_param_value(
+        Parameters.KEEP_NAN_VALUES, parameters, False
+    )
 
     # stream_filter_key is helpful to filter on cases containing an event with an attribute
     # in the specified value set, but such events shall have an activity in
@@ -147,9 +161,11 @@ def apply_numeric(
         Parameters.STREAM_FILTER_VALUE2, parameters, None
     )
 
-    filtered_df_by_ev = df[
-        (df[attribute_key] >= int1) & (df[attribute_key] <= int2)
-    ]
+    event_filter = (df[attribute_key] >= int1) & (df[attribute_key] <= int2)
+    if keep_nan:
+        event_filter = event_filter | df[attribute_key].isna()
+    filtered_df_by_ev = df[event_filter]
+
     if stream_filter_key1 is not None:
         filtered_df_by_ev = filtered_df_by_ev[
             filtered_df_by_ev[stream_filter_key1] == stream_filter_value1
@@ -189,6 +205,8 @@ def apply_events(
             Parameters.ATTRIBUTE_KEY -> Attribute we want to filter
             Parameters.POSITIVE -> Specifies if the filter should be applied including traces (positive=True) or
             excluding traces (positive=False)
+            Parameters.KEEP_NAN_VALUES -> Specifies if the filter should keep NAN values for the selected attribute. Default is False
+
     Returns
     ----------
     df
@@ -203,11 +221,18 @@ def apply_events(
     positive = exec_utils.get_param_value(
         Parameters.POSITIVE, parameters, True
     )
+    keep_nan = exec_utils.get_param_value(
+        Parameters.KEEP_NAN_VALUES, parameters, False
+    )
+
+    event_filter = df[attribute_key].isin(values)
+    if keep_nan:
+        event_filter = event_filter | df[attribute_key].isna()
 
     if positive:
-        ret = df[df[attribute_key].isin(values)]
+        ret = df[event_filter]
     else:
-        ret = df[~df[attribute_key].isin(values)]
+        ret = df[~event_filter]
 
     ret.attrs = copy(df.attrs) if hasattr(df, "attrs") else {}
     return ret
@@ -233,6 +258,8 @@ def apply(
             Parameters.ATTRIBUTE_KEY -> Attribute we want to filter
             Parameters.POSITIVE -> Specifies if the filter should be applied including traces (positive=True) or
             excluding traces (positive=False)
+            Parameters.KEEP_NAN_VALUES -> Specifies if the filter should keep NAN values for the selected attribute. Default is False
+
     Returns
     ----------
     df
@@ -250,13 +277,17 @@ def apply(
     positive = exec_utils.get_param_value(
         Parameters.POSITIVE, parameters, True
     )
+    keep_nan = exec_utils.get_param_value(
+        Parameters.KEEP_NAN_VALUES, parameters, False
+    )
 
     return filter_df_on_attribute_values(
         df,
         values,
         case_id_glue=case_id_glue,
         attribute_key=attribute_key,
         positive=positive,
+        keep_nan_values=keep_nan
     )
 
 
@@ -266,6 +297,7 @@ def filter_df_on_attribute_values(
     case_id_glue="case:concept:name",
     attribute_key="concept:name",
     positive=True,
+    keep_nan_values=False,
 ):
     """
     Filter dataframe on attribute values
@@ -283,6 +315,8 @@ def filter_df_on_attribute_values(
     positive
         Specifies if the filtered should be applied including traces (positive=True) or excluding traces
         (positive=False)
+    keep_nan_values
+        Specifies if the filter should keep NAN values for the selected attribute
 
     Returns
     ----------
@@ -291,7 +325,12 @@ def filter_df_on_attribute_values(
     """
     if values is None:
         values = []
-    filtered_df_by_ev = df[df[attribute_key].isin(values)]
+
+    event_filter = df[attribute_key].isin(values)
+    if keep_nan_values:
+        event_filter = event_filter | df[attribute_key].isna()
+    filtered_df_by_ev = df[event_filter]
+
     i1 = df.set_index(case_id_glue).index
     i2 = filtered_df_by_ev.set_index(case_id_glue).index
     if positive:

pm4py/objects/conversion/ocel/variants/ocel_to_nx.py

@@ -35,7 +35,7 @@ class Parameters(Enum):
 
 def apply(ocel: OCEL, parameters: Optional[Dict[Any, Any]] = None):
     """
-    Converts an OCEL to a NetworkX DiGraph object.
+    Converts an OCEL to a NetworkX MultiDiGraph object.
     The nodes are the events and objects of the OCEL.
     The edges are of two different types:
     - relation edges, connecting an event to its related objects
@@ -52,7 +52,7 @@ def apply(ocel: OCEL, parameters: Optional[Dict[Any, Any]] = None):
     Returns
     ---------------
     G
-        NetworkX DiGraph
+        NetworkX MultiDiGraph
     """
     if parameters is None:
         parameters = {}
@@ -64,7 +64,7 @@ def apply(ocel: OCEL, parameters: Optional[Dict[Any, Any]] = None):
         Parameters.INCLUDE_OBJECT_CHANGES, parameters, True
     )
 
-    G = nx_utils.DiGraph()
+    G = nx_utils.MultiDiGraph()
 
     stream = ocel.events.to_dict("records")
     stream = to_event_stream.__postprocess_stream(stream)

pm4py/statistics/attributes/common/get.py

@@ -148,8 +148,16 @@ def get_kde_numeric_attribute(values, parameters=None):
 
         # linear space including both endpoints
         xs1 = np.linspace(min_val, max_val, half, endpoint=True)
-        # geometric space including both endpoints (avoid zero)
-        xs2 = np.geomspace(max(min_val, eps), max_val, half, endpoint=True)
+        # try to enrich the sampling near the distribution tails; fall back when geomspace is not applicable
+        if min_val > 0 and max_val > 0:
+            # both bounds positive -> standard geometric spacing
+            xs2 = np.geomspace(max(min_val, eps), max_val, half, endpoint=True)
+        elif min_val < 0 and max_val < 0:
+            # both bounds negative -> mirror geometric spacing on the absolute values
+            xs2 = -np.geomspace(abs(min_val), max(abs(max_val), eps), half, endpoint=True)
+        else:
+            # bounds cross or hit zero -> stick to linear spacing to avoid invalid geometric ranges
+            xs2 = np.linspace(min_val, max_val, half, endpoint=True)
 
         # combine, add exact endpoints, dedupe & sort
         xs = np.unique(

tests/filtering_pandas_test.py

@@ -55,6 +55,77 @@ def test_filtering_attr_events(self):
         del df1
         del df2
 
+    def test_filtering_attr_values(self):
+        # to avoid static method warnings in tests,
+        # that by construction of the unittest package have to be expressed in such way
+        self.dummy_variable = "dummy_value"
+        input_log = os.path.join(INPUT_DATA_DIR, "running-example.csv")
+        dataframe = pandas_utils.read_csv(input_log)
+        dataframe = dataframe_utils.convert_timestamp_columns_in_df(dataframe, timest_format=constants.DEFAULT_TIMESTAMP_PARSE_FORMAT)
+        # add a row with a NAN value in the filter attribute
+        case_id_of_nan_case = len(dataframe)
+        dataframe.loc[len(dataframe)] = {"case:concept:name": case_id_of_nan_case,
+                                    "case:concept": "register request",
+                                    "Resource": None,
+                                    "time:timestamp": "2011-01-09 12:02:00+00:00"
+                                    }
+
+        pos = attributes_filter.apply(dataframe, ["Pete"],
+                                             parameters={attributes_filter.Parameters.POSITIVE: True,
+                                                         attributes_filter.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "Resource"})
+        neg = attributes_filter.apply(dataframe, ["Pete"],
+                                             parameters={attributes_filter.Parameters.POSITIVE: False,
+                                                         attributes_filter.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "Resource"})
+
+        pos_with_nan = attributes_filter.apply(dataframe, ["Pete"],
+                                             parameters={attributes_filter.Parameters.POSITIVE: True,
+                                                         attributes_filter.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "Resource",
+                                                         attributes_filter.Parameters.KEEP_NAN_VALUES: True})
+
+        # We configure the filter to keep NAN values, however because of attributes_filter.Parameters.POSITIVE: False
+        # the complement of the filter result is taken. This means that the NAN values are not included
+        neg_without_nan = attributes_filter.apply(dataframe, ["Pete"],
+                                             parameters={attributes_filter.Parameters.POSITIVE: False,
+                                                         attributes_filter.PARAMETER_CONSTANT_ATTRIBUTE_KEY: "Resource",
+                                                         attributes_filter.Parameters.KEEP_NAN_VALUES: True})
+
+        case_identifier = "case:concept:name"
+        num_cases = len(dataframe[case_identifier].unique())
+
+        cases_pos = pos[case_identifier].unique()
+        num_cases_pos = len(cases_pos)
+
+        cases_neg = neg[case_identifier].unique()
+        num_cases_neg = len(cases_neg)
+
+        cases_pos_with_nan = pos_with_nan[case_identifier].unique()
+        num_cases_pos_with_nan = len(cases_pos_with_nan)
+
+        cases_neg_without_nan = neg_without_nan[case_identifier].unique()
+        num_cases_neg_without_nan = len(cases_neg_without_nan)
+
+        self.assertEqual(num_cases_pos + num_cases_neg, num_cases)
+        self.assertEqual(num_cases_pos_with_nan + num_cases_neg_without_nan, num_cases)
+
+        self.assertEqual(num_cases_pos + 1, num_cases_pos_with_nan)
+        self.assertEqual(num_cases_neg, num_cases_neg_without_nan + 1)
+
+        self.assertNotIn(case_id_of_nan_case, cases_pos)
+        self.assertIn(case_id_of_nan_case, cases_neg)
+        self.assertIn(case_id_of_nan_case, cases_pos_with_nan)
+        self.assertNotIn(case_id_of_nan_case, cases_neg_without_nan)
+
+        del pos
+        del neg
+        del pos_with_nan
+        del neg_without_nan
+
+        del cases_pos
+        del cases_neg
+        del cases_pos_with_nan
+        del cases_neg_without_nan
+
+
     def test_filtering_paths(self):
         # to avoid static method warnings in tests,
         # that by construction of the unittest package have to be expressed in such way