Merge pull request #97 from Stanford-NavLab/derek/sort-where

Derek/sort where

Author: kanhereashwin

gnss_lib_py/parsers/navdata.py

@@ -265,20 +265,40 @@ def concat(self, navdata=None, axis=1, inplace=False):
         return new_navdata
 
     def where(self, key_idx, value, condition="eq"):
-        """Return NavData where conditions are met for the given row
+        """Return NavData where conditions are met for the given row.
+
+        For string rows, only the "eq" and "neq" conditions are valid.
+        The "value" argument can contain either a string, np.nan or an
+        array-like object of strings. If an array-like object of strings
+        is passed in then np.isin() is used to check the condition
+        meaning that the returned subset will contain one of the values
+        in the "value" array-like object for the "eq" condition or none
+        of the values in the "value" array-like object for the "neq"
+        condition.
+
+        For non-string rows, all valid conditions are listed in the
+        "condition" argument description. The "value" argument can either
+        contain a numeric or an array-like object of numerics for both
+        the "eq" and "neq" conditions.
+        If an array-like object is passed then the returned subset will
+        contain one of the values in the "value" array-like object for
+        the "eq" condition or none of the values in the "value"
+        array-like object for the "neq" condition.
+        For the "between" condition, the two limit values must be passed
+        into the "value" argument as an array-like object.
 
         Parameters
         ----------
         key_idx : string/int
             Key or index of the row in which conditions will be checked
-        value : float/list
-            Number (or list of two numbers for ) to compare array values
-            against
+        value : float/int/str/array-like
+            Value that the row is checked against, array-like object
+            possible for "eq", "neq", or "between" conditions.
         condition : string
             Condition type (greater than ("greater")/ less than ("lesser")/
             equal to ("eq")/ greater than or equal to ("geq")/
             lesser than or equal to ("leq") / in between ("between")
-            inclusive of the provided limits
+            inclusive of the provided limits / not equal to ("neq"))
 
         Returns
         -------
@@ -293,19 +313,40 @@ def where(self, key_idx, value, condition="eq"):
         return new_navdata
 
     def argwhere(self, key_idx, value, condition="eq"):
-        """Return columns where conditions are met for the given row
+        """Return columns where conditions are met for the given row.
+
+        For string rows, only the "eq" and "neq" conditions are valid.
+        The "value" argument can contain either a string, np.nan or an
+        array-like object of strings. If an array-like object of strings
+        is passed in then np.isin() is used to check the condition
+        meaning that the returned subset will contain one of the values
+        in the "value" array-like object for the "eq" condition or none
+        of the values in the "value" array-like object for the "neq"
+        condition.
+
+        For non-string rows, all valid conditions are listed in the
+        "condition" argument description. The "value" argument can either
+        contain a numeric or an array-like object of numerics for both
+        the "eq" and "neq" conditions.
+        If an array-like object is passed then the returned subset will
+        contain one of the values in the "value" array-like object for
+        the "eq" condition or none of the values in the "value"
+        array-like object for the "neq" condition.
+        For the "between" condition, the two limit values must be passed
+        into the "value" argument as an array-like object.
 
         Parameters
         ----------
         key_idx : string/int
             Key or index of the row in which conditions will be checked
-        value : float/list
-            Number (or list of two numbers for ) to compare array values against
+        value : float/int/str/array-like
+            Value that the row is checked against, array-like object
+            possible for "eq", "neq", or "between" conditions.
         condition : string
             Condition type (greater than ("greater")/ less than ("lesser")/
             equal to ("eq")/ greater than or equal to ("geq")/
-            lesser than or equal to ("leq") / in between ("between")/
-            not equal to ("neq"))
+            lesser than or equal to ("leq") / in between ("between")
+            inclusive of the provided limits / not equal to ("neq"))
 
         Returns
         -------
@@ -314,41 +355,52 @@ def argwhere(self, key_idx, value, condition="eq"):
             for specified row
         """
         rows, _ = self._parse_key_idx(key_idx)
-        inv_map = self.inv_map
         row_list, row_str = self._get_str_rows(rows)
         if len(row_list)>1:
             error_msg = "where does not currently support multiple rows"
             raise NotImplementedError(error_msg)
         row = row_list[0]
         row_str = row_str[0]
-        new_cols = None
+        new_cols = np.array([])
         if row_str:
             # Values in row are strings
             if condition not in ("eq","neq"):
-                raise ValueError("Inequality comparison not valid for strings")
-            key = inv_map[row]
-            for str_key, str_value in self.str_map[key].items():
-                if str_value==str(value):
-                    if condition == "eq":
-                        new_cols = np.argwhere(self.array[row, :]==str_key)
-                        break
-                    # condition == "neq"
-                    new_cols = np.argwhere(self.array[row, :]!=str_key)
-                    break
-            if new_cols is None:
-                new_cols = np.array([])
+                raise ValueError("Unsupported where condition for strings")
+            if isinstance(value,str):
+                str_check = [str(value)]
+            elif isinstance(value,(np.ndarray,list,tuple,set)):
+                str_check = [str(v) for v in value]
+            elif np.isnan(value):
+                str_check = [str(np.nan)]
+            else:
+                raise ValueError("Value must be string or array-like" \
+                               + "for string condition checks")
             # Extract columns where condition holds true and return new NavData
+            if condition == "eq":
+                new_cols = np.argwhere(np.isin(self[row, :],str_check))
+            else:
+                # condition == "neq"
+                new_cols = np.argwhere(~np.isin(self[row, :],str_check))
+
         else:
             # Values in row are numerical
             # Find columns where value can be found and return new NavData
             if condition=="eq":
-                if not isinstance(value,str) and np.isnan(value):
+                if isinstance(value,(np.ndarray,list,tuple,set)):
+                    # use numpy's isin() condition if list of values
+                    new_cols = np.argwhere(np.isin(self.array[row, :],
+                                           value))
+                elif not isinstance(value,str) and np.isnan(value):
                     # check isinstance b/c np.isnan can't handle strings
                     new_cols = np.argwhere(np.isnan(self.array[row, :]))
                 else:
                     new_cols = np.argwhere(self.array[row, :]==value)
             elif condition=="neq":
-                if not isinstance(value,str) and np.isnan(value):
+                if isinstance(value,(np.ndarray,list,tuple,set)):
+                    # use numpy's isin() condition if list of values
+                    new_cols = np.argwhere(~np.isin(self.array[row, :],
+                                           value))
+                elif not isinstance(value,str) and np.isnan(value):
                     # check isinstance b/c np.isnan can't handle strings
                     new_cols = np.argwhere(~np.isnan(self.array[row, :]))
                 else:
@@ -370,71 +422,50 @@ def argwhere(self, key_idx, value, condition="eq"):
         new_cols = np.squeeze(new_cols)
         return new_cols
 
-
-    def keep_cols_where(self, key_idx, values, condition='eq'):
-        """Return NavData containing columns that contain value from given array
-
-        Given a list of values, for the equality condition the returned
-        subset of measurements will contain one of the given values in
-        the given key_idx; for the inequality condition the returned
-        subset of measurements will contain none of the given values in
-        the given key_idx.
+    def sort(self, order=None, ind=None, ascending=True,
+             inplace=False):
+        """Sort values along given row or using given index
 
         Parameters
         ----------
-        key_idx : string/int
-            Key or index of the row in which conditions will be checked
-        values : list
-            List of values that form equality/inequality criteria for the
-            NavData subset that is returned.
-        condition : string
-            Only equality ("eq") and not-equal ("neq") condition types
-            are supported
+        order : string/int
+            Key or index of the row on which NavData will be sorted
+        ind : list/np.ndarray
+            Ordering of indices to be used for sorting
+        ascending : bool
+            If true, sorts "ascending", otherwise sorts "descending"
+        inplace : bool
+            If False, will return new NavData instance with rows
+            renamed. If True, will rename data rows in the
+            current NavData instance.
 
         Returns
         -------
-        subset_navdata : gnss_lib_py.parsers.navdata.NavData
-            NavData containing values in the key_idx that satisfy the
-            condition for the given values list
-        """
-        keep_cols = []
-        for value in values:
-            cols = self.argwhere(key_idx, value, condition)
-            if isinstance(cols, np.ndarray) and np.size(cols)==1:
-                cols = [cols]
-            keep_cols.extend(cols)
-        keep_cols = np.sort(keep_cols)
-        subset_navdata = self.copy(cols=keep_cols)
-        return subset_navdata
-
-
-    def sort(self, key_idx=None, ind=None, order="ascending"):
-        """Sort values along given row or using given index
+        new_navdata : gnss_lib_py.parsers.navdata.NavData or None
+            If inplace is False, returns NavData instance after renaming
+            specified rows. If inplace is True, returns
+            None.
 
-        Parameters
-        ----------
-        key_idx : string/int
-            Key or index of the row in which conditions will be checked
-        ind : list/np.ndarray
-            Ordering of indices to be used for sorting
-        order : string
-            Order in which to sort: "ascending" or "descending"
         """
         if ind is None:
-            assert key_idx is not None, \
-            "Provide row along which to sort because index is not given"
-            if order=="ascending":
-                ind = np.argsort(self[key_idx])
-            elif order=="descending":
-                ind = np.argsort(-self[key_idx])
+            assert order is not None, \
+            "Provide 'order' arg as row on which NavData is sorted"
+            if ascending:
+                ind = np.argsort(self[order])
             else:
-                raise RuntimeError("Can only sort in ascending or ", \
-                    "descending order")
-        new_navdata = NavData()
-        for row in self.rows:
-            new_navdata[row] = self[row][ind]
-        return new_navdata
+                ind = np.argsort(-self[order])
+
+        if not inplace:
+            new_navdata = self.copy()   # create copy to return
+        for row_idx in range(self.shape[0]):
+            if inplace:
+                self.array[row_idx,:] = self.array[row_idx,ind]
+            else:
+                new_navdata.array[row_idx,:] = new_navdata.array[row_idx,ind]
 
+        if inplace:
+            return None
+        return new_navdata
 
     def loop_time(self, time_row, delta_t_decimals=2):
         """Generator object to loop over columns from same times.

gnss_lib_py/utils/sv_models.py

@@ -314,7 +314,7 @@ def _filter_ephemeris_measurements(measurements, constellations, ephemeris_path)
     req_const_set = set(constellations)
     keep_consts = req_const_set.intersection(rx_const_set)
 
-    measurements_subset = measurements.keep_cols_where('gnss_id', keep_consts, condition="eq")
+    measurements_subset = measurements.where('gnss_id', keep_consts, condition="eq")
 
     # preprocessing of received quantities for downloading ephemeris file
     eph_sv = _combine_gnss_sv_ids(measurements)
@@ -381,7 +381,7 @@ def _sort_ephem_measures(measure_frame, ephem):
     sorted_sats_ind = np.argsort(gnss_sv_id)
     inv_sort_order = np.argsort(sorted_sats_ind)
     sorted_sats = gnss_sv_id[sorted_sats_ind]
-    rx_ephem = ephem.keep_cols_where('gnss_sv_id', sorted_sats, condition="eq")
+    rx_ephem = ephem.where('gnss_sv_id', sorted_sats, condition="eq")
     return rx_ephem, sorted_sats_ind, inv_sort_order
 
 

tests/parsers/test_navdata.py

@@ -1756,6 +1756,9 @@ def test_where_errors(csv_simple):
     # Test condition that is not defined
     with pytest.raises(ValueError):
         _ = data.where("integers", 10, condition="eq_sqrt")
+    # Test passing float in for string check
+    with pytest.raises(ValueError):
+        _ = data.where("names", 0.342, condition="eq")
 
 def test_time_looping(csv_simple):
     """Testing implementation to loop over times
@@ -2265,16 +2268,34 @@ def test_interpolate_fails():
 
 
 def test_keep_cols_where(data, df_simple):
+    """Test keep columns with where.
+
+    """
+    # test for strings
     keep_cols = ['gps', 'glonass']
-    data_subset = data.keep_cols_where('strings', keep_cols,
+
+    data_subset = data.where('strings', keep_cols,
                                         condition="eq")
     df_simple_subset = df_simple.loc[df_simple['strings'].isin(keep_cols), :]
 
     df_simple_subset = df_simple_subset.reset_index(drop=True)
     pd.testing.assert_frame_equal(data_subset.pandas_df(), df_simple_subset, check_dtype=False)
 
+    # test for floats
+    keep_cols = [0.5, 0.45]
+
+    data_subset = data.where('floats', keep_cols,
+                                        condition="neq")
+    df_simple_subset = df_simple.loc[~df_simple['floats'].isin(keep_cols), :]
+
+    df_simple_subset = df_simple_subset.reset_index(drop=True)
+    pd.testing.assert_frame_equal(data_subset.pandas_df(), df_simple_subset, check_dtype=False)
 
 def test_sort(data, df_simple):
+    """Test sorting function across simple dataframe.
+
+    """
+
     df_sorted_int = df_simple.sort_values('integers').reset_index(drop=True)
     df_sorted_float = df_simple.sort_values('floats').reset_index(drop=True)
     data_sorted_int = data.sort('integers').pandas_df()
@@ -2284,10 +2305,22 @@ def test_sort(data, df_simple):
     pd.testing.assert_frame_equal(data_sorted_int, df_sorted_int)
     pd.testing.assert_frame_equal(df_sorted_float, data_sorted_float)
     pd.testing.assert_frame_equal(df_sorted_float, data_sorted_ind)
+    # test strings as well:
+    df_sorted_names = df_simple.sort_values('names').reset_index(drop=True)
+    data_sorted_names = data.sort('names').pandas_df()
+    pd.testing.assert_frame_equal(df_sorted_names, data_sorted_names)
+
+    df_sorted_strings = df_simple.sort_values('strings').reset_index(drop=True)
+    data_sorted_strings = data.sort('strings').pandas_df()
+    pd.testing.assert_frame_equal(df_sorted_strings, data_sorted_strings)
+
     # Test usecase when descending order is given
     df_sorted_int_des = df_simple.sort_values('integers', ascending=False).reset_index(drop=True)
-    data_sorted_int_des = data.sort('integers', order="descending").pandas_df()
+    data_sorted_int_des = data.sort('integers', ascending=False).pandas_df()
+    pd.testing.assert_frame_equal(df_sorted_int_des, data_sorted_int_des)
+
+    # test inplace
+    data_sorted_int_des = data.copy()
+    data_sorted_int_des.sort('integers', ascending=False, inplace=True)
+    data_sorted_int_des = data_sorted_int_des.pandas_df()
     pd.testing.assert_frame_equal(df_sorted_int_des, data_sorted_int_des)
-    # Test usecase when incorrect order is given
-    with pytest.raises(RuntimeError):
-        _ = data.sort('integers', order="equality")