Merge pull request #76 from Stanford-NavLab/derek/concat

Derek/concat

Author: sriramyab

gnss_lib_py/parsers/navdata.py

@@ -15,7 +15,25 @@
 
 class NavData():
     """gnss_lib_py specific class for handling data.
-    Uses numpy for speed combined with pandas like intuitive indexing
+
+    Uses numpy for speed combined with pandas like intuitive indexing.
+
+    Can either be initialized empty, with a csv file by setting
+    ``csv_path``, a Pandas DataFrame by setting ``pandas_df`` or by a
+    Numpy array by setting ``numpy_array``.
+
+    Parameters
+    ----------
+    csv_path : string
+        Path to csv file containing data
+    pandas_df : pd.DataFrame
+        Data used to initialize NavData instance.
+    numpy_array : np.ndarray
+        Numpy array containing data used to initialize NavData
+        instance.
+    **kwargs : args
+        Additional arguments (e.g. ``sep`` or ``header``) passed into
+        ``pd.read_csv`` if csv_path is not None.
 
     Attributes
     ----------
@@ -89,7 +107,8 @@ def from_pandas_df(self, pandas_df):
 
         Parameters
         ----------
-        pandas_df : pd.DataFrame of data
+        pandas_df : pd.DataFrame
+            Data used to initialize NavData instance.
         """
 
         if not isinstance(pandas_df, pd.DataFrame):
@@ -112,54 +131,112 @@ def from_numpy_array(self, numpy_array):
         Parameters
         ----------
         numpy_array : np.ndarray
-            Numpy array containing data
+            Numpy array containing data used to initialize NavData
+            instance.
 
         """
 
+
         if not isinstance(numpy_array, np.ndarray):
             raise TypeError("numpy_array must be np.ndarray")
 
+
         self._build_navdata()
 
+        numpy_array = np.atleast_2d(numpy_array)
         for row_num in range(numpy_array.shape[0]):
             self[str(row_num)] = numpy_array[row_num,:]
 
-    def add(self, csv_path=None, pandas_df=None, numpy_array=None):
-        """Add new timesteps to existing array
+    def concat(self, navdata=None, axis=1, inplace=False):
+        """Concatenates second NavData instance by row or column.
+
+        Concatenates a second NavData instance to the existing NavData
+        instance by either row or column.
+
+        Each type of data is included in a row, so adding new rows with
+        ``axis=0``, means adding new types of data. Concat requires that
+        the new NavData matches the length of the existing NavData. Row
+        concatenation assumes the same ordering across both NavData
+        instances (e.g. sorted by timestamp) and does not perform any
+        matching/sorting itself.
+
+        You can also concatenate new columns ``axis=1``. If the row
+        names of the new NavData instance don't match the row names of
+        the existing NavData instance, the mismatched values will be
+        filled with np.nan.
 
         Parameters
         ----------
-        csv_path : string
-            Path to csv file containing data to add
-        pandas_df : pd.DataFrame
-            DataFrame containing data to add
-        numpy_array : np.ndarray
-            Array containing only numeric data to add
+        navdata : gnss_lib_py.parsers.navdata.NavData
+            Navdata instance to concatenate.
+        axis : int
+            Either add new rows (type) of data ``axis=0`` or new columns
+            (e.g. timesteps) of data ``axis=1``.
+        inplace : bool
+            If False, will return new concatenated NavData instance.
+            If True, will concatenate data to the current NavData
+            instance.
+
+        Returns
+        -------
+        new_navdata : gnss_lib_py.parsers.navdata.NavData or None
+            If inplace is False, returns NavData instance after
+            concatenating specified data. If inplace is True, returns
+            None.
+
         """
-        old_row_num = len(self.map)
-        old_len = len(self)
-        new_data_cols = slice(old_len, None)
-        if numpy_array is not None:
-            if old_row_num == 0:
-                self.from_numpy_array(numpy_array)
-            else:
-                if len(numpy_array.shape)==1:
-                    numpy_array = np.reshape(numpy_array, [1, -1])
-                self.array = np.hstack((self.array, np.empty_like(numpy_array,
-                                        dtype=self.arr_dtype)))
-                self[:, new_data_cols] = numpy_array
-        if csv_path is not None:
-            if old_row_num == 0:
-                self.from_csv_path(csv_path)
-            else:
-                pandas_df = pd.read_csv(csv_path)
-        if pandas_df is not None:
-            if old_row_num == 0:
-                self.from_pandas_df(pandas_df)
-            else:
-                self.array = np.hstack((self.array, np.empty(pandas_df.shape).T))
-                for col in pandas_df.columns:
-                    self[col, new_data_cols] = np.asarray(pandas_df[col].values)
+
+        if not isinstance(navdata,NavData):
+            raise TypeError("concat input data must be a NavData instance.")
+
+        if axis == 0: # concatenate new rows
+            if len(self) != len(navdata):
+                raise RuntimeError("concat input data must be same " \
+                                 + "length to concatenate new rows.")
+            if not inplace:
+                new_navdata = self.copy()
+            for row in navdata.rows:
+                new_row_name = row
+                suffix = None
+                while new_row_name in self.rows:
+                    if suffix is None:
+                        suffix = 0
+                    else:
+                        suffix += 1
+                    new_row_name = row + "_" + str(suffix)
+                if inplace:
+                    self[new_row_name] = navdata[row]
+                else:
+                    new_navdata[new_row_name] = navdata[row]
+
+        elif axis == 1: # concatenate new columns
+            new_navdata = NavData()
+            # get unique list of row names
+            combined_rows = set(self.rows + navdata.rows)
+
+            for row in combined_rows:
+                combined_row = np.array([])
+                # combine data from existing and new instance
+                for data in [self, navdata]:
+                    if row in data.rows:
+                        new_row = data[row]
+                    elif len(data) == 0:
+                        continue
+                    else:
+                        # add np.nan for missing values
+                        new_row = np.empty((len(data),))
+                        new_row.fill(np.nan)
+                    combined_row = np.concatenate((combined_row,
+                                                   new_row))
+                new_navdata[row] = combined_row
+            if inplace:
+                self.array = new_navdata.array
+                self.map = new_navdata.map
+                self.str_map = new_navdata.str_map
+
+        if inplace:
+            return None
+        return new_navdata
 
     def where(self, key_idx, value, condition="eq"):
         """Return NavData where conditions are met for the given row

notebooks/tutorials/navdata.ipynb

@@ -313,8 +313,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "To add new columns, use the `NavData.add()` method with the `csv_path`, \n",
-    "`pandas_df` and `numpy_array` flags depending on the input type."
+    "To add new columns, use the `NavData.concat()` method which concatenates two `NavData` instances."
    ]
   },
   {
@@ -341,7 +340,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "nav_data_np.add(numpy_array=np_array)\n",
+    "nav_data_np.concat(NavData(numpy_array=np_array),inplace=True)\n",
     "nav_data_np[:]"
    ]
   },

tests/parsers/test_android.py

@@ -291,6 +291,42 @@ def test_get_and_set_str(derived):
 
     np.testing.assert_equal(derived[key, :], value)
 
+def test_android_concat(derived, pd_df):
+    """Test concat on Android data.
+
+    Parameters
+    ----------
+    derived : AndroidDerived2021
+        Instance of AndroidDerived2021 for testing
+    pd_df : pytest.fixture
+        pd.DataFrame for testing measurements
+    """
+
+    # remove first timestamp to match
+    pd_df = pd_df[pd_df['millisSinceGpsEpoch'] != pd_df.loc[0,'millisSinceGpsEpoch']]
+
+    # extract and combine gps and glonass data
+    gps_data = derived.where("gnss_id","gps")
+    glonass_data = derived.where("gnss_id","glonass")
+    gps_glonass_navdata = gps_data.concat(glonass_data)
+    glonass_gps_navdata = glonass_data.concat(gps_data)
+
+    # combine using pandas
+    gps_df = pd_df[pd_df["constellationType"]==1]
+    glonass_df = pd_df[pd_df["constellationType"]==3]
+    gps_glonass_df = pd.concat((gps_df,glonass_df))
+    glonass_gps_df = pd.concat((glonass_df,gps_df))
+
+    for combined_navdata, combined_df in [(gps_glonass_navdata, gps_glonass_df),
+                                          (glonass_gps_navdata, glonass_gps_df)]:
+
+        # check a few rows to make sure they're equal
+        np.testing.assert_array_equal(combined_navdata["raw_pr_m"],
+                                      combined_df["rawPrM"])
+        np.testing.assert_array_equal(combined_navdata["raw_pr_sigma_m"],
+                                      combined_df["rawPrUncM"])
+        np.testing.assert_array_equal(combined_navdata["intersignal_bias_m"],
+                                      combined_df["isrbM"])
 
 def test_imu_raw(android_raw_path):
     """Test that AndroidRawImu initialization