-
Notifications
You must be signed in to change notification settings - Fork 138
impl(bigquery): parse date and time related types #6033
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,6 +14,18 @@ | |
|
|
||
| use crate::error::ConvertError; | ||
|
|
||
| pub(crate) const BIGQUERY_DATE_FORMAT: &[time::format_description::FormatItem<'static>] = | ||
| time::macros::format_description!("[year]-[month]-[day]"); | ||
| pub(crate) const BIGQUERY_TIME_FORMAT: &[time::format_description::FormatItem<'static>] = | ||
| time::macros::format_description!("[hour]:[minute]:[second]"); | ||
| pub(crate) const BIGQUERY_TIME_SUBSEC_FORMAT: &[time::format_description::FormatItem<'static>] = | ||
| time::macros::format_description!("[hour]:[minute]:[second].[subsecond]"); | ||
| pub(crate) const BIGQUERY_DATETIME_FORMAT: &[time::format_description::FormatItem<'static>] = | ||
| time::macros::format_description!("[year]-[month]-[day]T[hour]:[minute]:[second]"); | ||
| pub(crate) const BIGQUERY_DATETIME_SUBSEC_FORMAT: &[time::format_description::FormatItem< | ||
| 'static, | ||
| >] = time::macros::format_description!("[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond]"); | ||
|
|
||
| /// Converts BigQuery internal [wkt::Value] to Rust types. | ||
| pub trait FromSql: Sized { | ||
| /// Converts a BigQuery `wkt::Value` into the implementing type. | ||
|
|
@@ -126,8 +138,115 @@ impl FromSql for wkt::Struct { | |
| } | ||
| } | ||
|
|
||
| impl FromSql for wkt::Timestamp { | ||
| fn from_sql(value: wkt::Value) -> Result<Self, ConvertError> { | ||
| match value { | ||
| wkt::Value::String(s) => { | ||
| let micros = s | ||
| .parse::<i64>() | ||
| .map_err(|e| ConvertError::Convert(Box::new(e)))?; | ||
| timestamp_from_micros(micros) | ||
| } | ||
| wkt::Value::Number(n) => { | ||
| let micros = n.as_i64().ok_or_else(|| { | ||
| ConvertError::Convert("timestamp number is not valid i64".into()) | ||
| })?; | ||
| timestamp_from_micros(micros) | ||
| } | ||
| wkt::Value::Null => Err(ConvertError::NotNull), | ||
| other => Err(ConvertError::TypeMismatch { | ||
| expected: "string or number", | ||
| got: other, | ||
| }), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| fn timestamp_from_micros(micros: i64) -> Result<wkt::Timestamp, ConvertError> { | ||
| wkt::Timestamp::new( | ||
| micros.div_euclid(1_000_000), | ||
| (micros.rem_euclid(1_000_000) * 1_000) as i32, | ||
| ) | ||
| .map_err(|e| ConvertError::Convert(Box::new(e))) | ||
| } | ||
|
|
||
| impl FromSql for google_cloud_type::model::Date { | ||
| fn from_sql(value: wkt::Value) -> Result<Self, ConvertError> { | ||
| match value { | ||
| wkt::Value::String(s) => { | ||
| let date = time::Date::parse(s.as_str(), BIGQUERY_DATE_FORMAT) | ||
| .map_err(|e| ConvertError::Convert(Box::new(e)))?; | ||
| Ok(google_cloud_type::model::Date::new() | ||
| .set_year(date.year()) | ||
| .set_month(u8::from(date.month()) as i32) | ||
| .set_day(date.day() as i32)) | ||
| } | ||
| wkt::Value::Null => Err(ConvertError::NotNull), | ||
| other => Err(ConvertError::TypeMismatch { | ||
| expected: "string", | ||
| got: other, | ||
| }), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| impl FromSql for google_cloud_type::model::TimeOfDay { | ||
| fn from_sql(value: wkt::Value) -> Result<Self, ConvertError> { | ||
| match value { | ||
| wkt::Value::String(s) => { | ||
| let format = if s.contains('.') { | ||
| BIGQUERY_TIME_SUBSEC_FORMAT | ||
| } else { | ||
| BIGQUERY_TIME_FORMAT | ||
| }; | ||
| let t = time::Time::parse(s.as_str(), format) | ||
| .map_err(|e| ConvertError::Convert(Box::new(e)))?; | ||
| Ok(google_cloud_type::model::TimeOfDay::new() | ||
| .set_hours(t.hour() as i32) | ||
| .set_minutes(t.minute() as i32) | ||
| .set_seconds(t.second() as i32) | ||
| .set_nanos(t.nanosecond() as i32)) | ||
| } | ||
| wkt::Value::Null => Err(ConvertError::NotNull), | ||
| other => Err(ConvertError::TypeMismatch { | ||
| expected: "string", | ||
| got: other, | ||
| }), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| impl FromSql for google_cloud_type::model::DateTime { | ||
| fn from_sql(value: wkt::Value) -> Result<Self, ConvertError> { | ||
| match value { | ||
| wkt::Value::String(s) => { | ||
| let format = if s.contains('.') { | ||
| BIGQUERY_DATETIME_SUBSEC_FORMAT | ||
| } else { | ||
| BIGQUERY_DATETIME_FORMAT | ||
| }; | ||
| let dt = time::PrimitiveDateTime::parse(s.as_str(), format) | ||
| .map_err(|e| ConvertError::Convert(Box::new(e)))?; | ||
| Ok(google_cloud_type::model::DateTime::new() | ||
| .set_year(dt.year()) | ||
| .set_month(u8::from(dt.month()) as i32) | ||
| .set_day(dt.day() as i32) | ||
| .set_hours(dt.hour() as i32) | ||
| .set_minutes(dt.minute() as i32) | ||
| .set_seconds(dt.second() as i32) | ||
| .set_nanos(dt.nanosecond() as i32)) | ||
| } | ||
| wkt::Value::Null => Err(ConvertError::NotNull), | ||
| other => Err(ConvertError::TypeMismatch { | ||
| expected: "string", | ||
| got: other, | ||
| }), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // TODO(#5592): implement for more Rust | ||
| // types: f32, i32, Decimal, wkt::Timestamp, etc. | ||
| // types: f32, i32, Decimal, etc. | ||
|
|
||
| #[cfg(test)] | ||
| mod tests { | ||
|
|
@@ -216,4 +335,45 @@ mod tests { | |
| fn test_from_sql_struct(value: wkt::Value) -> Result<wkt::Struct, TestConvertError> { | ||
| FromSql::from_sql(value).map_err(TestConvertError::from) | ||
| } | ||
|
|
||
| #[test_case(wkt::Value::String("1779982200000000".to_string()) => Ok(wkt::Timestamp::new(1779982200, 0).unwrap()) ; "timestamp micro integer string")] | ||
| #[test_case(wkt::Value::Number(1779982200000000i64.into()) => Ok(wkt::Timestamp::new(1779982200, 0).unwrap()) ; "timestamp micro integer number")] | ||
|
alvarowolfx marked this conversation as resolved.
|
||
| #[test_case(wkt::Value::String("2026-05-28T15:30:00Z".to_string()) => Err(TestConvertError::Convert("invalid digit found in string".to_string())) ; "timestamp rfc3339 string fails")] | ||
| #[test_case(wkt::Value::Number(serde_json::Number::from_f64(1779982200.5).unwrap()) => Err(TestConvertError::Convert("timestamp number is not valid i64".to_string())) ; "timestamp f64 number fails")] | ||
| #[test_case(wkt::Value::Null => Err(TestConvertError::NotNull) ; "timestamp null")] | ||
| #[test_case(wkt::Value::Bool(true) => Err(TestConvertError::TypeMismatch("string or number")) ; "timestamp type mismatch")] | ||
| fn test_from_sql_timestamp(value: wkt::Value) -> Result<wkt::Timestamp, TestConvertError> { | ||
| FromSql::from_sql(value).map_err(TestConvertError::from) | ||
| } | ||
|
|
||
| #[test_case(wkt::Value::String("2026-05-28".to_string()) => Ok(google_cloud_type::model::Date::new().set_year(2026).set_month(5).set_day(28)) ; "date valid")] | ||
| #[test_case(wkt::Value::Null => Err(TestConvertError::NotNull) ; "date null")] | ||
| #[test_case(wkt::Value::Number(123.into()) => Err(TestConvertError::TypeMismatch("string")) ; "date type mismatch")] | ||
| #[test_case(wkt::Value::String("invalid-date".to_string()) => Err(TestConvertError::Convert("the 'year' component could not be parsed".to_string())) ; "date invalid format")] | ||
| #[test_case(wkt::Value::String("2026-abc-28".to_string()) => Err(TestConvertError::Convert("the 'month' component could not be parsed".to_string())) ; "date invalid digits")] | ||
| fn test_from_sql_date( | ||
| value: wkt::Value, | ||
| ) -> Result<google_cloud_type::model::Date, TestConvertError> { | ||
| FromSql::from_sql(value).map_err(TestConvertError::from) | ||
| } | ||
|
|
||
| #[test_case(wkt::Value::String("15:30:00".to_string()) => Ok(google_cloud_type::model::TimeOfDay::new().set_hours(15).set_minutes(30).set_seconds(0).set_nanos(0)) ; "time of day valid")] | ||
| #[test_case(wkt::Value::String("15:30:00.123456".to_string()) => Ok(google_cloud_type::model::TimeOfDay::new().set_hours(15).set_minutes(30).set_seconds(0).set_nanos(123_456_000)) ; "time of day fractional")] | ||
| #[test_case(wkt::Value::Null => Err(TestConvertError::NotNull) ; "time of day null")] | ||
| #[test_case(wkt::Value::Number(123.into()) => Err(TestConvertError::TypeMismatch("string")) ; "time of day type mismatch")] | ||
| fn test_from_sql_time_of_day( | ||
| value: wkt::Value, | ||
| ) -> Result<google_cloud_type::model::TimeOfDay, TestConvertError> { | ||
| FromSql::from_sql(value).map_err(TestConvertError::from) | ||
| } | ||
|
|
||
| #[test_case(wkt::Value::String("2026-05-28T15:30:00".to_string()) => Ok(google_cloud_type::model::DateTime::new().set_year(2026).set_month(5).set_day(28).set_hours(15).set_minutes(30).set_seconds(0).set_nanos(0)) ; "datetime without subseconds")] | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. TIL: This line of code is absurdly long, but it's fine. |
||
| #[test_case(wkt::Value::String("2026-05-28T15:30:00.123456".to_string()) => Ok(google_cloud_type::model::DateTime::new().set_year(2026).set_month(5).set_day(28).set_hours(15).set_minutes(30).set_seconds(0).set_nanos(123_456_000)) ; "datetime with subseconds")] | ||
| #[test_case(wkt::Value::Null => Err(TestConvertError::NotNull) ; "datetime null")] | ||
| #[test_case(wkt::Value::Number(123.into()) => Err(TestConvertError::TypeMismatch("string")) ; "datetime type mismatch")] | ||
| fn test_from_sql_datetime( | ||
| value: wkt::Value, | ||
| ) -> Result<google_cloud_type::model::DateTime, TestConvertError> { | ||
| FromSql::from_sql(value).map_err(TestConvertError::from) | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -249,6 +249,83 @@ pub async fn query_client() -> Result<()> { | |
| Ok(()) | ||
| } | ||
|
|
||
| pub async fn query_client_datatypes() -> Result<()> { | ||
| let project_id = project_id()?; | ||
| let bq = BigQuery::builder().build().await?; | ||
|
|
||
| let query = bq | ||
| .query( | ||
| "SELECT \ | ||
| 'John Doe' AS name, \ | ||
| 30 AS age, \ | ||
| 1.85 AS height, \ | ||
| true AS active, \ | ||
| TIMESTAMP '2026-05-28 15:30:00 UTC' AS created_at, \ | ||
| DATE '2026-05-28' AS birth_date, \ | ||
| TIME '15:30:00' AS daily_alarm, \ | ||
| DATETIME '2026-05-28 15:30:00' AS event_time, \ | ||
|
Comment on lines
+263
to
+266
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nice. |
||
| CAST(NULL AS STRING) AS nullable_name, \ | ||
| CAST(NULL AS INT64) AS nullable_age", | ||
| ) | ||
| .with_project_id(project_id) | ||
| .set_labels(vec![(INSTANCE_LABEL, "true")]) | ||
| .run() | ||
| .await?; | ||
|
|
||
| let complete_query = query.until_done().await?; | ||
| assert_eq!(complete_query.metadata().total_rows, Some(1)); | ||
|
|
||
| let mut iter = complete_query.read(); | ||
| let row = iter.next().await.expect("row must exist")?; | ||
|
|
||
| assert_eq!(row.get::<String, _>("name"), "John Doe"); | ||
| assert_eq!(row.get::<i64, _>("age"), 30); | ||
| assert_eq!(row.get::<f64, _>("height"), 1.85); | ||
| assert!(row.get::<bool, _>("active")); | ||
|
|
||
| let created_at = row.get::<wkt::Timestamp, _>("created_at"); | ||
| assert_eq!(created_at, wkt::Timestamp::new(1779982200, 0).unwrap()); | ||
|
|
||
| let birth_date = row.get::<google_cloud_type::model::Date, _>("birth_date"); | ||
| assert_eq!( | ||
| birth_date, | ||
| google_cloud_type::model::Date::new() | ||
| .set_year(2026) | ||
| .set_month(5) | ||
| .set_day(28) | ||
| ); | ||
|
|
||
| let daily_alarm = row.get::<google_cloud_type::model::TimeOfDay, _>("daily_alarm"); | ||
| assert_eq!( | ||
| daily_alarm, | ||
| google_cloud_type::model::TimeOfDay::new() | ||
| .set_hours(15) | ||
| .set_minutes(30) | ||
| .set_seconds(0) | ||
| .set_nanos(0) | ||
| ); | ||
|
|
||
| let event_time = row.get::<google_cloud_type::model::DateTime, _>("event_time"); | ||
| assert_eq!( | ||
| event_time, | ||
| google_cloud_type::model::DateTime::new() | ||
| .set_year(2026) | ||
| .set_month(5) | ||
| .set_day(28) | ||
| .set_hours(15) | ||
| .set_minutes(30) | ||
| .set_seconds(0) | ||
| .set_nanos(0) | ||
| ); | ||
|
|
||
| assert_eq!(row.get::<Option<String>, _>("nullable_name"), None); | ||
| assert_eq!(row.get::<Option<i64>, _>("nullable_age"), None); | ||
|
|
||
| assert!(iter.next().await.is_none()); | ||
|
|
||
| Ok(()) | ||
| } | ||
|
|
||
| pub async fn query_client_multi_page() -> Result<()> { | ||
| let project_id = project_id()?; | ||
| let bq = BigQuery::builder().build().await?; | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.