refactor: use schemas from tables pallet in proof plan RPCs

pallets/tables/Cargo.toml

@@ -26,6 +26,7 @@ frame-support.workspace = true
 frame-system.workspace = true
 pallet-permissions.workspace = true
 sxt-core.workspace = true
+sp-api.workspace = true
 sp-runtime = {default-features = false, workspace = true}
 sp-core.workspace = true
 pallet-commitments.workspace = true
@@ -44,6 +45,7 @@ std = [
 	"frame-support/std",
 	"frame-system/std",
 	"scale-info/std",
+	"sp-api/std",
 	"pallet-commitments/std",
 ]
 runtime-benchmarks = [

pallets/tables/src/lib.rs

@@ -16,6 +16,8 @@ mod mock;
 #[cfg(test)]
 mod tests;
 
+pub mod runtime_api;
+
 pub mod weights;
 pub use weights::*;
 
@@ -50,13 +52,15 @@ pub mod pallet {
         generate_namespace_uuid,
         generate_table_uuid,
         sqlparser_to_create_statement,
+        table_schema_from_create_statement,
         update_uuid_in_create_table_statement,
         uuids_from_create_statement,
         uuids_from_sqlparser,
         ColumnUuidList,
         CommitmentBytes,
         CommitmentScheme,
         CreateStatement,
+        GetTableSchemaError,
         IdentifierList,
         IndexerMode,
         InsertQuorumSize,
@@ -66,6 +70,7 @@ pub mod pallet {
         TableIdentifier,
         TableName,
         TableNamespace,
+        TableSchema,
         TableType,
         TableUuid,
         TableVersion,
@@ -849,5 +854,17 @@ pub mod pallet {
             Self::deposit_event(Event::<T>::SchemaUpdated(owner, tables_with_meta_columns));
             Ok(())
         }
+
+        /// Returns the schema for the given table identifier.
+        pub fn table_schema(
+            table_identifier: TableIdentifier,
+        ) -> Result<TableSchema, GetTableSchemaError> {
+            let create_statement = Self::schemas(table_identifier.namespace, table_identifier.name)
+                .ok_or(GetTableSchemaError::NoSuchTable)?;
+
+            let table_schema = table_schema_from_create_statement(create_statement)?;
+
+            Ok(table_schema)
+        }
     }
 }

pallets/tables/src/runtime_api.rs

@@ -0,0 +1,12 @@
+//! Runtime APIs for reading from pallet-tables.
+
+use sxt_core::tables::{GetTableSchemaError, TableIdentifier, TableSchema};
+
+sp_api::decl_runtime_apis! {
+    /// Runtime APIs for reading from pallet-tables.
+    pub trait TablesApi {
+        /// Returns the schema for the given table identifier, in the form of a simple mapping
+        /// between column name and type.
+        fn table_schema(table_identifier: TableIdentifier) -> Result<TableSchema, GetTableSchemaError>;
+    }
+}

pallets/tables/src/tests.rs

@@ -6,9 +6,12 @@ use pallet_permissions::Pallet;
 use proof_of_sql_commitment_map::CommitmentSchemeFlags;
 use sp_core::ConstU32;
 use sp_runtime::BoundedVec;
+use sqlparser::ast::{ColumnDef, DataType, ExactNumberInfo, Ident, TimezoneInfo};
 use sxt_core::permissions::{PermissionLevel, PermissionList, TablesPalletPermission};
 use sxt_core::tables::{
     CreateStatement,
+    GetTableSchemaError,
+    ScaleColumnSchema,
     Source,
     SourceAndMode,
     TableIdentifier,
@@ -435,3 +438,69 @@ fn test_get_or_generate_uuids_for_table_generates_uuids_if_missing() {
         println!("✅ Column UUIDs: {:?}", column_uuids);
     });
 }
+
+#[test]
+fn we_can_get_table_schemas() {
+    new_test_ext().execute_with(|| {
+        let test_identifier = TableIdentifier {
+            name: b"BLOCKS".to_vec().try_into().unwrap(),
+            namespace: b"ETHEREUM".to_vec().try_into().unwrap(),
+        };
+
+        assert!(matches!(
+            Tables::table_schema(test_identifier.clone()),
+            Err(GetTableSchemaError::NoSuchTable)
+        ));
+
+        let ddl = r#"CREATE TABLE IF NOT EXISTS ETHEREUM.BLOCKS (
+            TIME_STAMP TIMESTAMP NOT NULL,
+            BLOCK_NUMBER BIGINT NOT NULL,
+            BLOCK_HASH BINARY NOT NULL,
+            GAS_LIMIT DECIMAL(75, 0) NOT NULL,
+            TRANSACTION_COUNT INT NOT NULL,
+            PRIMARY KEY (BLOCK_NUMBER)
+        ) WITH (TABLE_UUID=F801A872785FAB3F16C51CF7A1969000);"#;
+
+        let create_statement: CreateStatement =
+            BoundedVec::try_from(ddl.as_bytes().to_vec()).expect("DDL should fit in BoundedVec");
+
+        let tables: UpdateTableList = BoundedVec::try_from(vec![UpdateTable {
+            ident: test_identifier.clone(),
+            create_statement: create_statement.clone(),
+            table_type: TableType::CoreBlockchain,
+            commitment: CommitmentCreationCmd::Empty(CommitmentSchemeFlags::default()),
+            source: Source::Ethereum,
+        }])
+        .expect("Table list should fit in BoundedVec");
+
+        Tables::create_tables(RuntimeOrigin::root(), tables.clone()).unwrap();
+
+        let table_schema = Tables::table_schema(test_identifier)
+            .unwrap()
+            .into_iter()
+            .map(|column_schema| ColumnDef::try_from(column_schema).unwrap())
+            .collect::<Vec<_>>();
+
+        let expected_columns = [
+            (
+                Ident::new("TIME_STAMP"),
+                DataType::Timestamp(None, TimezoneInfo::None),
+            ),
+            (Ident::new("BLOCK_NUMBER"), DataType::BigInt(None)),
+            (Ident::new("BLOCK_HASH"), DataType::Binary(None)),
+            (
+                Ident::new("GAS_LIMIT"),
+                DataType::Decimal(ExactNumberInfo::PrecisionAndScale(75, 0)),
+            ),
+            (Ident::new("TRANSACTION_COUNT"), DataType::Int(None)),
+        ]
+        .map(|(name, data_type)| ColumnDef {
+            name,
+            data_type,
+            options: Vec::new(),
+            collation: None,
+        });
+
+        assert_eq!(table_schema, expected_columns);
+    });
+}

rpc/Cargo.toml

@@ -30,6 +30,7 @@ pallet-system-contracts = { workspace = true, default-features = true }
 proof-of-sql = { workspace = true }
 proof-of-sql-planner = { workspace = true }
 proof-of-sql-commitment-map = { workspace = true }
+proof-of-sql-unversioned = { workspace = true }
 sc-chain-spec = { workspace = true, default-features = true }
 sc-client-api = { workspace = true, default-features = true }
 sc-consensus-babe = { workspace = true, default-features = true }
@@ -59,7 +60,4 @@ sxt-runtime = { workspace = true, default-features = false, features = ["std"] }
 hex.workspace = true
 
 [dev-dependencies]
-proof-of-sql-static-setups = { workspace = true, features = ["io"] }
-commitment-sql = { workspace = true }
-on-chain-table.workspace = true
 itertools.workspace = true

rpc/src/commitments/api_impl.rs

@@ -11,13 +11,12 @@ use attestation_tree::{
 };
 use codec::Decode;
 use frame_support::traits::StorageInstance;
-use pallet_commitments::runtime_api::CommitmentsApi;
 use pallet_system_contracts::_GeneratedPrefixForStorageStakingContract;
 use proof_of_sql::sql::evm_proof_plan::EVMProofPlan;
 use proof_of_sql::sql::proof::ProofPlan;
 use proof_of_sql::sql::proof_plans::DynProofPlan;
 use proof_of_sql_commitment_map::{CommitmentScheme, TableCommitmentBytes};
-use proof_of_sql_planner::statement_with_uppercase_identifiers;
+use proof_of_sql_planner::{get_table_refs_from_statement, statement_with_uppercase_identifiers};
 use sc_client_api::{Backend as BackendT, StorageKey, StorageProvider};
 use sp_api::ProvideRuntimeApi;
 use sp_blockchain::HeaderBackend;
@@ -27,10 +26,11 @@ use sqlparser::dialect::GenericDialect;
 use sqlparser::parser::Parser;
 use sxt_core::tables::TableIdentifier;
 use sxt_core::utils::proof_of_sql_bincode_config;
-use sxt_runtime::pallet_commitments;
+use sxt_runtime::pallet_tables;
+use sxt_runtime::pallet_tables::runtime_api::TablesApi;
 
-use super::proof_plan_for_query_and_commitments::ProofPlanForQueryAndCommitments;
-use super::statement_and_associated_table_refs::StatementAndAssociatedTableRefs;
+use super::proof_plan_no_normalization::proof_plan_no_normalization;
+use super::query_schema::QuerySchema;
 use crate::commitments::api::{
     ProofPlanResponse,
     VerifiableCommitment,
@@ -85,12 +85,12 @@ where
         + StorageProvider<Block, Backend>
         + ProvideRuntimeApi<Block>
         + 'static,
-    Client::Api: pallet_commitments::runtime_api::CommitmentsApi<Block>,
+    Client::Api: TablesApi<Block>,
     Backend: BackendT<Block> + 'static,
     Block: BlockT + 'static,
     Config: Send
         + Sync
-        + pallet_commitments::Config
+        + pallet_tables::Config
         + pallet_balances::Config<(), Balance = u128>
         + pallet_system_contracts::Config
         + 'static,
@@ -225,34 +225,31 @@ where
 
         let statement = statement_with_uppercase_identifiers(statement);
 
-        let statement_and_associated_table_refs =
-            StatementAndAssociatedTableRefs::try_from(statement)?;
+        let table_refs = get_table_refs_from_statement(&statement)?;
 
-        let num_tables = statement_and_associated_table_refs.table_refs().len();
+        let num_tables = table_refs.len();
         if num_tables > NUM_TABLES_LIMIT {
             return Err(CommitmentsApiError::NumTablesLimit { num_tables });
         }
 
-        let table_identifiers = statement_and_associated_table_refs
-            .table_refs()
-            .iter()
-            .cloned()
-            .map(TableIdentifier::try_from)
-            .collect::<Result<Vec<_>, _>>()?
-            .try_into()
-            .expect("We've already verified that there are fewer than 64 tables");
-
         let at = at.unwrap_or_else(|| self.client.info().best_hash);
 
-        let proof_plan = self
-            .client
-            .runtime_api()
-            .table_commitments_any_scheme(at, table_identifiers)?
-            .ok_or(CommitmentsApiError::IncompleteCommitmentCoverage)?
-            .map(ProofPlanForQueryAndCommitments(
-                statement_and_associated_table_refs,
-            ))
-            .unwrap()?;
+        let table_schemas = table_refs
+            .into_iter()
+            .map(|table_ref| {
+                let table_identifier = TableIdentifier::try_from(table_ref.clone())?;
+
+                let table_schema = self
+                    .client
+                    .runtime_api()
+                    .table_schema(at, table_identifier)??;
+                Ok((table_ref, table_schema))
+            })
+            .collect::<Result<Vec<_>, CommitmentsApiError>>()?;
+
+        let query_schema = QuerySchema::try_from_table_schemas(table_schemas)?;
+
+        let proof_plan = proof_plan_no_normalization(&statement, &query_schema)?;
 
         let proof_plan_bytes = bincode::serde::encode_to_vec(
             &proof_plan,

rpc/src/commitments/error.rs

@@ -2,8 +2,9 @@ use attestation_tree::{AttestationTreeError, AttestationTreeProofError};
 use jsonrpsee::types::ErrorObjectOwned;
 use proof_of_sql_planner::PlannerError;
 use snafu::Snafu;
-use sxt_core::tables::TableIdentifierConversionError;
+use sxt_core::tables::{GetTableSchemaError, TableIdentifierConversionError};
 
+use super::query_schema::TableToProofOfSqlSchemaError;
 use crate::commitments::limits::{NUM_TABLES_LIMIT, PROOF_PLAN_SIZE_LIMIT, QUERY_SIZE_LIMIT};
 
 /// The base error code used by the commitments RPCs.
@@ -106,18 +107,6 @@ pub enum CommitmentsApiError {
         /// The source runtime api error.
         source: sp_api::ApiError,
     },
-    /// Unexpected table to commitment mismap.
-    #[snafu(display("unexpected table ref to commitment mismap, statement has {num_tables} tables but runtime api returned {num_commitments} commitments"))]
-    UnexpectedTableCommitmentMismap {
-        num_tables: usize,
-        num_commitments: usize,
-    },
-    /// Failed to deserialize table commitment.
-    #[snafu(display("failed to deserialize table commitment: {source}"))]
-    DeserializeTableCommitment {
-        /// The source bincode error.
-        source: bincode::error::DecodeError,
-    },
     /// Encountered error in proof-of-sql planner.
     #[snafu(
         display("encountered error in proof-of-sql planner: {source}"),
@@ -127,17 +116,39 @@ pub enum CommitmentsApiError {
         /// The osource planner error.
         source: PlannerError,
     },
-    /// Tables do not exist or have incomplete commitment coverage for all schemes.
-    #[snafu(display(
-        "tables do not exist or have incomplete commitment coverage for all schemes"
-    ))]
-    IncompleteCommitmentCoverage,
     /// Failed to encode proof plan.
     #[snafu(display("failed to encode proof plan: {source}"), context(false))]
     EncodeProofPlan {
         /// The source bincode error.
         source: bincode::error::EncodeError,
     },
+    /// No such table
+    #[snafu(display("no such table"))]
+    NoSuchTable,
+    /// Invalid table metadata in storage for
+    #[snafu(display("invalid table metadata in storage: {error}"))]
+    InvalidTableSchema {
+        /// The source error.
+        error: GetTableSchemaError,
+    },
+    /// Unable to convert on-chain schema to proof-of-sql schema.
+    #[snafu(
+        display("unable to convert on-chain schema to proof-of-sql schema: {source}"),
+        context(false)
+    )]
+    ProofOfSqlSchemaConversion {
+        /// The source conversion error
+        source: TableToProofOfSqlSchemaError,
+    },
+}
+
+impl From<GetTableSchemaError> for CommitmentsApiError {
+    fn from(error: GetTableSchemaError) -> Self {
+        match error {
+            GetTableSchemaError::NoSuchTable => CommitmentsApiError::NoSuchTable,
+            error => CommitmentsApiError::InvalidTableSchema { error },
+        }
+    }
 }
 
 impl From<CommitmentsApiError> for ErrorObjectOwned {
@@ -160,11 +171,11 @@ impl From<CommitmentsApiError> for ErrorObjectOwned {
                 CommitmentsApiError::NotOneStatement { .. } => 12,
                 CommitmentsApiError::ProofOfSqlIncompatibleRelation { .. } => 13,
                 CommitmentsApiError::RuntimeApi { .. } => 14,
-                CommitmentsApiError::UnexpectedTableCommitmentMismap { .. } => 15,
-                CommitmentsApiError::DeserializeTableCommitment { .. } => 16,
                 CommitmentsApiError::Planner { .. } => 17,
-                CommitmentsApiError::IncompleteCommitmentCoverage => 18,
                 CommitmentsApiError::EncodeProofPlan { .. } => 19,
+                CommitmentsApiError::NoSuchTable { .. } => 20,
+                CommitmentsApiError::InvalidTableSchema { .. } => 21,
+                CommitmentsApiError::ProofOfSqlSchemaConversion { .. } => 22,
             };
 
         ErrorObjectOwned::owned(code, message, None::<()>)