From 4461b41ce728cdd4b53407b6692ce6615529c83d Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 22:49:05 +1000 Subject: [PATCH 1/4] feat(proxy): infer encrypt config from the EQL v3 schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The proxy's encrypt config is now derived from the database schema instead of the `eql_v2_configuration` table. EQL v3 columns are self-configuring domain types, so `eql_v2.add_search_config` and the config table are redundant — the schema is the single source of truth. - New encrypt_config/from_domain.rs: `column_config_from_domain` builds a ColumnConfig from a column's v3 domain typname — cast type from the token, indexes from the stored SEM terms (verified against cipherstash-client's indexers + eql-bindings v3::terms): hm→Unique, op→Ope, ob→Ore, bf→Match, JSON SteVec→SteVec{Compat}. Scalar non-text `_ord` domains store only `op` (a single Ope index, no HMAC); text carries `hm` alongside its ordering term. Storage-only and non-EQL domains yield no indexes / None. - load_encrypt_config now runs the shared SCHEMA_QUERY (which already returns information_schema.domain_name) and maps each encrypted column via column_config_from_domain. Removes the eql_v2_configuration query, the canonical-JSON path, ENCRYPT_CONFIG_QUERY + select_config.sql, and the MissingEncryptConfigTable startup handling (a load error is now a genuine database failure; an empty encrypted schema is a successful empty config). - The obsolete canonical-config parsing tests are dropped; from_domain.rs carries the equivalent domain→config coverage (9 tests). This unblocks running against a v3-only EQL install (which has no eql_v2_configuration). proxy builds; from_domain 9 tests pass; workspace check, clippy, fmt clean (pre-existing data_row::to_ciphertext_* failures are unrelated). Refs CIP-3595, CIP-3579. Stable-Commit-Id: q-6xh7ejw90r1kxzdd0jmmmkzd79 --- .../src/proxy/encrypt_config/from_domain.rs | 198 +++++++ .../src/proxy/encrypt_config/manager.rs | 554 ++---------------- .../src/proxy/encrypt_config/mod.rs | 1 + packages/cipherstash-proxy/src/proxy/mod.rs | 8 +- .../src/proxy/sql/select_config.sql | 1 - 5 files changed, 241 insertions(+), 521 deletions(-) create mode 100644 packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs delete mode 100644 packages/cipherstash-proxy/src/proxy/sql/select_config.sql diff --git a/packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs b/packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs new file mode 100644 index 00000000..2c6ce371 --- /dev/null +++ b/packages/cipherstash-proxy/src/proxy/encrypt_config/from_domain.rs @@ -0,0 +1,198 @@ +//! Derives the proxy's encrypt configuration from the EQL v3 schema. +//! +//! EQL v3 domain types are self-configuring: a column's Postgres domain (e.g. +//! `eql_v3_text_search`) encodes both the plaintext token type and the +//! searchable-encryption terms it stores. That is enough to build the +//! [`ColumnConfig`] the encrypt pipeline needs, so `eql_v2.add_search_config` +//! and the `eql_v2_configuration` table are redundant. +//! +//! The SEM term → index mapping mirrors the client's indexers (verified against +//! `cipherstash-client`'s encrypt pipeline and `eql-bindings`' `v3::terms`): +//! +//! | term | produced by | index (`add_search_config` name) | +//! |------|-------------|-----------| +//! | `hm` (HMAC) | `UniqueIndexer` | `Unique` (unique) | +//! | `op` (CLLW-OPE) | `OpeIndexer` | `Ope` (ope) | +//! | `ob` (block-ORE) | `OreIndexer` | `Ore` (ore) | +//! | `bf` (bloom) | `MatchIndexer` | `Match` (match) | +//! | (SteVec/JSON) | `JsonIndexer` | `SteVec` (ste_vec) | + +use cipherstash_client::schema::ColumnConfig; +use cipherstash_config::column::{ArrayIndexMode, Index, IndexType, SteVecMode}; +use cipherstash_config::ColumnType; +use eql_mapper::{DomainIdentity, TokenType}; + +/// Build the [`ColumnConfig`] for a column from its EQL v3 domain typname, or +/// `None` if `domain` is not a recognised v3 EQL domain (a plaintext column). +pub(crate) fn column_config_from_domain( + table: &str, + column: &str, + domain: &str, +) -> Option { + let identity = DomainIdentity::from_domain_name(domain)?; + let mut config = + ColumnConfig::build(column.to_string()).casts_as(token_to_column_type(identity.token)); + + if identity.token == TokenType::Json { + // Searchable encrypted JSON is a SteVec index; its terms live per entry, + // not on the domain, so the scalar term flags below do not apply. A + // storage-only `eql_v3_json` column has no searchable index. + if domain.ends_with("_search") { + config = config.add_index(Index::new(IndexType::SteVec { + prefix: format!("{table}/{column}"), + term_filters: Vec::new(), + array_index_mode: ArrayIndexMode::default(), + mode: SteVecMode::default(), // Compat (CLLW-OPE), the v3 default + })); + } + } else { + // Scalar domains: the stored terms map directly to index types. + // `DomainIdentity::stores_*` already encodes the text `hm` exception. + if identity.stores_hm() { + config = config.add_index(Index::new_unique()); + } + if identity.stores_op() { + config = config.add_index(Index::new_ope()); + } + if identity.stores_ob() { + config = config.add_index(Index::new_ore()); + } + if identity.stores_bf() { + config = config.add_index(Index::new_match()); + } + } + + Some(config) +} + +fn token_to_column_type(token: TokenType) -> ColumnType { + match token { + TokenType::SmallInt => ColumnType::SmallInt, + TokenType::Integer => ColumnType::Int, + TokenType::BigInt => ColumnType::BigInt, + TokenType::Real | TokenType::Double => ColumnType::Float, + TokenType::Numeric => ColumnType::Decimal, + TokenType::Text => ColumnType::Text, + TokenType::Boolean => ColumnType::Boolean, + TokenType::Date => ColumnType::Date, + TokenType::Timestamp => ColumnType::Timestamp, + TokenType::Json => ColumnType::Json, + } +} + +#[cfg(test)] +mod test { + use super::*; + + fn config(domain: &str) -> ColumnConfig { + column_config_from_domain("t", "c", domain) + .unwrap_or_else(|| panic!("{domain} did not resolve")) + } + + fn index_types(domain: &str) -> Vec { + config(domain) + .indexes + .into_iter() + .map(|i| i.index_type) + .collect() + } + + fn has(domain: &str, matcher: impl Fn(&IndexType) -> bool) -> bool { + index_types(domain).iter().any(matcher) + } + + #[test] + fn cast_type_comes_from_the_token() { + assert_eq!(config("eql_v3_integer_eq").cast_type, ColumnType::Int); + assert_eq!(config("eql_v3_bigint_eq").cast_type, ColumnType::BigInt); + assert_eq!(config("eql_v3_smallint_eq").cast_type, ColumnType::SmallInt); + assert_eq!(config("eql_v3_double_ord").cast_type, ColumnType::Float); + assert_eq!(config("eql_v3_text_search").cast_type, ColumnType::Text); + assert_eq!(config("eql_v3_boolean").cast_type, ColumnType::Boolean); + assert_eq!(config("eql_v3_date_ord").cast_type, ColumnType::Date); + assert_eq!(config("eql_v3_json_search").cast_type, ColumnType::Json); + } + + #[test] + fn eq_domain_has_a_unique_index() { + assert_eq!( + index_types("eql_v3_integer_eq"), + vec![IndexType::Unique { + token_filters: vec![] + }] + ); + } + + #[test] + fn scalar_ord_uses_ope_only_no_hmac() { + // A non-text `_ord` domain stores only `op` (no `hm`): a single Ope index. + assert_eq!(index_types("eql_v3_integer_ord"), vec![IndexType::Ope]); + // block-ORE ordering -> Ore. + assert_eq!(index_types("eql_v3_integer_ord_ore"), vec![IndexType::Ore]); + } + + #[test] + fn text_ord_carries_unique_plus_ordering() { + // text stores `hm` alongside its ordering term (equality-lossy ORE/OPE). + assert!(has("eql_v3_text_ord", |i| matches!( + i, + IndexType::Unique { .. } + ))); + assert!(has("eql_v3_text_ord", |i| *i == IndexType::Ope)); + assert!(has("eql_v3_text_ord_ore", |i| matches!( + i, + IndexType::Unique { .. } + ))); + assert!(has("eql_v3_text_ord_ore", |i| *i == IndexType::Ore)); + } + + #[test] + fn text_search_has_unique_ope_and_match() { + assert!(has("eql_v3_text_search", |i| matches!( + i, + IndexType::Unique { .. } + ))); + assert!(has("eql_v3_text_search", |i| *i == IndexType::Ope)); + assert!(has("eql_v3_text_search", |i| matches!( + i, + IndexType::Match { .. } + ))); + } + + #[test] + fn match_domain_has_a_match_index() { + assert!(has("eql_v3_text_match", |i| matches!( + i, + IndexType::Match { .. } + ))); + assert!(!has("eql_v3_text_match", |i| matches!( + i, + IndexType::Unique { .. } + ))); + } + + #[test] + fn json_search_has_a_ste_vec_index_in_compat_mode() { + let idx = index_types("eql_v3_json_search"); + assert_eq!(idx.len(), 1); + match &idx[0] { + IndexType::SteVec { mode, .. } => assert_eq!(*mode, SteVecMode::Compat), + other => panic!("expected SteVec, got {other:?}"), + } + } + + #[test] + fn storage_only_domains_have_no_indexes() { + assert!(config("eql_v3_integer").indexes.is_empty()); + assert!(config("eql_v3_boolean").indexes.is_empty()); + // storage-only json (no `_search`) is not searchable. + assert!(config("eql_v3_json").indexes.is_empty()); + } + + #[test] + fn non_eql_domains_do_not_resolve() { + assert!(column_config_from_domain("t", "c", "jsonb").is_none()); + assert!(column_config_from_domain("t", "c", "text").is_none()); + assert!(column_config_from_domain("t", "c", "eql_v2_encrypted").is_none()); + } +} diff --git a/packages/cipherstash-proxy/src/proxy/encrypt_config/manager.rs b/packages/cipherstash-proxy/src/proxy/encrypt_config/manager.rs index f9eff615..05b8721f 100644 --- a/packages/cipherstash-proxy/src/proxy/encrypt_config/manager.rs +++ b/packages/cipherstash-proxy/src/proxy/encrypt_config/manager.rs @@ -1,15 +1,10 @@ +use super::from_domain::column_config_from_domain; use crate::{ - config::DatabaseConfig, - connect, - error::{ConfigError, Error}, - log::ENCRYPT_CONFIG, - proxy::ENCRYPT_CONFIG_QUERY, + config::DatabaseConfig, connect, error::Error, log::ENCRYPT_CONFIG, proxy::SCHEMA_QUERY, }; use arc_swap::ArcSwap; use cipherstash_client::eql; use cipherstash_client::schema::ColumnConfig; -use cipherstash_config::CanonicalEncryptionConfig; -use serde_json::Value; use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::{task::JoinHandle, time}; use tracing::{debug, error, info, warn}; @@ -94,29 +89,15 @@ async fn init_reloader(config: DatabaseConfig) -> Result encrypt_config, Err(err) => { - match err { - // Similar messages are displayed on connection, defined in handler.rs - // Please keep the language in sync when making changes here. - Error::Config(ConfigError::MissingEncryptConfigTable) => { - error!(msg = "No Encrypt configuration table in database."); - warn!(msg = "Encrypt requires the Encrypt Query Language (EQL) to be installed in the target database"); - warn!(msg = "See https://github.com/cipherstash/encrypt-query-language"); - } - Error::Config(ConfigError::InvalidEncryptionConfig(ref inner)) => { - error!( - msg = "Invalid Encrypt configuration in database", - error = inner.to_string() - ); - } - _ => { - error!( - msg = "Error loading Encrypt configuration", - error = err.to_string() - ); - return Err(err); - } - } - EncryptConfig::new() + // Encrypt config is inferred from the schema (EQL v3 self-configuring + // domains), so a load error here is a database/connection failure, not + // a missing config table. A schema with no encrypted columns is a + // successful (empty) load, warned about below. + error!( + msg = "Error loading Encrypt configuration", + error = err.to_string() + ); + return Err(err); } }; @@ -205,500 +186,41 @@ async fn load_encrypt_config_with_retry(config: &DatabaseConfig) -> Result Result { let client = connect::database(config).await?; - match client.query(ENCRYPT_CONFIG_QUERY, &[]).await { - Ok(rows) => { - if rows.is_empty() { - return Ok(EncryptConfig::new()); - }; + let tables = client.query(SCHEMA_QUERY, &[]).await?; - // We know there is at least one row - let row = rows.first().unwrap(); + let mut map = EncryptConfigMap::new(); - let json_value: Value = row.get("data"); - let canonical: CanonicalEncryptionConfig = serde_json::from_value(json_value)?; - let encrypt_config = EncryptConfig::new_from_config(canonical_to_map(canonical)?); + for table in tables { + let table_name: String = table.get("table_name"); + let columns: Vec = table.get("columns"); + let column_domain_names: Vec> = table.get("column_domain_names"); - Ok(encrypt_config) - } - Err(err) => { - if configuration_table_not_found(&err) { - return Err(ConfigError::MissingEncryptConfigTable.into()); + for (column, domain) in columns.iter().zip(column_domain_names) { + let Some(domain) = domain else { continue }; + if let Some(column_config) = column_config_from_domain(&table_name, column, &domain) { + debug!( + target: ENCRYPT_CONFIG, + msg = "Encrypted column", + table = table_name, + column = column, + domain = domain + ); + map.insert( + eql::Identifier::new(table_name.clone(), column.clone()), + column_config, + ); } - Err(ConfigError::Database(err).into()) } } -} -fn configuration_table_not_found(e: &tokio_postgres::Error) -> bool { - let msg = e.to_string(); - msg.contains("eql_v2_configuration") && msg.contains("does not exist") -} - -fn canonical_to_map(canonical: CanonicalEncryptionConfig) -> Result { - Ok(canonical - .into_config_map()? - .into_iter() - .map(|(id, col)| (eql::Identifier::new(id.table, id.column), col)) - .collect()) -} - -#[cfg(test)] -mod tests { - use super::*; - use cipherstash_client::eql::Identifier; - use cipherstash_config::column::{ - ArrayIndexMode, IndexType, SteVecMode, TokenFilter, Tokenizer, - }; - use cipherstash_config::ColumnType; - use serde_json::json; - - fn parse(json: serde_json::Value) -> EncryptConfigMap { - let config: CanonicalEncryptionConfig = serde_json::from_value(json).unwrap(); - canonical_to_map(config).unwrap() - } - - #[test] - fn column_with_empty_options_gets_defaults() { - let json = json!({ - "v": 1, - "tables": { "users": { "email": {} } } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!(column.cast_type, ColumnType::Text); - assert!(column.indexes.is_empty()); - } - - #[test] - fn can_parse_column_with_cast_as() { - let json = json!({ - "v": 1, - "tables": { - "users": { "favourite_int": { "cast_as": "int" } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "favourite_int")) - .unwrap(); - - assert_eq!(column.cast_type, ColumnType::Int); - assert_eq!(column.name, "favourite_int"); - assert!(column.indexes.is_empty()); - } - - #[test] - fn cast_as_real_maps_to_float() { - let json = json!({ - "v": 1, - "tables": { - "users": { "rating": { "cast_as": "real" } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "rating")) - .unwrap(); - - assert_eq!(column.cast_type, ColumnType::Float); - } - - #[test] - fn cast_as_double_maps_to_float() { - let json = json!({ - "v": 1, - "tables": { - "users": { "rating": { "cast_as": "double" } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "rating")) - .unwrap(); - - assert_eq!(column.cast_type, ColumnType::Float); - } - - #[test] - fn can_parse_empty_indexes() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "indexes": {} } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert!(column.indexes.is_empty()); - } - - #[test] - fn can_parse_ore_index() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "indexes": { "ore": {} } } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!(column.indexes[0].index_type, IndexType::Ore); - } - - #[test] - fn can_parse_ope_index() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "indexes": { "ope": {} } } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!(column.indexes[0].index_type, IndexType::Ope); - } - - #[test] - fn can_parse_unique_index_with_defaults() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "indexes": { "unique": {} } } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Unique { - token_filters: vec![] - } - ); - } - - #[test] - fn can_parse_unique_index_with_token_filter() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "indexes": { - "unique": { - "token_filters": [{ "kind": "downcase" }] - } - } - } - } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Unique { - token_filters: vec![TokenFilter::Downcase] - } - ); - } - - #[test] - fn can_parse_match_index_with_defaults() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "indexes": { "match": {} } } } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Match { - tokenizer: Tokenizer::Standard, - token_filters: vec![], - k: 6, - m: 2048, - include_original: false, - } - ); - } - - #[test] - fn can_parse_match_index_with_all_opts_set() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "indexes": { - "match": { - "tokenizer": { "kind": "ngram", "token_length": 3 }, - "token_filters": [{ "kind": "downcase" }], - "k": 8, - "m": 1024, - "include_original": true - } - } - } - } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "email")) - .unwrap(); - - assert_eq!( - column.indexes[0].index_type, - IndexType::Match { - tokenizer: Tokenizer::Ngram { token_length: 3 }, - token_filters: vec![TokenFilter::Downcase], - k: 8, - m: 1024, - include_original: true, - } - ); - } - - #[test] - fn can_parse_ste_vec_index() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "event_data": { - "cast_as": "jsonb", - "indexes": { "ste_vec": { "prefix": "event-data" } } - } - } - } - }); - - let encrypt_config = parse(json); - let column = encrypt_config - .get(&Identifier::new("users", "event_data")) - .unwrap(); - - assert_eq!( - column.indexes[0].index_type, - IndexType::SteVec { - prefix: "event-data".into(), - term_filters: vec![], - array_index_mode: ArrayIndexMode::ALL, - mode: SteVecMode::Compat, - }, - ); - } - - #[test] - fn config_map_preserves_table_and_column_names() { - let json = json!({ - "v": 1, - "tables": { - "my_schema.users": { - "email_address": { - "cast_as": "text", - "indexes": { "unique": {} } - } - } - } - }); - - let config = parse(json); - let column = config - .get(&Identifier::new("my_schema.users", "email_address")) - .unwrap(); - assert_eq!(column.name, "email_address"); - assert_eq!(column.cast_type, ColumnType::Text); - } - - #[test] - fn config_map_handles_multiple_tables() { - let json = json!({ - "v": 1, - "tables": { - "users": { "email": { "cast_as": "text" } }, - "orders": { "total": { "cast_as": "int" } } - } - }); - - let config = parse(json); - - assert_eq!(config.len(), 2); - assert_eq!( - config - .get(&Identifier::new("users", "email")) - .unwrap() - .cast_type, - ColumnType::Text - ); - assert_eq!( - config - .get(&Identifier::new("orders", "total")) - .unwrap() - .cast_type, - ColumnType::Int - ); - } - - #[test] - fn invalid_config_returns_error() { - let json = json!({ - "v": 1, - "tables": { - "users": { - "email": { - "cast_as": "text", - "indexes": { "ste_vec": { "prefix": "test" } } - } - } - } - }); - - let config: CanonicalEncryptionConfig = serde_json::from_value(json).unwrap(); - assert!(canonical_to_map(config).is_err()); - } - #[test] - fn real_eql_config_produces_correct_encrypt_config() { - let json = json!({ - "v": 1, - "tables": { - "encrypted": { - "encrypted_text": { - "cast_as": "text", - "indexes": { "unique": {}, "match": {}, "ore": {} } - }, - "encrypted_bool": { - "cast_as": "boolean", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_int2": { - "cast_as": "small_int", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_int4": { - "cast_as": "int", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_int8": { - "cast_as": "big_int", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_float8": { - "cast_as": "double", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_date": { - "cast_as": "date", - "indexes": { "unique": {}, "ore": {} } - }, - "encrypted_jsonb": { - "cast_as": "jsonb", - "indexes": { - "ste_vec": { "prefix": "encrypted/encrypted_jsonb" } - } - }, - "encrypted_jsonb_filtered": { - "cast_as": "jsonb", - "indexes": { - "ste_vec": { - "prefix": "encrypted/encrypted_jsonb_filtered", - "term_filters": [{ "kind": "downcase" }] - } - } - } - } - } - }); - - let config = parse(json); - - assert_eq!(config.len(), 9); - - assert_eq!( - config - .get(&Identifier::new("encrypted", "encrypted_float8")) - .unwrap() - .cast_type, - ColumnType::Float - ); - assert_eq!( - config - .get(&Identifier::new("encrypted", "encrypted_jsonb")) - .unwrap() - .cast_type, - ColumnType::Json - ); - assert_eq!( - config - .get(&Identifier::new("encrypted", "encrypted_text")) - .unwrap() - .indexes - .len(), - 3 - ); - assert_eq!( - config - .get(&Identifier::new("encrypted", "encrypted_bool")) - .unwrap() - .indexes - .len(), - 2 - ); - assert_eq!( - config - .get(&Identifier::new("encrypted", "encrypted_jsonb_filtered")) - .unwrap() - .indexes - .len(), - 1 - ); - } - - #[test] - fn malformed_json_returns_parse_error() { - let json = json!({ - "v": 1, - "tables": "not a map" - }); - - let result = serde_json::from_value::(json); - assert!(result.is_err()); - } + Ok(EncryptConfig::new_from_config(map)) } diff --git a/packages/cipherstash-proxy/src/proxy/encrypt_config/mod.rs b/packages/cipherstash-proxy/src/proxy/encrypt_config/mod.rs index de29826e..4edf5db8 100644 --- a/packages/cipherstash-proxy/src/proxy/encrypt_config/mod.rs +++ b/packages/cipherstash-proxy/src/proxy/encrypt_config/mod.rs @@ -1,3 +1,4 @@ +mod from_domain; mod manager; pub use manager::{EncryptConfig, EncryptConfigManager}; diff --git a/packages/cipherstash-proxy/src/proxy/mod.rs b/packages/cipherstash-proxy/src/proxy/mod.rs index 894e60f4..be45ed32 100644 --- a/packages/cipherstash-proxy/src/proxy/mod.rs +++ b/packages/cipherstash-proxy/src/proxy/mod.rs @@ -25,10 +25,10 @@ type ReloadReceiver = UnboundedReceiver; pub type ReloadResponder = Sender<()>; -/// SQL Statement for loading encrypt configuration from database -const ENCRYPT_CONFIG_QUERY: &str = include_str!("./sql/select_config.sql"); - -/// SQL Statement for loading database schema +/// SQL Statement for loading database schema. +/// +/// Both the schema (capabilities) and the encrypt config are inferred from this +/// single schema load — EQL v3 columns are self-configuring domain types. const SCHEMA_QUERY: &str = include_str!("./sql/select_table_schemas.sql"); /// SQL Statement for loading aggregates as part of database schema diff --git a/packages/cipherstash-proxy/src/proxy/sql/select_config.sql b/packages/cipherstash-proxy/src/proxy/sql/select_config.sql deleted file mode 100644 index b748a9d6..00000000 --- a/packages/cipherstash-proxy/src/proxy/sql/select_config.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT data FROM public.eql_v2_configuration WHERE state = 'active' LIMIT 1; From 5c07eefa93a9bbd7e103d1d7684935230a340ac4 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 23:01:10 +1000 Subject: [PATCH 2/4] test(integration): convert the shared test schema to EQL v3 domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the EQL v2 `eql_v2_encrypted` columns + `add_search_config` / `add_encrypted_constraint` / `eql_v2_configuration` truncate with self-configuring v3 domain types (the proxy now infers encrypt config from the schema). - Main `encrypted` / `encrypted_elixir`: scalars use the default CLLW-OPE ordering domain `_ord` (op; also supports equality); `encrypted_text` is `text_search` (eq+ord+match); jsonb -> `json_search`. - `encrypted_bool` is `eql_v3_boolean` (storage-only) — boolean has no searchable capability in v3; the bool search fixtures/columns are dropped. - ORE/OPE fixture tables select their ordering family by `kind`: `ore` -> block-ORE (`_ord_ore`, `text_search_ore`), `ope` -> CLLW-OPE (`_ord_ope`, `text_ord_ope`). The `encrypted_bool` column and the `*_where_bool` fixtures are removed. - `unconfigured` columns become storage-only `eql_v3_text`. - schema-uninstall drops the tables; there is no v2 config table to remove. Refs CIP-3595. Stable-Commit-Id: q-5mf9na2j8evysprjfqpn7a8587 --- tests/sql/schema-uninstall.sql | 8 +- tests/sql/schema.sql | 374 ++++++--------------------------- 2 files changed, 69 insertions(+), 313 deletions(-) diff --git a/tests/sql/schema-uninstall.sql b/tests/sql/schema-uninstall.sql index 0c5fbeac..589bb547 100644 --- a/tests/sql/schema-uninstall.sql +++ b/tests/sql/schema-uninstall.sql @@ -1,10 +1,12 @@ -DROP TABLE IF EXISTS public.eql_v2_configuration; +-- EQL v3 has no `eql_v2_configuration` table (self-configuring domain types), +-- so there is nothing to drop there — just remove the test tables. --- Regular old table DROP TABLE IF EXISTS plaintext; --- Exciting cipherstash table DROP TABLE IF EXISTS encrypted; DROP TABLE IF EXISTS unconfigured; +DROP TABLE IF EXISTS encrypted_elixir; + +DROP TABLE IF EXISTS unconfigured_elixir; diff --git a/tests/sql/schema.sql b/tests/sql/schema.sql index 0c637a7d..1a5b32a8 100644 --- a/tests/sql/schema.sql +++ b/tests/sql/schema.sql @@ -1,5 +1,19 @@ - -TRUNCATE TABLE public.eql_v2_configuration; +-- EQL v3 integration-test schema. +-- +-- Encrypted columns are self-configuring domain types (`eql_v3__`), +-- so there is no `eql_v2.add_search_config`, `add_encrypted_constraint`, or +-- `eql_v2_configuration` table — the domain type declares the encryption and +-- its capabilities, and the proxy infers its config from the schema. +-- +-- Capability -> domain suffix: +-- equality only -> _eq (HMAC) +-- ordering (default, CLLW-OPE) -> _ord (op; also supports equality) +-- ordering (block-ORE) -> _ord_ore (ob) +-- text search (eq+ord+match) -> _search / _search_ore +-- fuzzy match -> _match (bloom) +-- encrypted JSON (SteVec) -> json_search +-- `boolean` is storage-only in v3 (a two-value column leaks its distribution +-- under any index), so encrypted bool columns carry no searchable capability. -- Regular old table DROP TABLE IF EXISTS plaintext; @@ -18,170 +32,49 @@ DO $$ $$; --- Exciting cipherstash table +-- Exciting cipherstash table. Scalars use the default CLLW-OPE ordering domain +-- (`_ord`, which also supports equality); text is fully searchable +-- (`text_search` = eq + ord + match); bool is storage-only. DROP TABLE IF EXISTS encrypted; CREATE TABLE encrypted ( id bigint, plaintext text, plaintext_date date, plaintext_domain domain_type_with_check, - encrypted_text eql_v2_encrypted, - encrypted_bool eql_v2_encrypted, - encrypted_int2 eql_v2_encrypted, - encrypted_int4 eql_v2_encrypted, - encrypted_int8 eql_v2_encrypted, - encrypted_float8 eql_v2_encrypted, - encrypted_date eql_v2_encrypted, - encrypted_jsonb eql_v2_encrypted, - encrypted_jsonb_filtered eql_v2_encrypted, + encrypted_text eql_v3_text_search, + encrypted_bool eql_v3_boolean, + encrypted_int2 eql_v3_smallint_ord, + encrypted_int4 eql_v3_integer_ord, + encrypted_int8 eql_v3_bigint_ord, + encrypted_float8 eql_v3_double_ord, + encrypted_date eql_v3_date_ord, + encrypted_jsonb eql_v3_json_search, + encrypted_jsonb_filtered eql_v3_json_search, PRIMARY KEY(id) ); +-- A storage-only encrypted column (encrypt/decrypt, no searchable capability). DROP TABLE IF EXISTS unconfigured; CREATE TABLE unconfigured ( id bigint, - encrypted_unconfigured eql_v2_encrypted, + encrypted_unconfigured eql_v3_text, PRIMARY KEY(id) ); -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_text', - 'unique', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_text', - 'match', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_text', - 'ore', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_bool', - 'unique', - 'boolean' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_bool', - 'ore', - 'boolean' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int2', - 'unique', - 'small_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int2', - 'ore', - 'small_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int4', - 'unique', - 'int' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int4', - 'ore', - 'int' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int8', - 'unique', - 'big_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_int8', - 'ore', - 'big_int' -); - - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_float8', - 'unique', - 'double' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_float8', - 'ore', - 'double' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_date', - 'unique', - 'date' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_date', - 'ore', - 'date' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_jsonb', - 'ste_vec', - 'jsonb', - '{"prefix": "encrypted/encrypted_jsonb"}' -); - -SELECT eql_v2.add_search_config( - 'encrypted', - 'encrypted_jsonb_filtered', - 'ste_vec', - 'jsonb', - '{"prefix": "encrypted/encrypted_jsonb_filtered", "term_filters": [{"kind": "downcase"}]}' -); - -SELECT eql_v2.add_encrypted_constraint('encrypted', 'encrypted_text'); - -- Per-test encrypted index fixture tables. -- -- Each integration test that exercises ORE/OPE range or order operators gets --- its own table. This eliminates parallel-test races on a shared `encrypted` --- table without having to mark tests `#[serial]`. --- --- The schema mirrors `encrypted` minus the jsonb columns (these tests never --- touch jsonb). `kind` is `ore` or `ope`; ORE text columns additionally get a --- `match` index that the OPE fixtures don't need. +-- its own table to avoid parallel-test races on a shared table. `kind` selects +-- the ordering domain family: `ore` -> block-ORE (`_ord_ore`, `text_search_ore`); +-- `ope` -> CLLW-OPE (`_ord_ope`, `text_ord_ope`). Boolean is storage-only in v3, +-- so the fixtures no longer carry an encrypted_bool column. DO $$ DECLARE spec record; tn text; + text_domain text; + ord_suffix text; BEGIN FOR spec IN -- map_ore_index_where (one per column type) + map_ore_index_order (one per test fn) @@ -192,7 +85,6 @@ BEGIN 'encrypted_ore_where_float8', 'encrypted_ore_where_date', 'encrypted_ore_where_text', - 'encrypted_ore_where_bool', 'encrypted_ore_order_text', 'encrypted_ore_order_text_desc', 'encrypted_ore_order_nulls_last', @@ -221,7 +113,6 @@ BEGIN 'encrypted_ope_where_float8', 'encrypted_ope_where_date', 'encrypted_ope_where_text', - 'encrypted_ope_where_bool', 'encrypted_ope_order_text_asc', 'encrypted_ope_order_text_desc', 'encrypted_ope_order_int4_asc', @@ -232,47 +123,35 @@ BEGIN LOOP tn := spec.table_name; + IF spec.kind = 'ore' THEN + text_domain := 'eql_v3_text_search_ore'; + ord_suffix := '_ord_ore'; + ELSE + text_domain := 'eql_v3_text_ord_ope'; + ord_suffix := '_ord_ope'; + END IF; + EXECUTE format('DROP TABLE IF EXISTS %I CASCADE', tn); EXECUTE format( 'CREATE TABLE %I ( id bigint, plaintext text, plaintext_date date, - encrypted_text eql_v2_encrypted, - encrypted_bool eql_v2_encrypted, - encrypted_int2 eql_v2_encrypted, - encrypted_int4 eql_v2_encrypted, - encrypted_int8 eql_v2_encrypted, - encrypted_float8 eql_v2_encrypted, - encrypted_date eql_v2_encrypted, + encrypted_text %s, + encrypted_int2 eql_v3_smallint%s, + encrypted_int4 eql_v3_integer%s, + encrypted_int8 eql_v3_bigint%s, + encrypted_float8 eql_v3_double%s, + encrypted_date eql_v3_date%s, PRIMARY KEY(id) - )', tn); - - PERFORM eql_v2.add_search_config(tn, 'encrypted_text', 'unique', 'text'); - IF spec.kind = 'ore' THEN - PERFORM eql_v2.add_search_config(tn, 'encrypted_text', 'match', 'text'); - END IF; - PERFORM eql_v2.add_search_config(tn, 'encrypted_text', spec.kind, 'text'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_bool', 'unique', 'boolean'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_bool', spec.kind, 'boolean'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int2', 'unique', 'small_int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int2', spec.kind, 'small_int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int4', 'unique', 'int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int4', spec.kind, 'int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int8', 'unique', 'big_int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_int8', spec.kind, 'big_int'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_float8', 'unique', 'double'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_float8', spec.kind, 'double'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_date', 'unique', 'date'); - PERFORM eql_v2.add_search_config(tn, 'encrypted_date', spec.kind, 'date'); - - PERFORM eql_v2.add_encrypted_constraint(tn, 'encrypted_text'); + )', tn, text_domain, ord_suffix, ord_suffix, ord_suffix, ord_suffix, ord_suffix); END LOOP; END $$; --- This is the exact same schema as above but using a database-generated primary key. --- It is required to remove flake form the Elixir integration test suite. +-- This is the exact same schema as `encrypted` but using a database-generated +-- primary key. It is required to remove flake from the Elixir integration test +-- suite. -- TODO: port all the rest of our integration tests to this schema. DROP TABLE IF EXISTS encrypted_elixir; CREATE TABLE encrypted_elixir ( @@ -280,146 +159,21 @@ CREATE TABLE encrypted_elixir ( plaintext text, plaintext_date date, plaintext_domain domain_type_with_check, - encrypted_text eql_v2_encrypted, - encrypted_bool eql_v2_encrypted, - encrypted_int2 eql_v2_encrypted, - encrypted_int4 eql_v2_encrypted, - encrypted_int8 eql_v2_encrypted, - encrypted_float8 eql_v2_encrypted, - encrypted_date eql_v2_encrypted, - encrypted_jsonb eql_v2_encrypted, - encrypted_jsonb_filtered eql_v2_encrypted, + encrypted_text eql_v3_text_search, + encrypted_bool eql_v3_boolean, + encrypted_int2 eql_v3_smallint_ord, + encrypted_int4 eql_v3_integer_ord, + encrypted_int8 eql_v3_bigint_ord, + encrypted_float8 eql_v3_double_ord, + encrypted_date eql_v3_date_ord, + encrypted_jsonb eql_v3_json_search, + encrypted_jsonb_filtered eql_v3_json_search, PRIMARY KEY(id) ); DROP TABLE IF EXISTS unconfigured_elixir; CREATE TABLE unconfigured_elixir ( id serial, - encrypted_unconfigured eql_v2_encrypted, + encrypted_unconfigured eql_v3_text, PRIMARY KEY(id) ); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_text', - 'unique', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_text', - 'match', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_text', - 'ore', - 'text' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_bool', - 'unique', - 'boolean' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_bool', - 'ore', - 'boolean' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int2', - 'unique', - 'small_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int2', - 'ore', - 'small_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int4', - 'unique', - 'int' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int4', - 'ore', - 'int' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int8', - 'unique', - 'big_int' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_int8', - 'ore', - 'big_int' -); - - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_float8', - 'unique', - 'double' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_float8', - 'ore', - 'double' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_date', - 'unique', - 'date' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_date', - 'ore', - 'date' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_jsonb', - 'ste_vec', - 'jsonb', - '{"prefix": "encrypted/encrypted_jsonb"}' -); - -SELECT eql_v2.add_search_config( - 'encrypted_elixir', - 'encrypted_jsonb_filtered', - 'ste_vec', - 'jsonb', - '{"prefix": "encrypted/encrypted_jsonb_filtered", "term_filters": [{"kind": "downcase"}]}' -); - -SELECT eql_v2.add_encrypted_constraint('encrypted_elixir', 'encrypted_text'); - From 35b92b328fdded727cf0ec300035bb479e3282d3 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 23:06:38 +1000 Subject: [PATCH 3/4] test(integration): make the integration tests v3-shaped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follows the v3 schema conversion. The test crate now compiles and is consistent with EQL v3 semantics (runtime validation still needs a live Proxy + database with EQL v3 installed). - disable_mapping: the EqlEncrypted struct maps to `eql_v3_text_search` (the encrypted_text column's v3 domain) instead of `eql_v2_encrypted`. - jsonb_containment_index: the explicit `eql_v2.jsonb_contains(...)` call and comments become `eql_v3.jsonb_contains(...)`. - indexing: the encrypted index is now the v3 functional form `CREATE INDEX ON encrypted (eql_v3.ord_term(encrypted_text))`. - Bool search tests dropped (v3 boolean is storage-only): map_unique_index_bool, map_ore_where_generic_bool, map_ope_where_generic_bool. Bool roundtrip tests (encrypt/decrypt of a storage-only bool) are kept. - eql_regression (v2->v3 backwards-compat, which is out of scope — prior releases are not in use) is #[ignore]d with a note; regenerate fixtures from a v3 baseline to re-enable. Compiles clean; fmt clean. Refs CIP-3595. Stable-Commit-Id: q-17gkjm4bew1pevb1j47f7z0jvb --- .../src/disable_mapping.rs | 6 ++-- .../src/eql_regression.rs | 10 +++++++ .../src/map_ope_index_where.rs | 5 +--- .../src/map_ore_index_where.rs | 5 +--- .../src/map_unique_index.rs | 30 ++----------------- .../src/select/indexing.rs | 10 ++----- .../src/select/jsonb_containment_index.rs | 12 ++++---- 7 files changed, 27 insertions(+), 51 deletions(-) diff --git a/packages/cipherstash-proxy-integration/src/disable_mapping.rs b/packages/cipherstash-proxy-integration/src/disable_mapping.rs index caa63c14..180d3ef9 100644 --- a/packages/cipherstash-proxy-integration/src/disable_mapping.rs +++ b/packages/cipherstash-proxy-integration/src/disable_mapping.rs @@ -9,7 +9,7 @@ mod tests { use tokio_postgres::types::{FromSql, ToSql}; #[derive(Clone, Debug, ToSql, FromSql, PartialEq, Deserialize)] - #[postgres(name = "eql_v2_encrypted")] + #[postgres(name = "eql_v3_text_search")] pub struct EqlEncrypted { pub data: Value, } @@ -35,10 +35,10 @@ mod tests { let insert_sql = "INSERT INTO encrypted (id, encrypted_text) VALUES ($1, $2)"; let result = client.query(insert_sql, &[&id, &encrypted_text]).await; - // This error is actually a `WrongType` error from the tokio client as encrypted_text is actually eql_v2_encrypted + // This error is actually a `WrongType` error from the tokio client as encrypted_text is actually eql_v3_text_search assert!(result.is_err()); - // Force the eql_v2_encrypted type + // Force the eql_v3_text_search type let encrypted = EqlEncrypted { data: Value::from(encrypted_text.to_owned()), }; diff --git a/packages/cipherstash-proxy-integration/src/eql_regression.rs b/packages/cipherstash-proxy-integration/src/eql_regression.rs index 03949ccc..aa1a0890 100644 --- a/packages/cipherstash-proxy-integration/src/eql_regression.rs +++ b/packages/cipherstash-proxy-integration/src/eql_regression.rs @@ -154,6 +154,7 @@ mod tests { /// /// Set CS_GENERATE_EQL_FIXTURES=1 to enable fixture generation. #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn generate_fixtures() { if std::env::var("CS_GENERATE_EQL_FIXTURES").is_err() { println!("Skipping fixture generation. Set CS_GENERATE_EQL_FIXTURES=1 to generate."); @@ -264,6 +265,7 @@ mod tests { /// Regression test: verify that data encrypted by a previous proxy version /// can still be decrypted by the current version. #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_text() { trace(); @@ -295,6 +297,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_int2() { trace(); @@ -324,6 +327,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_int4() { trace(); @@ -353,6 +357,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_int8() { trace(); @@ -382,6 +387,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_float8() { trace(); @@ -411,6 +417,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_bool() { trace(); @@ -440,6 +447,7 @@ mod tests { } #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_decrypt_legacy_jsonb() { trace(); @@ -469,6 +477,7 @@ mod tests { /// Test JSONB field access (-> operator) on legacy encrypted data #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_jsonb_field_access() { trace(); @@ -520,6 +529,7 @@ mod tests { /// Test JSONB array operations on legacy encrypted data #[tokio::test] + #[ignore = "EQL v2->v3 backwards-compat regression: prior releases are not in use, and v2 ciphertext / ::eql_v2_encrypted casts do not exist under v3. Regenerate fixtures from a v3 baseline to re-enable."] async fn regression_jsonb_array_operations() { trace(); diff --git a/packages/cipherstash-proxy-integration/src/map_ope_index_where.rs b/packages/cipherstash-proxy-integration/src/map_ope_index_where.rs index 92013f9b..cd21af86 100644 --- a/packages/cipherstash-proxy-integration/src/map_ope_index_where.rs +++ b/packages/cipherstash-proxy-integration/src/map_ope_index_where.rs @@ -51,10 +51,7 @@ mod tests { .await; } - #[tokio::test] - async fn map_ope_where_generic_bool() { - map_ope_where_generic("encrypted_ope_where_bool", "encrypted_bool", false, true).await; - } + // bool OPE-range case removed: EQL v3 `boolean` is storage-only. /// Tests OPE operations against a per-test fixture table. /// Mirrors `map_ore_where_generic` but targets the OPE-indexed mirror tables. diff --git a/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs b/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs index 29ca98c2..c8bf8715 100644 --- a/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs +++ b/packages/cipherstash-proxy-integration/src/map_ore_index_where.rs @@ -51,10 +51,7 @@ mod tests { .await; } - #[tokio::test] - async fn map_ore_where_generic_bool() { - map_ore_where_generic("encrypted_ore_where_bool", "encrypted_bool", false, true).await; - } + // bool ORE-range case removed: EQL v3 `boolean` is storage-only. /// Tests ORE operations with 2 values - high & low - against a per-test /// fixture table. `table` and `col_name` must match. diff --git a/packages/cipherstash-proxy-integration/src/map_unique_index.rs b/packages/cipherstash-proxy-integration/src/map_unique_index.rs index b86c4b67..8929bda5 100644 --- a/packages/cipherstash-proxy-integration/src/map_unique_index.rs +++ b/packages/cipherstash-proxy-integration/src/map_unique_index.rs @@ -28,33 +28,9 @@ mod tests { } } - #[tokio::test] - async fn map_unique_index_bool() { - trace(); - - clear().await; - - let client = connect_with_tls(PROXY).await; - - let id = random_id(); - let encrypted_bool: bool = true; - - let sql = "INSERT INTO encrypted (id, encrypted_bool) VALUES ($1, $2)"; - client.query(sql, &[&id, &encrypted_bool]).await.unwrap(); - - let sql = "SELECT id, encrypted_bool FROM encrypted WHERE encrypted_bool = $1"; - let rows = client.query(sql, &[&encrypted_bool]).await.unwrap(); - - assert_eq!(rows.len(), 1); - - for row in rows { - let result_id: i64 = row.get("id"); - let result_bool: bool = row.get("encrypted_bool"); - - assert_eq!(id, result_id); - assert_eq!(encrypted_bool, result_bool); - } - } + // `map_unique_index_bool` removed: EQL v3 `boolean` is storage-only (a + // two-value column leaks its distribution under any index), so equality + // search on an encrypted bool is not supported. #[tokio::test] async fn map_unique_index_int2() { diff --git a/packages/cipherstash-proxy-integration/src/select/indexing.rs b/packages/cipherstash-proxy-integration/src/select/indexing.rs index 2b1c23ed..654706d7 100644 --- a/packages/cipherstash-proxy-integration/src/select/indexing.rs +++ b/packages/cipherstash-proxy-integration/src/select/indexing.rs @@ -20,8 +20,8 @@ mod tests { // let id = random_id(); // let encrypted_val = Domain("ZZ".to_string()); - // CREATE INDEX ON encrypted (e eql_v2.encrypted_operator_class); - // SELECT ore.e FROM ore WHERE id = 42 INTO ore_term; + // EQL v3 uses functional indexes over the term-extraction functions: + // CREATE INDEX ON encrypted (eql_v3.ord_term(encrypted_text)); for n in 1..=10 { let id = random_id(); @@ -34,13 +34,9 @@ mod tests { let client = connect_with_tls(PROXY).await; - let sql = "CREATE INDEX ON encrypted (encrypted_text eql_v2.encrypted_operator_class)"; + let sql = "CREATE INDEX ON encrypted (eql_v3.ord_term(encrypted_text))"; let _ = client.simple_query(sql).await; - // let sql = - // "EXPLAIN ANALYZE SELECT encrypted_text FROM encrypted WHERE encrypted_text <= '{\"hm\": \"abc\"}'::jsonb::eql_v2_encrypted"; - // let result = simple_query::(sql).await; - let sql = "EXPLAIN ANALYZE SELECT encrypted_text FROM encrypted WHERE encrypted_text <= $1"; let encrypted_text = "hello_10".to_string(); diff --git a/packages/cipherstash-proxy-integration/src/select/jsonb_containment_index.rs b/packages/cipherstash-proxy-integration/src/select/jsonb_containment_index.rs index d9085c14..fe9762bf 100644 --- a/packages/cipherstash-proxy-integration/src/select/jsonb_containment_index.rs +++ b/packages/cipherstash-proxy-integration/src/select/jsonb_containment_index.rs @@ -1,8 +1,8 @@ //! Tests for JSONB containment operators //! //! Verifies that the containment operator transformation works correctly: -//! - @> operator is transformed to eql_v2.jsonb_contains() -//! - eql_v2.jsonb_contains() function works with encrypted data +//! - @> operator is transformed to eql_v3.jsonb_contains() +//! - eql_v3.jsonb_contains() function works with encrypted data //! - Both return correct results matching the expected data pattern //! //! ## Test Data @@ -291,9 +291,9 @@ mod tests { rhs = EncryptedColumn ); - /// Test: Verify eql_v2.jsonb_contains() function works through proxy + /// Test: Verify eql_v3.jsonb_contains() function works through proxy /// - /// Tests explicit eql_v2.jsonb_contains() function call works correctly. + /// Tests explicit eql_v3.jsonb_contains() function call works correctly. /// Uses fixture data in ID range FIXTURE_ID_START to FIXTURE_ID_END. /// /// With 500 rows and "string": "value_N" where N = n % 10, @@ -309,11 +309,11 @@ mod tests { // Filter by fixture ID range to isolate from other test data let search_value = json!({"string": "value_1"}); let sql = format!( - "SELECT COUNT(*) FROM encrypted WHERE eql_v2.jsonb_contains(encrypted_jsonb, $1) AND id BETWEEN {} AND {}", + "SELECT COUNT(*) FROM encrypted WHERE eql_v3.jsonb_contains(encrypted_jsonb, $1) AND id BETWEEN {} AND {}", FIXTURE_ID_START, FIXTURE_ID_END ); - info!("Testing eql_v2.jsonb_contains() function with SQL: {}", sql); + info!("Testing eql_v3.jsonb_contains() function with SQL: {}", sql); let rows = client.query(&sql, &[&search_value]).await.unwrap(); let count: i64 = rows[0].get(0); From d14efc89251bd09839e96951332d8c417c44a650 Mon Sep 17 00:00:00 2001 From: James Sadler Date: Wed, 22 Jul 2026 23:20:03 +1000 Subject: [PATCH 4/4] docs: sweep remaining EQL v2 references to v3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert the last docs, comments, and example SQL that still described the EQL v2 model (opaque `eql_v2_encrypted` type + `eql_v2.add_search_config` + `eql_v2_configuration` table) to the v3 self-configuring domain-type model. - ARCHITECTURE.md: rewrite the transformation-rules table (v3 rule set incl. RewriteEqlComparisonOps / RewriteEqlMatchOps) and the schema-loading paragraph (config inferred from domain types, no config table). - CONTEXT-MAP.md: update context/glossary entries; rewrite the "capability across the seam" note — the v3 schema loader now derives real per-column traits from the domain type, so the old "currently broken" bug is resolved. - docs/how-to/index.md: teach the `eql_v3__` domain-type model in place of add_search_config index setup; eql_v3.version(). - docs/reference/index.md: domain type enforces the encrypted-payload constraint; no add_encrypted_constraint call. - docs/reference/searchable-json.md: eql_v3_json_search schema + self-config note; caveat the v2 add_search_config option examples. - docs/errors.md, docs/sql/schema-example.sql, benchmark-schema.sql, parse.rs, psql-passthrough.sh, CLAUDE.md: v3 domain-type phrasing. Intentional v2 mentions retained: contrastive prose, the legacy-warn arm in schema/manager.rs, and the #[ignore]'d backwards-compat regression tests. Stable-Commit-Id: q-76caz2g4z3fptnpp8dn70s8v6g --- ARCHITECTURE.md | 14 ++-- CLAUDE.md | 19 +++-- CONTEXT-MAP.md | 29 +++++--- docs/errors.md | 4 +- docs/how-to/index.md | 73 +++++-------------- docs/reference/index.md | 4 +- docs/reference/searchable-json.md | 43 ++++++----- docs/sql/schema-example.sql | 53 +++----------- .../src/postgresql/messages/parse.rs | 9 ++- tests/benchmark/sql/benchmark-schema.sql | 11 +-- .../test/integration/psql-passthrough.sh | 14 +--- 11 files changed, 107 insertions(+), 166 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 36a0d08e..f2423d2c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -4,7 +4,7 @@ This document describes the internal architecture of CipherStash Proxy. It's int ## Overview -CipherStash Proxy sits between an application and PostgreSQL. It intercepts SQL statements over the PostgreSQL wire protocol, determines which columns are encrypted, rewrites queries to use [EQL v2](https://github.com/cipherstash/encrypt-query-language) operations, encrypts literals and parameters, forwards the transformed query to PostgreSQL, and decrypts results before returning them to the application. +CipherStash Proxy sits between an application and PostgreSQL. It intercepts SQL statements over the PostgreSQL wire protocol, determines which columns are encrypted, rewrites queries to use [EQL v3](https://github.com/cipherstash/encrypt-query-language) operations, encrypts literals and parameters, forwards the transformed query to PostgreSQL, and decrypts results before returning them to the application. The two most interesting pieces of the system are: @@ -116,10 +116,12 @@ The current rules: | Rule | What it does | |---|---| -| `CastLiteralsAsEncrypted` | Replaces plaintext literals with `eql_v2.cast_as_encrypted(ciphertext)` | -| `CastParamsAsEncrypted` | Wraps parameter placeholders (`$1`, `$2`, ...) with encrypted casts | -| `RewriteContainmentOps` | Transforms `col @> val` to `eql_v2.jsonb_contains(col, val)` | -| `RewriteStandardSqlFnsOnEqlTypes` | Rewrites `min()`, `max()`, `jsonb_path_query()` etc. to `eql_v2.*` equivalents | +| `RewriteEqlComparisonOps` | Rewrites scalar comparisons: `col x` → `eql_v3.(col) eql_v3.(x)` (term chosen from the column's domain: `eq_term`/`ord_term`/`ord_term_ore`) | +| `RewriteEqlMatchOps` | Rewrites `LIKE`/`ILIKE`/`@@` to `eql_v3.match_term(a) @> eql_v3.match_term(b)` | +| `RewriteContainmentOps` | Rewrites JSON `@>`/`<@` to `eql_v3.jsonb_contains`/`jsonb_contained_by`, and `->`/`->>` to `eql_v3."->"`/`"->>"` | +| `RewriteStandardSqlFnsOnEqlTypes` | Rewrites `min()`, `max()`, `jsonb_path_query()` etc. to their `eql_v3.*` counterparts (`count()` stays native) | +| `CastLiteralsAsEncrypted` | Replaces plaintext literals with the ciphertext cast to the column's v3 domain (`::public.eql_v3_*`) or, for a query operand, its query twin (`::eql_v3.query_*`) | +| `CastParamsAsEncrypted` | Wraps parameter placeholders (`$1`, `$2`, ...) with the same v3 domain casts | | `PreserveEffectiveAliases` | Maintains column aliases through transformations | | `FailOnPlaceholderChange` | Postcondition check that prepared statement placeholders weren't corrupted | @@ -191,7 +193,7 @@ When encrypting values for a statement, many columns may be `NULL` or non-encryp ## Schema Management -The proxy discovers the database schema at startup and reloads it periodically. Schema loading queries PostgreSQL's `information_schema` to discover tables and columns, then checks `eql_v2_configuration` to determine which columns are encrypted and what index types they support. +The proxy discovers the database schema at startup and reloads it periodically. Schema loading queries PostgreSQL's `information_schema` to discover tables and columns, including each column's domain type. EQL v3 columns are self-configuring domain types (e.g. `eql_v3_text_search`), so both the type-checker's capability view and the encrypt config are inferred from that single schema load — the column's domain name determines which columns are encrypted, their token type, and their searchable capabilities. There is no `eql_v2_configuration` table. Schema state is stored behind an `ArcSwap`, which provides lock-free reads with atomic updates. This means query processing never blocks on a schema reload — readers always get a consistent snapshot. diff --git a/CLAUDE.md b/CLAUDE.md index 6030345e..a6d4d39c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ CipherStash Proxy is a PostgreSQL proxy that provides **transparent, searchable Key capabilities: - Zero-change SQL queries - applications connect to Proxy instead of directly to PostgreSQL -- EQL v2 (Encrypt Query Language) for searchable encryption using CipherStash ZeroKMS +- EQL v3 (Encrypt Query Language) for searchable encryption using CipherStash ZeroKMS - Support for encrypted equality, comparison, ordering, and grouping operations - Written in Rust for performance with strongly-typed SQL statement mapping @@ -34,14 +34,14 @@ Key capabilities: - Language-specific integration tests (Python, Go) **Showcase (`packages/showcase/`):** -- Healthcare data model demonstrating EQL v2 encryption +- Healthcare data model demonstrating EQL v3 encryption - Example of realistic encrypted application with foreign keys and relationships ### Request Flow 1. Application connects to Proxy (port 6432) using standard PostgreSQL protocol 2. Proxy intercepts SQL statements and uses EQL Mapper to analyze query structure -3. For encrypted columns, Proxy transforms SQL using EQL v2 operations +3. For encrypted columns, Proxy transforms SQL using EQL v3 operations 4. Encrypted queries are sent to actual PostgreSQL database 5. Results are decrypted before returning to application @@ -166,12 +166,17 @@ Available targets: `DEVELOPMENT`, `AUTHENTICATION`, `CONFIG`, `CONTEXT`, `ENCODI ## EQL Integration -CipherStash Proxy uses EQL v2 for searchable encryption. Key concepts: +CipherStash Proxy uses EQL v3 for searchable encryption. Key concepts: - **Plaintext columns** - standard PostgreSQL data types -- **Encrypted columns** - use `eql_v2_encrypted` type in schema -- **Searchable operations** - equality, comparison, ordering work on encrypted data -- **Index support** - ORE (Order Revealing Encryption) and Match indexes for performance +- **Encrypted columns** - use a self-configuring EQL v3 domain type in the schema + (e.g. `eql_v3_text_search`, `eql_v3_integer_ord`, `eql_v3_json_search`); the + domain encodes the token type and searchable capabilities, so there is no + separate `add_search_config` call +- **Searchable operations** - equality, comparison, ordering, text match, and JSON + traversal work on encrypted data, gated by the column's domain capability +- **Index support** - functional indexes over the term-extraction functions + (e.g. `CREATE INDEX ON t (eql_v3.ord_term(col))`) EQL is automatically downloaded and installed during setup. Use `CS_EQL_PATH` to point to local EQL development version. diff --git a/CONTEXT-MAP.md b/CONTEXT-MAP.md index 31c69ffa..7ab0b328 100644 --- a/CONTEXT-MAP.md +++ b/CONTEXT-MAP.md @@ -10,9 +10,9 @@ resolved — a missing one is expected, not a gap to fill upfront. | Context | Path | Domain | |---|---|---| | Proxy | [`packages/cipherstash-proxy/`](./packages/cipherstash-proxy/CONTEXT.md) | PostgreSQL wire protocol, connection and message handling, client authentication, TLS, ZeroKMS key management, encrypt/decrypt of column values | -| EQL Mapper | [`packages/eql-mapper/`](./packages/eql-mapper/CONTEXT.md) | SQL parsing, type inference over statements, schema analysis, transformation rules that rewrite plaintext SQL into EQL v2 operations | +| EQL Mapper | [`packages/eql-mapper/`](./packages/eql-mapper/CONTEXT.md) | SQL parsing, type inference over statements, schema analysis, transformation rules that rewrite plaintext SQL into EQL v3 operations | | Integration | `packages/cipherstash-proxy-integration/` | End-to-end test harness — container fixtures, encrypted-scenario coverage across the proxy and mapper together | -| Showcase | `packages/showcase/` | Healthcare example data model demonstrating EQL v2 encryption with realistic relationships | +| Showcase | `packages/showcase/` | Healthcare example data model demonstrating EQL v3 encryption with realistic relationships | `packages/eql-mapper-macros/` is proc-macro support for EQL Mapper, not a context of its own — treat it as part of the EQL Mapper context. @@ -26,22 +26,27 @@ own — treat it as part of the EQL Mapper context. - **Identity across the seam**: EQL Mapper's `TableColumn` and Proxy's `Identifier` are the same `table.column` pair under two names. That pair is the only key joining a typed AST node to its encryption config. -- **Capability across the seam — currently broken.** Proxy marks every encrypted column - with *all* `EqlTrait`s (`packages/cipherstash-proxy/src/proxy/schema/manager.rs:146`) - because it derives them from the PostgreSQL column type alone. The column encrypt - config, which knows the SEM terms actually configured, is loaded by a separate manager - that never meets the schema loader. EQL Mapper's bound checking is therefore - unreachable in production: a query needing `Ord` on a column with no ordering SEM term - type-checks cleanly and fails later. Read `EqlTraits` on a column as *intended* - capability, not observed. +- **Capability across the seam.** Under EQL v3 each encrypted column is a self-configuring + domain type (e.g. `eql_v3_text_search`) whose typname encodes both the token type and the + searchable capabilities. The schema loader resolves that domain to a `DomainIdentity` and + the exact `EqlTraits` it supports (`packages/cipherstash-proxy/src/proxy/schema/manager.rs`, + via `proxy/schema/eql_domains.rs`), so the traits handed to EQL Mapper are *observed*, not + a blanket grant. EQL Mapper's bound checking is therefore effective in production: a query + needing `Ord` on a column whose domain has no ordering capability is rejected at type-check + time. The encrypt config is derived from the same domain type + (`proxy/encrypt_config/from_domain.rs`), so schema view and encrypt config no longer + disagree. ## Shared vocabulary Terms defined once for the whole system live here rather than in any one context. -- **EQL v2** — Encrypt Query Language; the SQL-level encoding that makes encrypted +- **EQL v3** — Encrypt Query Language; the SQL-level encoding that makes encrypted values searchable. -- **`eql_v2_encrypted`** — the PostgreSQL column type holding an encrypted value. +- **EQL v3 domain types** — encrypted columns are self-configuring PostgreSQL DOMAINs over + `jsonb` (e.g. `eql_v3_text_search`, `eql_v3_int8_ord`, `eql_v3_json_search`). The domain's + typname encodes the token type and the searchable capabilities, replacing EQL v2's opaque + `eql_v2_encrypted` type plus a separate `eql_v2_configuration` table. - **ZeroKMS** — CipherStash's key management service, which the proxy calls to encrypt and decrypt. - **Keyset** — the ZeroKMS key collection a workspace encrypts against. diff --git a/docs/errors.md b/docs/errors.md index 7099e66a..6d5035ac 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -479,7 +479,7 @@ For example: ## Unknown Column -The column has an encrypted type (PostgreSQL `eql_v2_encrypted` type ) with no encryption configuration. +The column has an encrypted type (an EQL v3 encrypted domain type, e.g. `eql_v3_text_search`) with no encryption configuration. Without the configuration, Cipherstash Proxy does not know how to encrypt the column. Any data is unprotected and unencrypted. @@ -506,7 +506,7 @@ Column 'column_name' in table 'table_name' has no Encrypt configuration ## Unknown Table -The table has one or more encrypted columns (PostgreSQL `eql_v2_encrypted` type ) with no encryption configuration. +The table has one or more encrypted columns (an EQL v3 encrypted domain type, e.g. `eql_v3_text_search`) with no encryption configuration. Without the configuration, Cipherstash Proxy does not know how to encrypt the column. Any data is unprotected and unencrypted. diff --git a/docs/how-to/index.md b/docs/how-to/index.md index 9f8309cc..c98e8192 100644 --- a/docs/how-to/index.md +++ b/docs/how-to/index.md @@ -173,7 +173,7 @@ You can also install EQL by running [the installation script](https://github.com Once you have installed EQL, you can see what version is installed by querying the database: ```sql -SELECT eql_v2.version(); +SELECT eql_v3.version(); ``` This will output the version of EQL installed. @@ -182,79 +182,44 @@ This will output the version of EQL installed. In your existing PostgreSQL database, you store your data in tables and columns. Those columns have types like `integer`, `text`, `timestamp`, and `boolean`. -When storing encrypted data in PostgreSQL with Proxy, you use a special column type called `eql_v2_encrypted`, which is [provided by EQL](#setting-up-the-database-schema). -`eql_v2_encrypted` is a container column type that can be used for any type of encrypted data you want to store or search, whether they are numbers (`int`, `small_int`, `big_int`), text (`text`), dates and times (`date`. `timestamp`), or booleans (`boolean`). +When storing encrypted data in PostgreSQL with Proxy, you use one of EQL's **encrypted domain types**, which are [provided by EQL](#setting-up-the-database-schema). -Create a table with an encrypted column for `email`: +In EQL v3 these domain types are **self-configuring**: the type you choose for a column both marks it as encrypted *and* declares which searches it supports. This replaces EQL v2's model of a single opaque `eql_v2_encrypted` container type plus a separate `eql_v2.add_search_config` call per index — there is no separate index-configuration step, and no `eql_v2_configuration` table. + +Domain types follow the naming pattern `eql_v3__`: + +- **Storage only** — `eql_v3_text`, `eql_v3_integer`, `eql_v3_bigint`, `eql_v3_date`, `eql_v3_boolean`, and so on store an encrypted value that can be read back but not searched. (`boolean` is always storage-only: a two-value column would leak its distribution under any index.) +- **Ordering and range** — the `_ord` suffix (e.g. `eql_v3_integer_ord`, `eql_v3_date_ord`) adds ordering (`ORDER BY`) and range comparisons (`<`, `<=`, `>`, `>=`), and also supports equality (`=`). This is the recommended default and uses CLLW-OPE ordering. +- **Ordering and range via ORE** — the `_ord_ore` suffix (e.g. `eql_v3_integer_ord_ore`) is an alternative ordering scheme backed by block-ORE. Choose `_ord` or `_ord_ore` for a column, not both. +- **Full text search** — for `text`, `eql_v3_text_search` bundles equality, ordering, and fuzzy `LIKE`/`ILIKE` match in one type; `eql_v3_text_search_ore` is the ORE-backed variant, and `eql_v3_text_ord_ope` provides OPE ordering. +- **Encrypted JSON** — `eql_v3_json_search` stores encrypted JSON with SteVec containment (`@>`, `<@`) and path (`->`, `->>`) search. See [Searchable JSON](../reference/searchable-json.md). + +Create a `users` table with an encrypted, fully-searchable `email` column: ```sql CREATE TABLE users ( id SERIAL PRIMARY KEY, - email eql_v2_encrypted + email eql_v3_text_search ) ``` This creates a `users` table with two columns: - `id`, an autoincrementing integer column that is the primary key for the record - - `email`, a `eql_v2_encrypted` column + - `email`, an encrypted `text` column that supports equality (`=`), ordering, and fuzzy `LIKE`/`ILIKE` matching — because it uses the `eql_v3_text_search` domain type There are important differences between the plaintext columns you've traditionally used in PostgreSQL and encrypted columns with CipherStash Proxy: - **Plaintext columns can be searched if they don't have an index**, albeit with the performance cost of a full table scan. -- **Encrypted columns cannot be searched without an encrypted index**, and the encrypted indexes you define determine what kind of searches you can do on encrypted data. - -In the previous step we created a table with an encrypted column, but without any encrypted indexes. - -Now you can add an encrypted index for that encrypted column: - -```sql -SELECT eql_v2.add_search_config( - 'users', - 'email', - 'unique', - 'text' -); -``` - -This statement adds a `unique` index for the `email` column in the `users` table, which has an underlying data type of `text`. - -`unique` indexes are used to find records with columns with unique values, like with the `=` operator. - -There are other types of encrypted indexes you can use on `text` data: - -```sql -SELECT eql_v2.add_search_config( - 'users', - 'email', - 'match', - 'text' -); - -SELECT eql_v2.add_search_config( - 'users', - 'email', - 'ore', - 'text' -); - -SELECT eql_v2.add_search_config( - 'users', - 'email', - 'ope', - 'text' -); -``` +- **An encrypted column can only be searched in the ways its domain type allows.** Choose the domain type up front to match the queries you need: `eql_v3_text` if you only store and retrieve the value, `eql_v3_text_search` if you also need to compare and match it. -The first SQL statement adds a `match` index, which is used for partial matches with `LIKE`. -The second SQL statement adds an `ore` index, which is used for ordering with `ORDER BY` and range comparisons (`<`, `<=`, `>`, `>=`). -The third SQL statement adds an `ope` index, which supports the same range and ordering operators as `ore`. +If you only needed equality on `email` — for example a lookup by exact address — you could store it as a scalar ordering type such as `eql_v3_text_ord_ope`, or use `eql_v3_text_search` when you also want partial matches with `LIKE`. -`ore` and `ope` are alternatives for range and ordering queries — add one or the other to a column, not both. `ore` is the recommended default. `ope` produces ciphertexts that sort under PostgreSQL's native byte ordering, which makes ordering and range scans cheaper, but as an order-preserving scheme it reveals more about the relative order of stored values than `ore` does. Choose based on your performance and threat-model requirements; see the [EQL `INDEX` documentation](https://github.com/cipherstash/encrypt-query-language/blob/main/docs/reference/INDEX.md) for the full tradeoffs. +`_ord` (CLLW-OPE) produces ciphertexts that sort under PostgreSQL's native byte ordering, which makes ordering and range scans cheaper, but as an order-preserving scheme it reveals more about the relative order of stored values than the block-ORE `_ord_ore` variant does. Choose based on your performance and threat-model requirements; see the [EQL `INDEX` documentation](https://github.com/cipherstash/encrypt-query-language/blob/main/docs/reference/INDEX.md) for the full tradeoffs. > [!IMPORTANT] -> Adding, updating, or deleting encrypted indexes on columns that already contain encrypted data will not re-index that data. To use the new indexes, you must `SELECT` the data out of the column, and `UPDATE` it again. +> The searches an encrypted column supports are fixed by its domain type. To change them you change the column's type (e.g. `ALTER TABLE users ALTER COLUMN email TYPE eql_v3_text_search`), and any data already stored must be re-encrypted under the new type — `SELECT` it out of the column and `UPDATE` it back — before the new capabilities apply to it. To learn how to use encrypted indexes for other encrypted data types like `text`, `int`, `boolean`, `date`, and `jsonb`, see the [EQL documentation](https://github.com/cipherstash/encrypt-query-language/blob/main/docs/reference/INDEX.md). diff --git a/docs/reference/index.md b/docs/reference/index.md index e3ac358f..335d243d 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -470,10 +470,10 @@ The parameter is always scoped to the connection `SESSION` - mapping is only eve CipherStash Proxy and EQL do provide some protection against writing plaintext into and reading plaintext from encrypted columns. -Always use `eql_v2.add_encrypted_constraint(table, column)` when defining encrypted columns to ensure plaintext data cannot be written. +In EQL v3 this protection is built into the column's domain type: an `eql_v3_*` domain is a checked `jsonb` domain, so PostgreSQL rejects a plaintext value that is not a valid encrypted payload. There is no separate `eql_v2.add_encrypted_constraint` call to apply — defining the column with an `eql_v3_*` type is sufficient. Unmapped `SELECT` statements should always return the encrypted payload. -If the constraint has been applied, unmapped `INSERT`/`UPDATE` statements should return a PostgreSQL type error. +Unmapped `INSERT`/`UPDATE` statements that try to write plaintext should return a PostgreSQL type error. ### Disable mapping diff --git a/docs/reference/searchable-json.md b/docs/reference/searchable-json.md index 6b5e3798..9f9f5d9d 100644 --- a/docs/reference/searchable-json.md +++ b/docs/reference/searchable-json.md @@ -28,25 +28,28 @@ This document outlines the supported JSONB functions and operators in CipherStas ```sql CREATE TABLE cipherstash ( id SERIAL PRIMARY KEY, - encrypted_jsonb eql_v2_encrypted + encrypted_jsonb eql_v3_json_search ) ``` ### Encrypted column configuration -```sql -SELECT eql_v2.add_search_config( - 'cipherstash', - 'encrypted_jsonb', - 'ste_vec', - 'jsonb', - '{"prefix": "cipherstash/encrypted_jsonb"}' -); -``` + +EQL v3 encrypted-JSON columns are self-configuring: the `eql_v3_json_search` +domain type is the SteVec (searchable encrypted JSON) configuration, so the +column type alone enables JSON search. There is no separate +`add_search_config` call as in EQL v2. > **Note:** JSONB literals in INSERT and UPDATE statements work directly without explicit `::jsonb` type casts. The proxy infers the JSONB type from the target column and handles encryption transparently. #### Configuration options +> **EQL v2 legacy:** In EQL v2 the `ste_vec` index was configured explicitly via +> `add_search_config`, and the options below (and the `add_search_config` examples +> in this section) describe that mechanism. In EQL v3 the `eql_v3_json_search` +> domain type carries a fixed default configuration, so these options are not +> set per-column via SQL. The descriptions are retained to explain the indexing +> behaviour. + The `ste_vec` index configuration accepts the following options: | Option | Type | Default | Description | @@ -144,7 +147,7 @@ Examples: ## Operators -### `-> text returns eql_v2_encrypted decrypted as jsonb` +### `-> text returns eql_v3_json_search decrypted as jsonb` Extracts JSON object field with the given key. @@ -198,7 +201,7 @@ SELECT encrypted_jsonb -> 'string_array' FROM cipherstash; -### `->> text returns eql_v2_encrypted decrypted as jsonb` +### `->> text returns eql_v3_json_search decrypted as jsonb` Extracts JSON object field with the given key. @@ -264,9 +267,9 @@ SELECT encrypted_jsonb -> 'string_array' FROM cipherstash; -### `eql_v2_encrypted @> eql_v2_encrypted returns boolean` +### `eql_v3_json_search @> eql_v3_json_search returns boolean` -Does the left `eql_v2_encrypted` value contain the right `eql_v2_encrypted` path/value entries at the top level? +Does the left `eql_v3_json_search` value contain the right `eql_v3_json_search` path/value entries at the top level? #### Syntax @@ -319,7 +322,7 @@ SELECT encrypted_jsonb @> '{"object": {"string": "world", "number": 99}}' FROM c -### `eql_v2_encrypted <@ eql_v2_encrypted returns boolean` +### `eql_v3_json_search <@ eql_v3_json_search returns boolean` Is the first JSON value contained in the second? @@ -373,7 +376,7 @@ SELECT '{"object": {"string": "world", "number": 99}}' <@ encrypted_jsonb FROM c ## Functions -### `jsonb_path_query(target eql_v2_encrypted, path jsonpath) returns setof eql_v2_encrypted decrypted as jsonb` +### `jsonb_path_query(target eql_v3_json_search, path jsonpath) returns setof eql_v3_json_search decrypted as jsonb` Returns all JSON items returned by the JSON path for the specified JSON value. @@ -437,7 +440,7 @@ SELECT jsonb_path_query(encrypted_jsonb, '$.string_array') FROM cipherstash; -### `jsonb_path_query_first(target eql_v2_encrypted, path jsonpath) returns eql_v2_encrypted decrypted as jsonb` +### `jsonb_path_query_first(target eql_v3_json_search, path jsonpath) returns eql_v3_json_search decrypted as jsonb` Returns all JSON items returned by the JSON path for the specified JSON value. @@ -476,7 +479,7 @@ SELECT jsonb_path_query_first(encrypted_jsonb, '$.numeric_array[*]') FROM cipher --------------------------------------------------------------- -### `jsonb_path_exists(target eql_v2_encrypted, path jsonpath) returns bool` +### `jsonb_path_exists(target eql_v3_json_search, path jsonpath) returns bool` Checks whether the JSON path returns any item for the specified JSON value. @@ -514,7 +517,7 @@ SELECT jsonb_path_exists(encrypted_jsonb, '$.unknown') FROM cipherstash; -### `jsonb_array_elements(target eql_v2_encrypted) returns setof eql_v2_encrypted decrypted as jsonb` +### `jsonb_array_elements(target eql_v3_json_search) returns setof eql_v3_json_search decrypted as jsonb` Expands the top-level JSON array into a set of values. @@ -571,7 +574,7 @@ SELECT jsonb_array_elements(jsonb_path_query(encrypted_jsonb, '$.numeric_array[@ -### `jsonb_array_length(target eql_v2_encrypted) returns integer` +### `jsonb_array_length(target eql_v3_json_search) returns integer` Returns the number of elements in the top-level JSON array. diff --git a/docs/sql/schema-example.sql b/docs/sql/schema-example.sql index c89bb1b7..1e631143 100644 --- a/docs/sql/schema-example.sql +++ b/docs/sql/schema-example.sql @@ -1,47 +1,18 @@ -TRUNCATE TABLE public.eql_v2_configuration; +-- EQL v3 example schema. +-- +-- Encrypted columns are self-configuring domain types (`eql_v3__`): +-- the column type both marks the column as encrypted and declares which searches +-- it supports. There is no `eql_v2_configuration` table to truncate and no +-- `eql_v2.add_search_config` call — the proxy infers the encrypt config from the +-- column's domain type. -- Exciting cipherstash table DROP TABLE IF EXISTS users; CREATE TABLE users ( id SERIAL PRIMARY KEY, - encrypted_email eql_v2_encrypted, - encrypted_dob eql_v2_encrypted, - encrypted_salary eql_v2_encrypted -); - -SELECT eql_v2.add_search_config( - 'users', - 'encrypted_email', - 'unique', - 'text' -); - -SELECT eql_v2.add_search_config( - 'users', - 'encrypted_email', - 'match', - 'text' -); - --- 'ore' supports ordering and range comparisons. 'ope' is a drop-in --- alternative with the same operator support — choose one per column. -SELECT eql_v2.add_search_config( - 'users', - 'encrypted_email', - 'ore', - 'text' -); - -SELECT eql_v2.add_search_config( - 'users', - 'encrypted_salary', - 'ore', - 'int' -); - -SELECT eql_v2.add_search_config( - 'users', - 'encrypted_dob', - 'ore', - 'date' + -- equality + ordering + fuzzy LIKE/ILIKE match + encrypted_email eql_v3_text_search, + -- ordering + range comparisons (and equality) + encrypted_dob eql_v3_date_ord, + encrypted_salary eql_v3_bigint_ord ); diff --git a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs index 6a71bf3b..10a2038a 100644 --- a/packages/cipherstash-proxy/src/postgresql/messages/parse.rs +++ b/packages/cipherstash-proxy/src/postgresql/messages/parse.rs @@ -24,11 +24,12 @@ impl Parse { } /// - /// Encrypted columns are the eql_v2_encrypted Domain Type - /// eql_v2_encrypted wraps JSONB + /// EQL v3 encrypted columns are JSONB-backed domain types (e.g. + /// `eql_v3_text_search`). /// - /// Using JSONB to avoid the complexity of loading the OID of eql_v2_encrypted - /// PostgreSQL will coerce JSONB to eql_v2_encrypted if it passes the constaint check + /// Using JSONB to avoid the complexity of loading each domain's OID — + /// PostgreSQL coerces JSONB to the domain type if it passes the CHECK + /// constraint. /// pub fn rewrite_param_types(&mut self, columns: &[Option]) { for (idx, col) in columns.iter().enumerate() { diff --git a/tests/benchmark/sql/benchmark-schema.sql b/tests/benchmark/sql/benchmark-schema.sql index c2024fe0..a5f66e11 100644 --- a/tests/benchmark/sql/benchmark-schema.sql +++ b/tests/benchmark/sql/benchmark-schema.sql @@ -1,4 +1,5 @@ -TRUNCATE TABLE public.eql_v2_configuration; +-- EQL v3: the encrypted column is a self-configuring domain type; there is no +-- `eql_v2_configuration` table or `eql_v2.add_column` call. DROP TABLE IF EXISTS benchmark_plaintext; CREATE TABLE benchmark_plaintext ( @@ -11,11 +12,5 @@ DROP TABLE IF EXISTS benchmark_encrypted; CREATE TABLE benchmark_encrypted ( id serial primary key, username text, - email eql_v2_encrypted + email eql_v3_text_search ); - -SELECT eql_v2.add_column( - 'benchmark_encrypted', - 'email' -); - diff --git a/tests/tasks/test/integration/psql-passthrough.sh b/tests/tasks/test/integration/psql-passthrough.sh index caa91f9a..d169ff68 100755 --- a/tests/tasks/test/integration/psql-passthrough.sh +++ b/tests/tasks/test/integration/psql-passthrough.sh @@ -22,16 +22,10 @@ SELECT 1; EOF -# Confirm that there is indeed no config -set +e -OUTPUT="$(docker exec -i postgres${CONTAINER_SUFFIX} psql postgresql://cipherstash:${encoded_password}@proxy:6432/cipherstash --command 'SELECT * FROM eql_v2_configuration' 2>&1)" -retval=$? -if echo ${OUTPUT} | grep -v 'relation "eql_v2_configuration" does not exist'; then - echo "error: did not see string in output: \"relation "eql_v2_configuration" does not exist\"" - exit 1 -fi - -set -e +# EQL v3 has no configuration table: encrypted columns are self-configuring +# domain types (e.g. `eql_v3_text_search`) and the proxy infers the encrypt +# config directly from the schema. There is no `eql_v2_configuration` table to +# probe, so the passthrough sanity checks above are sufficient here. echo "----------------------------------" echo "Unconfigurated connection tests complete"