diff --git a/Cargo.toml b/Cargo.toml index 387950ad7..28336f547 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,6 +104,7 @@ swimos_rtree = { path = "swimos_utilities/swimos_rtree", version = "0.1.1" } swimos_sync = { path = "swimos_utilities/swimos_sync", version = "0.1.1" } swimos_time = { path = "swimos_utilities/swimos_time", version = "0.1.1" } swimos_encoding = { path = "swimos_utilities/swimos_encoding", version = "0.1.1" } +swimos_connector_util = { path = "server/swimos_connector_util", version = "0.1.1" } bytes = "1.3" tokio = "1.22" @@ -182,4 +183,5 @@ http-body-util = "0.1.2" hyper-util = "0.1.5" rdkafka = "0.36" apache-avro = "0.17.0" -time = "0.3.36" \ No newline at end of file +time = "0.3.36" +fluvio = "0.23.2" diff --git a/example_apps/kafka_ingress_connector/src/main.rs b/example_apps/kafka_ingress_connector/src/main.rs index 823dc1902..f086d92ec 100644 --- a/example_apps/kafka_ingress_connector/src/main.rs +++ b/example_apps/kafka_ingress_connector/src/main.rs @@ -34,7 +34,6 @@ use swimos::{ }; use swimos_connector::IngressConnectorModel; use swimos_connector_kafka::{KafkaIngressConfiguration, KafkaIngressConnector}; -use swimos_recon::parser::parse_recognize; mod params; @@ -99,11 +98,10 @@ async fn load_config( } else { CONNECTOR_CONFIG }; - let config = parse_recognize::(recon, true)?; - Ok(config) + KafkaIngressConfiguration::from_str(recon) } -pub fn setup_logging() -> Result<(), Box> { +pub fn setup_logging() -> Result<(), Box> { let filter = example_filter()?.add_directive(LevelFilter::INFO.into()); tracing_subscriber::fmt().with_env_filter(filter).init(); Ok(()) diff --git a/server/swimos_connector/Cargo.toml b/server/swimos_connector/Cargo.toml index 0ecba646f..07b777048 100644 --- a/server/swimos_connector/Cargo.toml +++ b/server/swimos_connector/Cargo.toml @@ -8,6 +8,12 @@ license.workspace = true repository = "https://github.com/swimos/swim-rust/tree/main/server/swimos_connector" homepage.workspace = true +[features] +default = [] +json = ["dep:serde_json"] +avro = ["dep:apache-avro", "dep:chrono", "tokio/fs"] +pubsub = [] + [dependencies] futures = { workspace = true } swimos_utilities = { workspace = true } @@ -22,11 +28,21 @@ swimos_agent = { workspace = true } swimos_agent_protocol = { workspace = true } tokio-stream = { workspace = true } tracing = { workspace = true } -frunk = { workspace = true } uuid = { workspace = true } thiserror = { workspace = true } bitflags = { workspace = true } +chrono = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } +apache-avro = { workspace = true, optional = true } +regex = { workspace = true } +frunk = { workspace = true } +nom = { workspace = true } +nom_locate = { workspace = true } [dev-dependencies] tokio = { workspace = true, features = ["rt", "test-util", "time"] } parking_lot = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +uuid = { workspace = true, features = ["v4"] } +swimos_connector_util = { workspace = true } \ No newline at end of file diff --git a/server/swimos_connector_kafka/src/config/format.rs b/server/swimos_connector/src/config/format.rs similarity index 89% rename from server/swimos_connector_kafka/src/config/format.rs rename to server/swimos_connector/src/config/format.rs index 991611918..70de609f4 100644 --- a/server/swimos_connector_kafka/src/config/format.rs +++ b/server/swimos_connector/src/config/format.rs @@ -14,21 +14,19 @@ use swimos_form::Form; -use crate::{ - deser::{ - BoxMessageDeserializer, BytesDeserializer, F32Deserializer, F64Deserializer, - I32Deserializer, I64Deserializer, MessageDeserializer, ReconDeserializer, - StringDeserializer, U32Deserializer, U64Deserializer, UuidDeserializer, - }, - ser::{ - BytesSerializer, F32Serializer, F64Serializer, I32Serializer, I64Serializer, - MessageSerializer, ReconSerializer, SharedMessageSerializer, StringSerializer, - U32Serializer, U64Serializer, UuidSerializer, - }, - Endianness, LoadError, +use crate::deser::{ + BoxMessageDeserializer, BytesDeserializer, Endianness, F32Deserializer, F64Deserializer, + I32Deserializer, I64Deserializer, MessageDeserializer, ReconDeserializer, StringDeserializer, + U32Deserializer, U64Deserializer, UuidDeserializer, +}; +use crate::ser::{ + BytesSerializer, F32Serializer, F64Serializer, I32Serializer, I64Serializer, MessageSerializer, + ReconSerializer, SharedMessageSerializer, StringSerializer, U32Serializer, U64Serializer, + UuidSerializer, }; +use crate::LoadError; -/// Supported deserialization formats to use to interpret a component of a Kafka message. +/// Supported deserialization formats to use to interpret a component of a message. #[derive(Clone, Form, Debug, Default, PartialEq, Eq)] pub enum DataFormat { #[default] diff --git a/server/swimos_connector/src/config/ingress.rs b/server/swimos_connector/src/config/ingress.rs new file mode 100644 index 000000000..509933ddc --- /dev/null +++ b/server/swimos_connector/src/config/ingress.rs @@ -0,0 +1,174 @@ +use crate::selector::{BadSelector, PubSubSelector, Relay, Relays}; +use swimos_form::Form; + +/// Specification of a value lane for the connector. +#[derive(Clone, Debug, Form, PartialEq, Eq)] +#[form(tag = "ValueLaneSpec")] +pub struct IngressValueLaneSpec { + /// A name to use for the lane. If not specified, the connector will attempt to infer one from the selector. + pub name: Option, + /// String representation of a selector to extract values for the lane from messages. + pub selector: String, + /// Whether the lane is required. If this is `true` and the selector returns nothing for a Message, the + /// connector will fail with an error. + pub required: bool, +} + +impl IngressValueLaneSpec { + /// # Arguments + /// * `name` - A name to use for the lane. If not specified the connector will attempt to infer a name from the selector. + /// * `selector` - String representation of the selector to extract values from the message. + /// * `required` - Whether the lane is required. If this is `true` and the selector returns nothing for a Message, the + /// connector will fail with an error. + pub fn new>(name: Option, selector: S, required: bool) -> Self { + IngressValueLaneSpec { + name: name.map(Into::into), + selector: selector.into(), + required, + } + } +} + +/// Specification of a value lane for the connector. +#[derive(Clone, Debug, Form, PartialEq, Eq)] +#[form(tag = "MapLaneSpec")] +pub struct IngressMapLaneSpec { + /// The name of the lane. + pub name: String, + /// String representation of a selector to extract the map keys from the messages. + pub key_selector: String, + /// String representation of a selector to extract the map values from the messages. + pub value_selector: String, + /// Whether to remove an entry from the map if the value selector does not return a value. Otherwise, missing + /// values will be treated as a failed extraction from the message. + pub remove_when_no_value: bool, + /// Whether the lane is required. If this is `true` and the selector returns nothing for a Message, the + /// connector will fail with an error. + pub required: bool, +} + +impl IngressMapLaneSpec { + /// # Arguments + /// * `name` - The name of the lane. + /// * `key_selector` - String representation of a selector to extract the map keys from the messages. + /// * `value_selector` - String representation of a selector to extract the map values from the messages. + /// * `remove_when_no_value` - Whether to remove an entry from the map if the value selector does not return a value. Otherwise, missing + /// values will be treated as a failed extraction from the message. + /// * `required` - Whether the lane is required. If this is `true` and the selector returns nothing for a Message, the + /// connector will fail with an error. + pub fn new>( + name: S, + key_selector: S, + value_selector: S, + remove_when_no_value: bool, + required: bool, + ) -> Self { + IngressMapLaneSpec { + name: name.into(), + key_selector: key_selector.into(), + value_selector: value_selector.into(), + remove_when_no_value, + required, + } + } +} + +/// Specification of a value relay for the connector. +#[cfg(feature = "pubsub")] +#[derive(Clone, Debug, Form, PartialEq, Eq)] +#[form(tag = "ValueRelaySpec")] +pub struct PubSubValueRelaySpecification { + /// A node URI selector. See [`crate::selector::NodeSelector`] for more information. + pub node: String, + /// A lane URI selector. See [`crate::selector::LaneSelector`] for more information. + pub lane: String, + /// A payload URI selector. See [`crate::selector::RelayPayloadSelector::value`] for more information. + pub payload: String, + /// Whether the payload selector must yield a value. If it does not, then the selector will + /// yield an error. + pub required: bool, +} + +/// Specification of a map relay for the connector. +#[cfg(feature = "pubsub")] +#[derive(Clone, Debug, Form, PartialEq, Eq)] +#[form(tag = "MapRelaySpec")] +pub struct PubSubMapRelaySpecification { + /// A node URI selector. See [`crate::selector::NodeSelector`] for more information. + pub node: String, + /// A lane URI selector. See [`crate::selector::LaneSelector`] for more information. + pub lane: String, + /// A payload URI selector. See [`crate::selector::RelayPayloadSelector::map`] for more information. + pub key: String, + /// A payload URI selector. See [`crate::selector::RelayPayloadSelector::map`] for more information. + pub value: String, + /// Whether the payload selector must yield a value. If it does not, then the selector will + /// yield an error. + pub required: bool, + /// If the value selector fails to select, then it will emit a map remove command to remove the + /// corresponding entry. + pub remove_when_no_value: bool, +} + +/// Specification of a relay for the connector. +#[cfg(feature = "pubsub")] +#[derive(Clone, Debug, Form, PartialEq, Eq)] +pub enum PubSubRelaySpecification { + /// Specification of a value relay for the connector. + Value(PubSubValueRelaySpecification), + /// Specification of a map relay for the connector. + Map(PubSubMapRelaySpecification), +} + +#[cfg(feature = "pubsub")] +impl TryFrom> for Relays { + type Error = BadSelector; + + fn try_from(value: Vec) -> Result { + use crate::selector::{ + parse_lane_selector, parse_map_selector, parse_node_selector, parse_value_selector, + }; + + let mut chain = Vec::with_capacity(value.len()); + + for spec in value { + match spec { + PubSubRelaySpecification::Value(PubSubValueRelaySpecification { + node, + lane, + payload, + required, + }) => { + let relay = Relay::new( + parse_node_selector(node.as_str())?, + parse_lane_selector(lane.as_str())?, + parse_value_selector(payload.as_str(), required)?, + ); + chain.push(relay); + } + PubSubRelaySpecification::Map(PubSubMapRelaySpecification { + node, + lane, + key, + value, + required, + remove_when_no_value, + }) => { + let relay = Relay::new( + parse_node_selector(node.as_str())?, + parse_lane_selector(lane.as_str())?, + parse_map_selector( + key.as_str(), + value.as_str(), + required, + remove_when_no_value, + )?, + ); + chain.push(relay); + } + } + } + + Ok(Relays::new(chain)) + } +} diff --git a/server/swimos_connector/src/config/mod.rs b/server/swimos_connector/src/config/mod.rs new file mode 100644 index 000000000..1194832f7 --- /dev/null +++ b/server/swimos_connector/src/config/mod.rs @@ -0,0 +1,4 @@ +pub mod format; +mod ingress; + +pub use ingress::*; diff --git a/server/swimos_connector/src/connector/ingress/tests.rs b/server/swimos_connector/src/connector/ingress/tests.rs index 8f5dc1263..bc7827423 100644 --- a/server/swimos_connector/src/connector/ingress/tests.rs +++ b/server/swimos_connector/src/connector/ingress/tests.rs @@ -12,25 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::test_support::{fail, run_handler, TestSpawner}; +use crate::{ConnectorAgent, ConnectorStream}; use bytes::BytesMut; -use futures::stream::FuturesUnordered; use futures::StreamExt; use parking_lot::Mutex; -use std::collections::HashMap; use std::{convert::Infallible, sync::Arc}; -use swimos_agent::agent_model::downlink::BoxDownlinkChannelFactory; -use swimos_agent::event_handler::{ - ActionContext, DownlinkSpawnOnDone, EventHandler, HandlerAction, HandlerFuture, - LaneSpawnOnDone, LaneSpawner, LinkSpawner, Spawner, StepResult, -}; +use swimos_agent::event_handler::{ActionContext, HandlerAction, StepResult}; use swimos_agent::AgentMetadata; -use swimos_api::address::Address; -use swimos_api::agent::WarpLaneKind; -use swimos_api::error::{CommanderRegistrationError, DynamicRegistrationError}; -use swimos_model::Text; - -use crate::test_support::{make_meta, make_uri}; -use crate::{ConnectorAgent, ConnectorStream}; #[derive(Debug)] struct Handler { @@ -77,87 +66,16 @@ async fn drive_connector_stream() { let agent = ConnectorAgent::default(); let handler = super::suspend_connector(make_stream(&state)); - run_handler(&spawner, &agent, handler); + run_handler(&spawner, &mut BytesMut::new(), &agent, handler, fail); let mut n = 0; - while !spawner.futures.is_empty() { + while !spawner.is_empty() { n += 1; - let h = spawner.futures.next().await.expect("Expected future."); - run_handler(&spawner, &agent, h); + let h = spawner.next().await.expect("Expected future."); + run_handler(&spawner, &mut BytesMut::new(), &agent, h, fail); } assert_eq!(n, 4); let guard = state.lock(); assert_eq!(guard.as_ref(), vec![1, 2, 3]) } - -#[derive(Default)] -struct TestSpawner { - futures: FuturesUnordered>, -} - -impl Spawner for TestSpawner { - fn spawn_suspend(&self, fut: HandlerFuture) { - self.futures.push(fut); - } - - fn schedule_timer(&self, _at: tokio::time::Instant, _id: u64) { - panic!("Unexpected timer."); - } -} - -impl LinkSpawner for TestSpawner { - fn spawn_downlink( - &self, - _path: Address, - _make_channel: BoxDownlinkChannelFactory, - _on_done: DownlinkSpawnOnDone, - ) { - panic!("Spawning downlinks not supported."); - } - - fn register_commander(&self, _path: Address) -> Result { - panic!("Registering commanders not supported."); - } -} - -impl LaneSpawner for TestSpawner { - fn spawn_warp_lane( - &self, - _name: &str, - _kind: WarpLaneKind, - _on_done: LaneSpawnOnDone, - ) -> Result<(), DynamicRegistrationError> { - panic!("Spawning lanes not supported."); - } -} - -fn run_handler(spawner: &TestSpawner, agent: &ConnectorAgent, mut handler: H) -where - H: EventHandler, -{ - let uri = make_uri(); - let route_params = HashMap::new(); - let meta = make_meta(&uri, &route_params); - - let mut join_lane_init = HashMap::new(); - let mut command_buffer = BytesMut::new(); - - let mut action_context = ActionContext::new( - spawner, - spawner, - spawner, - &mut join_lane_init, - &mut command_buffer, - ); - - loop { - match handler.step(&mut action_context, meta, agent) { - StepResult::Continue { .. } => {} - StepResult::Fail(err) => panic!("{:?}", err), - StepResult::Complete { .. } => { - break; - } - } - } -} diff --git a/server/swimos_connector_kafka/src/deser/avro/mod.rs b/server/swimos_connector/src/deser/avro/mod.rs similarity index 94% rename from server/swimos_connector_kafka/src/deser/avro/mod.rs rename to server/swimos_connector/src/deser/avro/mod.rs index bae637b92..f67dd70cb 100644 --- a/server/swimos_connector_kafka/src/deser/avro/mod.rs +++ b/server/swimos_connector/src/deser/avro/mod.rs @@ -22,7 +22,7 @@ use chrono::{DateTime, Local, NaiveDateTime, TimeDelta, Utc}; use swimos_form::Form; use swimos_model::{BigInt, Blob, Timestamp}; -use super::{MessageDeserializer, MessagePart, MessageView}; +use super::MessageDeserializer; /// Error type for the Avro deserializer. #[derive(Error, Debug)] @@ -187,17 +187,9 @@ impl ValueAcc { impl MessageDeserializer for AvroDeserializer { type Error = AvroError; - fn deserialize<'a>( - &self, - message: &'a MessageView<'a>, - part: MessagePart, - ) -> Result { + fn deserialize(&self, buf: &[u8]) -> Result { let AvroDeserializer { schema } = self; - let payload = match part { - MessagePart::Key => message.key(), - MessagePart::Payload => message.payload(), - }; - let cursor = Cursor::new(payload); + let cursor = Cursor::new(buf); let reader = if let Some(schema) = schema { apache_avro::Reader::with_schema(schema, cursor) } else { diff --git a/server/swimos_connector_kafka/src/deser/json/mod.rs b/server/swimos_connector/src/deser/json/mod.rs similarity index 81% rename from server/swimos_connector_kafka/src/deser/json/mod.rs rename to server/swimos_connector/src/deser/json/mod.rs index 669089e11..57a9373cb 100644 --- a/server/swimos_connector_kafka/src/deser/json/mod.rs +++ b/server/swimos_connector/src/deser/json/mod.rs @@ -18,7 +18,7 @@ mod tests; use serde_json::Value as JsonValue; use swimos_model::{Item, Value}; -use super::{MessageDeserializer, MessagePart, MessageView}; +use super::MessageDeserializer; fn convert_json_value(input: JsonValue) -> Value { match input { @@ -54,16 +54,8 @@ pub struct JsonDeserializer; impl MessageDeserializer for JsonDeserializer { type Error = serde_json::Error; - fn deserialize<'a>( - &self, - message: &'a MessageView<'a>, - part: MessagePart, - ) -> Result { - let payload = match part { - MessagePart::Key => message.key(), - MessagePart::Payload => message.payload(), - }; - let v: serde_json::Value = serde_json::from_slice(payload)?; + fn deserialize(&self, buf: &[u8]) -> Result { + let v: serde_json::Value = serde_json::from_slice(buf)?; Ok(convert_json_value(v)) } } diff --git a/server/swimos_connector_kafka/src/deser/json/tests.rs b/server/swimos_connector/src/deser/json/tests.rs similarity index 61% rename from server/swimos_connector_kafka/src/deser/json/tests.rs rename to server/swimos_connector/src/deser/json/tests.rs index 4a800b1ba..6f52c8419 100644 --- a/server/swimos_connector_kafka/src/deser/json/tests.rs +++ b/server/swimos_connector/src/deser/json/tests.rs @@ -14,28 +14,14 @@ use swimos_model::{Item, Value}; -use crate::deser::{tests::view_of, JsonDeserializer, MessageDeserializer, MessagePart}; +use crate::deser::{JsonDeserializer, MessageDeserializer}; #[test] fn json_deserializer() { - let deserializer = JsonDeserializer; + let deserializer = JsonDeserializer.boxed(); let bytes: &[u8] = "{ \"a\": 1, \"b\": true}".as_bytes(); let expected = Value::Record(vec![], vec![Item::slot("a", 1), Item::slot("b", true)]); - let view = view_of(bytes, MessagePart::Key); - assert_eq!( - deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), - expected - ); - - let view = view_of(bytes, MessagePart::Payload); - assert_eq!( - deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), - expected - ); + assert_eq!(deserializer.deserialize(bytes).expect("Failed."), expected); } diff --git a/server/swimos_connector_kafka/src/deser/mod.rs b/server/swimos_connector/src/deser/mod.rs similarity index 66% rename from server/swimos_connector_kafka/src/deser/mod.rs rename to server/swimos_connector/src/deser/mod.rs index f07fcd9f0..b2681818c 100644 --- a/server/swimos_connector_kafka/src/deser/mod.rs +++ b/server/swimos_connector/src/deser/mod.rs @@ -26,17 +26,17 @@ pub use json::JsonDeserializer; #[cfg(feature = "avro")] pub use avro::AvroDeserializer; +use std::error::Error; use std::{array::TryFromSliceError, convert::Infallible}; - use swimos_form::Form; use swimos_model::{Blob, Value}; use swimos_recon::parser::{parse_recognize, AsyncParseError}; -use uuid::Uuid; +use uuid::{Bytes, Uuid}; use crate::error::DeserializationError; -/// An uninterpreted view of the components of a Kafka message. +/// An uninterpreted view of the components of a message. pub struct MessageView<'a> { pub topic: &'a str, pub key: &'a [u8], @@ -71,15 +71,11 @@ pub enum MessagePart { Payload, } -/// A deserializer that will attempt to produce a [value](Value) from a component of a Kafka message. +/// A deserializer that will attempt to produce a [value](Value) from a component of a message. pub trait MessageDeserializer { type Error: std::error::Error; - fn deserialize<'a>( - &'a self, - message: &'a MessageView<'a>, - part: MessagePart, - ) -> Result; + fn deserialize(&self, buf: &[u8]) -> Result; fn boxed(self) -> BoxMessageDeserializer where @@ -105,48 +101,24 @@ pub struct ReconDeserializer; impl MessageDeserializer for StringDeserializer { type Error = std::str::Utf8Error; - fn deserialize<'a>( - &self, - message: &'a MessageView<'a>, - part: MessagePart, - ) -> Result { - let payload = match part { - MessagePart::Key => message.key_str(), - MessagePart::Payload => message.payload_str(), - }; - payload.map(Value::text) + fn deserialize(&self, buf: &[u8]) -> Result { + std::str::from_utf8(buf).map(Value::text) } } impl MessageDeserializer for BytesDeserializer { type Error = Infallible; - fn deserialize<'a>( - &self, - message: &'a MessageView<'a>, - part: MessagePart, - ) -> Result { - let payload = match part { - MessagePart::Key => message.key(), - MessagePart::Payload => message.payload(), - }; - Ok(Value::Data(Blob::from_vec(payload.to_vec()))) + fn deserialize(&self, buf: &[u8]) -> Result { + Ok(Value::Data(Blob::from_vec(buf.to_vec()))) } } impl MessageDeserializer for ReconDeserializer { type Error = AsyncParseError; - fn deserialize<'a>( - &self, - message: &'a MessageView<'a>, - part: MessagePart, - ) -> Result { - let payload = match part { - MessagePart::Key => message.key_str(), - MessagePart::Payload => message.payload_str(), - }; - let payload_str = match payload { + fn deserialize(&self, buf: &[u8]) -> Result { + let payload_str = match std::str::from_utf8(buf) { Ok(string) => string, Err(err) => return Err(AsyncParseError::BadUtf8(err)), }; @@ -176,19 +148,11 @@ macro_rules! num_deser { impl MessageDeserializer for $deser { type Error = TryFromSliceError; - fn deserialize<'a>( - &self, - message: &'a MessageView<'a>, - part: MessagePart, - ) -> Result { + fn deserialize<'a>(&self, buf: &[u8]) -> Result { let $deser(endianness) = self; - let payload = match part { - MessagePart::Key => message.key(), - MessagePart::Payload => message.payload(), - }; let x = match endianness { - Endianness::LittleEndian => <$numt>::from_le_bytes(payload.try_into()?), - Endianness::BigEndian => <$numt>::from_be_bytes(payload.try_into()?), + Endianness::LittleEndian => <$numt>::from_le_bytes(buf.try_into()?), + Endianness::BigEndian => <$numt>::from_be_bytes(buf.try_into()?), }; Ok(Value::$variant(x.into())) } @@ -210,16 +174,8 @@ pub struct UuidDeserializer; impl MessageDeserializer for UuidDeserializer { type Error = TryFromSliceError; - fn deserialize<'a>( - &self, - message: &'a MessageView<'a>, - part: MessagePart, - ) -> Result { - let payload = match part { - MessagePart::Key => message.key(), - MessagePart::Payload => message.payload(), - }; - let x = Uuid::from_bytes(payload.try_into()?); + fn deserialize(&self, buf: &[u8]) -> Result { + let x = Uuid::from_bytes(Bytes::try_from(buf)?); Ok(Value::BigInt(x.as_u128().into())) } } @@ -234,13 +190,9 @@ where { type Error = DeserializationError; - fn deserialize<'a>( - &self, - message: &'a MessageView<'a>, - part: MessagePart, - ) -> Result { + fn deserialize(&self, buf: &[u8]) -> Result { self.inner - .deserialize(message, part) + .deserialize(buf) .map_err(DeserializationError::new) } } @@ -251,12 +203,8 @@ pub type BoxMessageDeserializer = impl MessageDeserializer for BoxMessageDeserializer { type Error = DeserializationError; - fn deserialize<'a>( - &self, - message: &'a MessageView<'a>, - part: MessagePart, - ) -> Result { - (**self).deserialize(message, part) + fn deserialize(&self, buf: &[u8]) -> Result { + (**self).deserialize(buf) } fn boxed(self) -> BoxMessageDeserializer @@ -267,3 +215,63 @@ impl MessageDeserializer for BoxMessageDeserializer { self } } + +/// Deserializer which delegates to a function. +#[derive(Clone, Copy, Default, Debug)] +pub struct FnDeserializer(F); + +impl FnDeserializer { + pub fn new(f: F) -> FnDeserializer { + FnDeserializer(f) + } +} + +impl MessageDeserializer for FnDeserializer +where + F: for<'a> Fn(&'a [u8]) -> Result, + E: Error, +{ + type Error = E; + + fn deserialize(&self, buf: &[u8]) -> Result { + self.0(buf) + } +} + +pub struct Deferred<'a> { + buf: &'a [u8], + deser: &'a BoxMessageDeserializer, + state: Option, +} + +impl<'a> Deferred<'a> { + pub fn new(buf: &'a [u8], deser: &'a BoxMessageDeserializer) -> Deferred<'a> { + Deferred { + buf, + deser, + state: None, + } + } + + pub fn get(&mut self) -> Result { + let Deferred { buf, deser, state } = self; + if let Some(v) = state { + Ok(v.clone()) + } else { + Ok(state.insert(deser.deserialize(buf)?).clone()) + } + } + + pub fn with(&mut self, f: F) -> Result, DeserializationError> + where + F: FnOnce(&Value) -> Option, + { + let Deferred { buf, deser, state } = self; + if let Some(v) = state { + Ok(f(v)) + } else { + let val = state.insert(deser.deserialize(buf)?); + Ok(f(val)) + } + } +} diff --git a/server/swimos_connector_kafka/src/deser/tests.rs b/server/swimos_connector/src/deser/tests.rs similarity index 65% rename from server/swimos_connector_kafka/src/deser/tests.rs rename to server/swimos_connector/src/deser/tests.rs index 0564067cc..0bfab2a53 100644 --- a/server/swimos_connector_kafka/src/deser/tests.rs +++ b/server/swimos_connector/src/deser/tests.rs @@ -15,15 +15,12 @@ use swimos_model::{Attr, Item, Value}; use uuid::Uuid; -use crate::{ - deser::{ - F32Deserializer, F64Deserializer, I32Deserializer, I64Deserializer, MessagePart, - ReconDeserializer, U32Deserializer, U64Deserializer, UuidDeserializer, - }, - Endianness, +use crate::deser::{ + F32Deserializer, F64Deserializer, I32Deserializer, I64Deserializer, MessagePart, + ReconDeserializer, U32Deserializer, U64Deserializer, UuidDeserializer, }; -use super::{MessageDeserializer, MessageView, StringDeserializer}; +use super::{Endianness, MessageDeserializer, MessageView, StringDeserializer}; pub fn view_of(bytes: &[u8], part: MessagePart) -> MessageView<'_> { match part { @@ -47,14 +44,11 @@ fn string_deserializer() { let bytes = "hello".as_bytes(); let view = view_of(bytes, MessagePart::Key); - assert_eq!( - deserializer.deserialize(&view, MessagePart::Key), - Ok(Value::text("hello")) - ); + assert_eq!(deserializer.deserialize(view.key), Ok(Value::text("hello"))); let view = view_of(bytes, MessagePart::Payload); assert_eq!( - deserializer.deserialize(&view, MessagePart::Payload), + deserializer.deserialize(view.payload), Ok(Value::text("hello")) ); } @@ -69,33 +63,25 @@ fn i32_deserializer() { let view = view_of(le_bytes, MessagePart::Key); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + le_deserializer.deserialize(view.key).expect("Failed."), Value::from(567i32) ); let view = view_of(le_bytes, MessagePart::Payload); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + le_deserializer.deserialize(view.payload).expect("Failed."), Value::from(567i32) ); let view = view_of(be_bytes, MessagePart::Key); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + be_deserializer.deserialize(view.key).expect("Failed."), Value::from(-874636i32) ); let view = view_of(be_bytes, MessagePart::Payload); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + be_deserializer.deserialize(view.payload).expect("Failed."), Value::from(-874636i32) ); } @@ -110,33 +96,25 @@ fn i64_deserializer() { let view = view_of(le_bytes, MessagePart::Key); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + le_deserializer.deserialize(view.key).expect("Failed."), Value::from(7476383847i64) ); let view = view_of(le_bytes, MessagePart::Payload); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + le_deserializer.deserialize(view.payload).expect("Failed."), Value::from(7476383847i64) ); let view = view_of(be_bytes, MessagePart::Key); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + be_deserializer.deserialize(view.key).expect("Failed."), Value::from(-84728282872734i64) ); let view = view_of(be_bytes, MessagePart::Payload); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + be_deserializer.deserialize(view.payload).expect("Failed."), Value::from(-84728282872734i64) ); } @@ -151,33 +129,25 @@ fn u32_deserializer() { let view = view_of(le_bytes, MessagePart::Key); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + le_deserializer.deserialize(view.key).expect("Failed."), Value::from(567u32) ); let view = view_of(le_bytes, MessagePart::Payload); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + le_deserializer.deserialize(view.payload).expect("Failed."), Value::from(567u32) ); let view = view_of(be_bytes, MessagePart::Key); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + be_deserializer.deserialize(view.key).expect("Failed."), Value::from(874636u32) ); let view = view_of(be_bytes, MessagePart::Payload); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + be_deserializer.deserialize(view.payload).expect("Failed."), Value::from(874636u32) ); } @@ -192,33 +162,25 @@ fn u64_deserializer() { let view = view_of(le_bytes, MessagePart::Key); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + le_deserializer.deserialize(view.key).expect("Failed."), Value::from(7476383847u64) ); let view = view_of(le_bytes, MessagePart::Payload); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + le_deserializer.deserialize(view.payload).expect("Failed."), Value::from(7476383847u64) ); let view = view_of(be_bytes, MessagePart::Key); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + be_deserializer.deserialize(view.key).expect("Failed."), Value::from(84728282872734u64) ); let view = view_of(be_bytes, MessagePart::Payload); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + be_deserializer.deserialize(view.payload).expect("Failed."), Value::from(84728282872734u64) ); } @@ -233,33 +195,25 @@ fn f64_deserializer() { let view = view_of(le_bytes, MessagePart::Key); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + le_deserializer.deserialize(view.key).expect("Failed."), Value::from(4.657366e7) ); let view = view_of(le_bytes, MessagePart::Payload); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + le_deserializer.deserialize(view.payload).expect("Failed."), Value::from(4.657366e7) ); let view = view_of(be_bytes, MessagePart::Key); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + be_deserializer.deserialize(view.key).expect("Failed."), Value::from(-84.657366e-87) ); let view = view_of(be_bytes, MessagePart::Payload); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + be_deserializer.deserialize(view.payload).expect("Failed."), Value::from(-84.657366e-87) ); } @@ -274,33 +228,25 @@ fn f32_deserializer() { let view = view_of(le_bytes, MessagePart::Key); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + le_deserializer.deserialize(view.key).expect("Failed."), Value::from(4.657366e7f32 as f64) ); let view = view_of(le_bytes, MessagePart::Payload); assert_eq!( - le_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + le_deserializer.deserialize(view.payload).expect("Failed."), Value::from(4.657366e7f32 as f64) ); let view = view_of(be_bytes, MessagePart::Key); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + be_deserializer.deserialize(view.key).expect("Failed."), Value::from(-84.6573e-87f32 as f64) ); let view = view_of(be_bytes, MessagePart::Payload); assert_eq!( - be_deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + be_deserializer.deserialize(view.payload).expect("Failed."), Value::from(-84.6573e-87f32 as f64) ); } @@ -316,17 +262,13 @@ fn uuid_deserializer() { let view = view_of(bytes, MessagePart::Key); assert_eq!( - deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + deserializer.deserialize(view.key).expect("Failed."), expected ); let view = view_of(bytes, MessagePart::Payload); assert_eq!( - deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + deserializer.deserialize(view.payload).expect("Failed."), expected ); } @@ -343,17 +285,13 @@ fn recon_deserializer() { let view = view_of(bytes, MessagePart::Key); assert_eq!( - deserializer - .deserialize(&view, MessagePart::Key) - .expect("Failed."), + deserializer.deserialize(view.key).expect("Failed."), expected ); let view = view_of(bytes, MessagePart::Payload); assert_eq!( - deserializer - .deserialize(&view, MessagePart::Payload) - .expect("Failed."), + deserializer.deserialize(view.payload).expect("Failed."), expected ); } diff --git a/server/swimos_connector/src/error.rs b/server/swimos_connector/src/error.rs index e7f5b6c42..419dc60a0 100644 --- a/server/swimos_connector/src/error.rs +++ b/server/swimos_connector/src/error.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use swimos_model::{Value, ValueKind}; use thiserror::Error; /// An error type that is produced by the [`crate::IngressConnectorLifecycle`] if the [`crate::IngressConnector`] that it wraps @@ -19,3 +20,49 @@ use thiserror::Error; #[derive(Clone, Copy, Default, Debug, Error)] #[error("The connector initialization failed to complete.")] pub struct ConnectorInitError; + +/// An error type that boxes any type of error that could be returned by a message deserializer. +#[derive(Debug, Error)] +#[error(transparent)] +pub struct DeserializationError(Box); + +impl DeserializationError { + pub fn new(error: E) -> Self + where + E: std::error::Error + Send + 'static, + { + DeserializationError(Box::new(error)) + } +} + +/// An error type that boxes any type of error that could be returned by a message serializer. +#[derive(Debug, Error)] +pub enum SerializationError { + /// The value is not supported by the serialization format. + #[error("The serializations scheme does not support values of kind: {0}")] + InvalidKind(ValueKind), + /// An integer in a value was out of range for the serialization format. + #[error("Integer value {0} out of range for the serialization scheme.")] + IntegerOutOfRange(Value), + /// An floating point number in a value was out of range for the serialization format. + #[error("Float value {0} out of range for the serialization scheme.")] + FloatOutOfRange(f64), + /// The serializer failed with an error. + #[error("A value serializer failed: {0}")] + SerializerFailed(Box), +} + +/// An error type that can be produced when attempting to load a serializer or deserializer. +#[derive(Debug, Error)] +pub enum LoadError { + /// Attempting to read a required resource (for example, a file) failed. + #[error(transparent)] + Io(#[from] std::io::Error), + /// A required resource was invalid. + #[error(transparent)] + InvalidDescriptor(#[from] Box), + #[error("The configuration provided for the serializer or deserializer is invalid: {0}")] + InvalidConfiguration(String), + #[error("Loading of the configuration was cancelled.")] + Cancelled, +} diff --git a/server/swimos_connector/src/ingress/lanes.rs b/server/swimos_connector/src/ingress/lanes.rs new file mode 100644 index 000000000..1eb29a479 --- /dev/null +++ b/server/swimos_connector/src/ingress/lanes.rs @@ -0,0 +1,91 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::config::{IngressMapLaneSpec, IngressValueLaneSpec}; +use crate::selector::{InvalidLaneSpec, InvalidLanes, MapLaneSelector, ValueLaneSelector}; +use std::collections::HashSet; + +// Information about the lanes of the connector. These are computed from the configuration in the `on_start` handler +// and stored in the lifecycle to be used to start the consumer stream. +#[derive(Debug, Clone)] +pub struct Lanes { + value_lanes: Vec>, + map_lanes: Vec>, +} + +impl Default for Lanes { + fn default() -> Self { + Lanes { + value_lanes: vec![], + map_lanes: vec![], + } + } +} + +impl Lanes { + pub fn value_lanes(&self) -> &[ValueLaneSelector] { + &self.value_lanes + } + + pub fn map_lanes(&self) -> &[MapLaneSelector] { + &self.map_lanes + } + + pub fn try_from_lane_specs<'a>( + value_lanes: &'a [IngressValueLaneSpec], + map_lanes: &'a [IngressMapLaneSpec], + ) -> Result, InvalidLanes> + where + ValueLaneSelector: TryFrom<&'a IngressValueLaneSpec, Error = InvalidLaneSpec>, + MapLaneSelector: TryFrom<&'a IngressMapLaneSpec, Error = InvalidLaneSpec>, + { + let value_selectors = value_lanes + .iter() + .map(ValueLaneSelector::::try_from) + .collect::, _>>()?; + let map_selectors = map_lanes + .iter() + .map(MapLaneSelector::::try_from) + .collect::, _>>()?; + check_selectors(&value_selectors, &map_selectors)?; + Ok(Lanes { + value_lanes: value_selectors, + map_lanes: map_selectors, + }) + } +} + +fn check_selectors( + value_selectors: &[ValueLaneSelector], + map_selectors: &[MapLaneSelector], +) -> Result<(), InvalidLanes> { + let mut names = HashSet::new(); + for value_selector in value_selectors { + let name = value_selector.name(); + if names.contains(name) { + return Err(InvalidLanes::NameCollision(name.to_string())); + } else { + names.insert(name); + } + } + for map_selector in map_selectors { + let name = map_selector.name(); + if names.contains(name) { + return Err(InvalidLanes::NameCollision(name.to_string())); + } else { + names.insert(name); + } + } + Ok(()) +} diff --git a/server/swimos_connector/src/ingress/mod.rs b/server/swimos_connector/src/ingress/mod.rs new file mode 100644 index 000000000..db013aff5 --- /dev/null +++ b/server/swimos_connector/src/ingress/mod.rs @@ -0,0 +1,6 @@ +mod lanes; +#[cfg(feature = "pubsub")] +pub mod pubsub; +#[cfg(test)] +mod tests; +pub use lanes::*; diff --git a/server/swimos_connector/src/ingress/pubsub.rs b/server/swimos_connector/src/ingress/pubsub.rs new file mode 100644 index 000000000..f5cc3784e --- /dev/null +++ b/server/swimos_connector/src/ingress/pubsub.rs @@ -0,0 +1,89 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::deser::{BoxMessageDeserializer, Deferred, MessageView}; +use crate::ingress::lanes::Lanes; +use crate::selector::{PubSubSelector, Relays, SelectHandler, SelectorError}; +use crate::ConnectorAgent; +use frunk::hlist; +use swimos_agent::event_handler::{EventHandler, HandlerActionExt, Sequentially}; +use swimos_model::Value; +use tracing::trace; + +// Uses the information about the lanes of the agent to convert messages into event handlers that update the lanes. +pub struct MessageSelector { + key_deserializer: BoxMessageDeserializer, + value_deserializer: BoxMessageDeserializer, + lanes: Lanes, + relays: Relays, +} + +impl MessageSelector { + pub fn new( + key_deserializer: BoxMessageDeserializer, + value_deserializer: BoxMessageDeserializer, + lanes: Lanes, + relays: Relays, + ) -> Self { + MessageSelector { + key_deserializer, + value_deserializer, + lanes, + relays, + } + } + + pub fn handle_message<'a>( + &self, + message: &'a MessageView<'a>, + ) -> Result + Send + 'static, SelectorError> { + let MessageSelector { + key_deserializer, + value_deserializer, + lanes, + relays, + } = self; + + let value_lanes = lanes.value_lanes(); + let map_lanes = lanes.map_lanes(); + + trace!(topic = { message.topic() }, "Handling a message."); + + let mut value_lane_handlers = Vec::with_capacity(value_lanes.len()); + let mut map_lane_handlers = Vec::with_capacity(map_lanes.len()); + let mut relay_handlers = Vec::with_capacity(relays.len()); + + { + let topic = Value::text(message.topic()); + let key = Deferred::new(message.key, key_deserializer); + let value = Deferred::new(message.payload, value_deserializer); + let mut args = hlist![topic, key, value]; + + for value_lane in value_lanes { + value_lane_handlers.push(value_lane.select_handler(&mut args)?); + } + for map_lane in map_lanes { + map_lane_handlers.push(map_lane.select_handler(&mut args)?); + } + for relay in relays { + relay_handlers.push(relay.select_handler(&mut args)?); + } + } + + let handler = Sequentially::new(value_lane_handlers) + .followed_by(Sequentially::new(map_lane_handlers)) + .followed_by(Sequentially::new(relay_handlers)); + Ok(handler) + } +} diff --git a/server/swimos_connector/src/ingress/tests.rs b/server/swimos_connector/src/ingress/tests.rs new file mode 100644 index 000000000..47ffdbf44 --- /dev/null +++ b/server/swimos_connector/src/ingress/tests.rs @@ -0,0 +1,504 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::config::{IngressMapLaneSpec, IngressValueLaneSpec}; +use crate::deser::MessageDeserializer; +use crate::deser::{Deferred, MessageView, ReconDeserializer}; +use crate::ingress::lanes::Lanes; +use crate::selector::{ + BasicSelector, InvalidLanes, KeySelector, MapLaneSelector, PayloadSelector, PubSubSelector, + PubSubValueLaneSelector, SelectHandler, SelectorError, SlotSelector, +}; +use crate::ConnectorAgent; +use frunk::hlist; +use futures::future::join; +use std::{ + collections::{HashMap, HashSet}, + time::Duration, +}; +use swimos_agent::agent_lifecycle::HandlerContext; +use swimos_agent::agent_model::{AgentSpec, ItemDescriptor, ItemFlags, WarpLaneKind}; +use swimos_agent::event_handler::HandlerActionExt; +use swimos_connector_util::{run_handler, run_handler_with_futures, TestSpawner}; +use swimos_model::{Item, Value}; +use swimos_recon::print_recon_compact; +use swimos_utilities::trigger; +use tokio::time::timeout; + +#[test] +fn lanes_from_spec() { + let value_lanes = vec![ + IngressValueLaneSpec::new(None, "$key", true), + IngressValueLaneSpec::new(Some("name"), "$payload.field", false), + ]; + let map_lanes = vec![IngressMapLaneSpec::new( + "map", "$key", "$payload", true, false, + )]; + let lanes = + Lanes::try_from_lane_specs(&value_lanes, &map_lanes).expect("Invalid specification."); + + let value_lanes = lanes + .value_lanes() + .iter() + .map(|l| l.name()) + .collect::>(); + let map_lanes = lanes + .map_lanes() + .iter() + .map(|l| l.name()) + .collect::>(); + + assert_eq!(&value_lanes, &["key", "name"]); + assert_eq!(&map_lanes, &["map"]); +} + +#[test] +fn value_lane_collision() { + let value_lanes = vec![ + IngressValueLaneSpec::new(None, "$key", true), + IngressValueLaneSpec::new(Some("key"), "$payload.field", false), + ]; + let map_lanes = vec![]; + let err = Lanes::try_from_lane_specs(&value_lanes, &map_lanes).expect_err("Should fail."); + assert_eq!(err, InvalidLanes::NameCollision("key".to_string())) +} + +#[test] +fn map_lane_collision() { + let value_lanes = vec![]; + let map_lanes = vec![ + IngressMapLaneSpec::new("map", "$key", "$payload", true, false), + IngressMapLaneSpec::new("map", "$key[0]", "$payload", true, true), + ]; + let err = Lanes::try_from_lane_specs(&value_lanes, &map_lanes).expect_err("Should fail."); + assert_eq!(err, InvalidLanes::NameCollision("map".to_string())) +} + +#[test] +fn value_map_lane_collision() { + let value_lanes = vec![IngressValueLaneSpec::new( + Some("field"), + "$payload.field", + false, + )]; + let map_lanes = vec![IngressMapLaneSpec::new( + "field", "$key", "$payload", true, false, + )]; + let err = Lanes::try_from_lane_specs(&value_lanes, &map_lanes).expect_err("Should fail."); + assert_eq!(err, InvalidLanes::NameCollision("field".to_string())) +} + +const TEST_TIMEOUT: Duration = Duration::from_secs(5); + +fn setup_agent() -> (ConnectorAgent, HashMap) { + let agent = ConnectorAgent::default(); + let mut ids = HashMap::new(); + let id1 = agent + .register_dynamic_item( + "key", + ItemDescriptor::WarpLane { + kind: WarpLaneKind::Value, + flags: ItemFlags::TRANSIENT, + }, + ) + .expect("Registration failed."); + let id2 = agent + .register_dynamic_item( + "map", + ItemDescriptor::WarpLane { + kind: WarpLaneKind::Map, + flags: ItemFlags::TRANSIENT, + }, + ) + .expect("Registration failed."); + ids.insert("key".to_string(), id1); + ids.insert("map".to_string(), id2); + (agent, ids) +} + +fn make_key_value(key: impl Into, value: impl Into) -> Value { + Value::record(vec![Item::slot("key", key), Item::slot("value", value)]) +} + +fn make_key_only(key: impl Into) -> Value { + Value::record(vec![Item::slot("key", key)]) +} + +#[test] +fn value_lane_selector_handler() { + let (mut agent, ids) = setup_agent(); + + let selector = PubSubValueLaneSelector::new( + "key".to_string(), + PubSubSelector::inject(KeySelector::default()), + true, + ); + + let topic = Value::text("topic_name"); + + let key = Value::from(3).to_string(); + let value = make_key_value("a", 7).to_string(); + let deser = ReconDeserializer.boxed(); + + let deferred_key = Deferred::new(key.as_bytes(), &deser); + let deferred_value = Deferred::new(value.as_bytes(), &deser); + + let mut args = hlist![topic, deferred_key, deferred_value]; + + let handler = selector + .select_handler(&mut args) + .expect("Selector failed."); + let spawner = TestSpawner::default(); + let modified = run_handler(&agent, &spawner, handler); + + assert_eq!(modified, [ids["key"]].into_iter().collect::>()); + let lane = agent.value_lane("key").expect("Lane missing."); + lane.read(|v| assert_eq!(v, &Value::from(3))); +} + +#[test] +fn value_lane_selector_handler_optional_field() { + let (agent, _) = setup_agent(); + + let selector = PubSubSelector::inject(PayloadSelector::from(vec![BasicSelector::Slot( + SlotSelector::for_field("other"), + )])); + + let selector = PubSubValueLaneSelector::new("other".to_string(), selector, false); + + let topic = Value::text("topic_name"); + let key = Value::from(3).to_string(); + let value = make_key_value("a", 7).to_string(); + let deser = ReconDeserializer.boxed(); + + let deferred_key = Deferred::new(key.as_bytes(), &deser); + let deferred_value = Deferred::new(value.as_bytes(), &deser); + + let mut args = hlist![topic, deferred_key, deferred_value]; + + let handler = selector + .select_handler(&mut args) + .expect("Selector failed."); + let spawner = TestSpawner::default(); + let modified = run_handler(&agent, &spawner, handler); + + assert!(modified.is_empty()); +} + +#[test] +fn value_lane_selector_handler_missing_field() { + let selector = PubSubSelector::inject(PayloadSelector::from(vec![BasicSelector::Slot( + SlotSelector::for_field("other"), + )])); + + let selector = PubSubValueLaneSelector::new("other".to_string(), selector, true); + + let topic = Value::text("topic_name"); + let key = Value::from(3).to_string(); + let value = make_key_value("a", 7).to_string(); + let deser = ReconDeserializer.boxed(); + + let deferred_key = Deferred::new(key.as_bytes(), &deser); + let deferred_value = Deferred::new(value.as_bytes(), &deser); + + let mut args = hlist![topic, deferred_key, deferred_value]; + + let error = selector + .select_handler(&mut args) + .expect_err("Should fail."); + assert!(matches!(error, SelectorError::MissingRequiredLane(name) if &name == "other")); +} + +#[test] +fn map_lane_selector_handler() { + let (mut agent, ids) = setup_agent(); + + let key = PubSubSelector::inject(PayloadSelector::from(vec![BasicSelector::Slot( + SlotSelector::for_field("key"), + )])); + let value = PubSubSelector::inject(PayloadSelector::from(vec![BasicSelector::Slot( + SlotSelector::for_field("value"), + )])); + + let selector = MapLaneSelector::new("map".to_string(), key, value, true, false); + + let topic = Value::text("topic_name"); + let key = Value::from(3).to_string(); + let value = make_key_value("a", 7).to_string(); + let deser = ReconDeserializer.boxed(); + + let deferred_key = Deferred::new(key.as_bytes(), &deser); + let deferred_value = Deferred::new(value.as_bytes(), &deser); + + let mut args = hlist![topic, deferred_key, deferred_value]; + + let handler = selector + .select_handler(&mut args) + .expect("Selector failed."); + let spawner = TestSpawner::default(); + let modified = run_handler(&agent, &spawner, handler); + + assert_eq!(modified, [ids["map"]].into_iter().collect::>()); + let lane = agent.map_lane("map").expect("Lane missing."); + lane.get_map(|m| { + let expected = [(Value::text("a"), Value::from(7))] + .into_iter() + .collect::>(); + assert_eq!(m, &expected); + }); +} + +#[test] +fn map_lane_selector_handler_optional_field() { + let (agent, _) = setup_agent(); + + let key = PubSubSelector::inject(PayloadSelector::from(vec![BasicSelector::Slot( + SlotSelector::for_field("key"), + )])); + let value = PubSubSelector::inject(PayloadSelector::from(vec![BasicSelector::Slot( + SlotSelector::for_field("value"), + )])); + + let selector = MapLaneSelector::new("map".to_string(), key, value, false, false); + + let topic = Value::text("topic_name"); + let key = Value::from(3).to_string(); + let value = Value::Extant.to_string(); + let deser = ReconDeserializer.boxed(); + + let deferred_key = Deferred::new(key.as_bytes(), &deser); + let deferred_value = Deferred::new(value.as_bytes(), &deser); + + let mut args = hlist![topic, deferred_key, deferred_value]; + + let handler = selector + .select_handler(&mut args) + .expect("Selector failed."); + let spawner = TestSpawner::default(); + let modified = run_handler(&agent, &spawner, handler); + + assert!(modified.is_empty()); +} + +#[test] +fn map_lane_selector_handler_missing_field() { + let key = PubSubSelector::inject(PayloadSelector::from(vec![BasicSelector::Slot( + SlotSelector::for_field("key"), + )])); + let value = PubSubSelector::inject(PayloadSelector::from(vec![BasicSelector::Slot( + SlotSelector::for_field("value"), + )])); + + let selector = MapLaneSelector::new("map".to_string(), key, value, true, false); + + let topic = Value::text("topic_name"); + let key = Value::from(3).to_string(); + let value = Value::Extant.to_string(); + let deser = ReconDeserializer.boxed(); + + let deferred_key = Deferred::new(key.as_bytes(), &deser); + let deferred_value = Deferred::new(value.as_bytes(), &deser); + + let mut args = hlist![topic, deferred_key, deferred_value]; + + let error = selector + .select_handler(&mut args) + .expect_err("Should fail."); + assert!(matches!(error, SelectorError::MissingRequiredLane(name) if &name == "map")); +} + +#[test] +fn map_lane_selector_remove() { + let (mut agent, ids) = setup_agent(); + + let key = PubSubSelector::inject(PayloadSelector::from(vec![BasicSelector::Slot( + SlotSelector::for_field("key"), + )])); + let value = PubSubSelector::inject(PayloadSelector::from(vec![BasicSelector::Slot( + SlotSelector::for_field("value"), + )])); + + let selector = MapLaneSelector::new("map".to_string(), key, value, true, true); + + let topic = Value::text("topic_name"); + let key = Value::from(3).to_string(); + let value = make_key_value("a", 7).to_string(); + let deser = ReconDeserializer.boxed(); + + let deferred_key = Deferred::new(key.as_bytes(), &deser); + let deferred_value = Deferred::new(value.as_bytes(), &deser); + + let mut args = hlist![topic.clone(), deferred_key, deferred_value]; + + let update_handler = selector + .select_handler(&mut args) + .expect("Selector failed."); + let spawner = TestSpawner::default(); + let modified = run_handler(&agent, &spawner, update_handler); + + assert_eq!(modified, [ids["map"]].into_iter().collect::>()); + let lane = agent.map_lane("map").expect("Lane missing."); + lane.get_map(|m| { + let expected = [(Value::text("a"), Value::from(7))] + .into_iter() + .collect::>(); + assert_eq!(m, &expected); + }); + + drop(lane); + + let value2 = make_key_only("a").to_string(); + let deferred_key = Deferred::new(key.as_bytes(), &deser); + let deferred_value = Deferred::new(value2.as_bytes(), &deser); + let mut args = hlist![topic, deferred_key, deferred_value]; + + let remove_handler = selector + .select_handler(&mut args) + .expect("Selector failed."); + let modified = run_handler(&agent, &spawner, remove_handler); + + assert_eq!(modified, [ids["map"]].into_iter().collect::>()); + let lane = agent.map_lane("map").expect("Lane missing."); + lane.get_map(|m| { + assert!(m.is_empty()); + }); +} + +#[cfg(feature = "pubsub")] +#[tokio::test] +async fn handle_message() { + let value_specs = vec![IngressValueLaneSpec::new(None, "$key", true)]; + let map_specs = vec![IngressMapLaneSpec::new( + "map", + "$payload.key", + "$payload.value", + true, + true, + )]; + let lanes = + Lanes::try_from_lane_specs(&value_specs, &map_specs).expect("Invalid specifications."); + + let (agent, ids) = setup_agent(); + + let selector = super::pubsub::MessageSelector::new( + ReconDeserializer.boxed(), + ReconDeserializer.boxed(), + lanes, + Default::default(), + ); + + let key = Value::from(3); + let payload = make_key_value("ab", 67); + let key_str = format!("{}", print_recon_compact(&key)); + let payload_str = format!("{}", print_recon_compact(&payload)); + + let message = MessageView { + topic: "topic_name", + key: key_str.as_bytes(), + payload: payload_str.as_bytes(), + }; + + let (tx, rx) = trigger::trigger(); + + let handler = selector + .handle_message(&message) + .map(|handler| { + handler.followed_by(HandlerContext::default().effect(move || { + let _ = tx.trigger(); + })) + }) + .expect("Selector failed."); + + let handler_task = run_handler_with_futures(&agent, handler); + + let (modified, done_result) = timeout(TEST_TIMEOUT, join(handler_task, rx)) + .await + .expect("Test timed out."); + + assert!(done_result.is_ok()); + assert_eq!(modified, ids.values().copied().collect::>()); +} + +#[cfg(feature = "pubsub")] +#[tokio::test] +async fn handle_message_missing_field() { + let value_specs = vec![IngressValueLaneSpec::new(None, "$key", true)]; + let map_specs = vec![IngressMapLaneSpec::new( + "map", + "$payload.key", + "$payload.value", + true, + true, + )]; + let lanes = + Lanes::try_from_lane_specs(&value_specs, &map_specs).expect("Invalid specifications."); + + let selector = super::pubsub::MessageSelector::new( + ReconDeserializer.boxed(), + ReconDeserializer.boxed(), + lanes, + Default::default(), + ); + + let key = Value::from(3); + let payload = Value::text("word"); + let key_str = format!("{}", print_recon_compact(&key)); + let payload_str = format!("{}", print_recon_compact(&payload)); + + let message = MessageView { + topic: "topic_name", + key: key_str.as_bytes(), + payload: payload_str.as_bytes(), + }; + + let result = selector.handle_message(&message); + assert!(matches!(result, Err(SelectorError::MissingRequiredLane(name)) if name == "map")); +} + +#[cfg(feature = "pubsub")] +#[tokio::test] +async fn handle_message_bad_data() { + let value_specs = vec![IngressValueLaneSpec::new(None, "$key", true)]; + let map_specs = vec![IngressMapLaneSpec::new( + "map", + "$payload.key", + "$payload.value", + true, + true, + )]; + let lanes = + Lanes::try_from_lane_specs(&value_specs, &map_specs).expect("Invalid specifications."); + + let selector = super::pubsub::MessageSelector::new( + ReconDeserializer.boxed(), + ReconDeserializer.boxed(), + lanes, + Default::default(), + ); + + let key = Value::from(3); + let key_str = format!("{}", print_recon_compact(&key)); + + let message = MessageView { + topic: "topic_name", + key: key_str.as_bytes(), + payload: b"^*$&@*@", + }; + + let result = selector.handle_message(&message); + assert!(matches!( + result, + Err(SelectorError::DeserializationFailed(_)) + )); +} diff --git a/server/swimos_connector/src/lib.rs b/server/swimos_connector/src/lib.rs index d20a2cff6..78d79272b 100644 --- a/server/swimos_connector/src/lib.rs +++ b/server/swimos_connector/src/lib.rs @@ -17,15 +17,21 @@ mod error; mod generic; mod lifecycle; mod route; - #[cfg(test)] mod test_support; + +pub mod config; +pub mod deser; +pub mod ingress; +pub mod selector; +pub mod ser; + pub use connector::{ BaseConnector, ConnectorFuture, ConnectorHandler, ConnectorStream, EgressConnector, EgressConnectorSender, EgressContext, IngressConnector, IngressContext, MessageSource, SendResult, }; -pub use error::ConnectorInitError; +pub use error::{ConnectorInitError, DeserializationError, LoadError, SerializationError}; pub use generic::{ConnectorAgent, MapLaneSelectorFn, ValueLaneSelectorFn}; pub use lifecycle::{EgressConnectorLifecycle, IngressConnectorLifecycle}; pub use route::{EgressConnectorModel, IngressConnectorModel}; diff --git a/server/swimos_connector/src/selector/base.rs b/server/swimos_connector/src/selector/base.rs new file mode 100644 index 000000000..26a9e76c3 --- /dev/null +++ b/server/swimos_connector/src/selector/base.rs @@ -0,0 +1,425 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::selector::{SelectHandler, Selector, SelectorError, ValueSelector}; +use crate::{ConnectorAgent, MapLaneSelectorFn, ValueLaneSelectorFn}; +use frunk::Coprod; +use swimos_agent::event_handler::{Discard, HandlerActionExt}; +use swimos_agent::lanes::{MapLaneSelectRemove, MapLaneSelectUpdate, ValueLaneSelectSet}; +use swimos_model::{Attr, Item, Text, Value}; +use tracing::{error, trace}; + +type MapLaneUpdate = MapLaneSelectUpdate; +type MapLaneRemove = MapLaneSelectRemove; +type MapLaneOp = Coprod!(MapLaneUpdate, MapLaneRemove); +pub type GenericMapLaneOp = Discard>; +pub type GenericValueLaneSet = + Discard>>; + +#[derive(Debug, PartialEq)] +pub struct ValueLaneSelector { + name: String, + selector: S, + required: bool, +} + +impl Clone for ValueLaneSelector +where + S: Clone, +{ + fn clone(&self) -> Self { + ValueLaneSelector { + name: self.name.clone(), + selector: self.selector.clone(), + required: self.required, + } + } +} + +impl ValueLaneSelector { + pub fn new(name: String, selector: S, required: bool) -> ValueLaneSelector { + ValueLaneSelector { + name, + selector, + required, + } + } + + pub fn name(&self) -> &str { + self.name.as_str() + } + + pub fn is_required(&self) -> bool { + self.required + } + + pub fn into_selector(self) -> S { + self.selector + } +} + +impl SelectHandler for ValueLaneSelector +where + S: Selector, +{ + type Handler = GenericValueLaneSet; + + fn select_handler(&self, args: &mut A) -> Result { + let ValueLaneSelector { + name, + selector, + required, + } = self; + + let maybe_value = selector.select(args)?; + let handler = match maybe_value { + Some(value) => { + trace!(name, value = %value, "Setting a value extracted from a message to a value lane."); + let select_lane = ValueLaneSelectorFn::new(name.clone()); + Some(ValueLaneSelectSet::new(select_lane, value)) + } + None => { + if *required { + error!(name, "A message did not contain a required value."); + return Err(SelectorError::MissingRequiredLane(name.clone())); + } else { + None + } + } + }; + Ok(handler.discard()) + } +} + +/// Trivial selector that chooses the entire input value. +#[derive(Debug, Clone, Copy, Default)] +pub struct IdentitySelector; + +impl ValueSelector for IdentitySelector { + fn select_value<'a>(&self, value: &'a Value) -> Option<&'a Value> { + Some(value) + } +} + +/// A selector that chooses the value of a named attribute if the value is a record and that attribute exists. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AttrSelector { + select_name: String, +} + +impl AttrSelector { + /// # Arguments + /// * `name` - The name of the attribute. + pub fn new(name: String) -> Self { + AttrSelector { select_name: name } + } +} + +impl ValueSelector for AttrSelector { + fn select_value<'a>(&self, value: &'a Value) -> Option<&'a Value> { + let AttrSelector { select_name } = self; + match value { + Value::Record(attrs, _) => attrs.iter().find_map(|Attr { name, value }: &Attr| { + if name.as_str() == select_name.as_str() { + Some(value) + } else { + None + } + }), + _ => None, + } + } +} + +/// A selector that chooses the value of a slot if the value is a record and that slot exists. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SlotSelector { + select_key: Value, +} + +impl SlotSelector { + /// Construct a slot selector for a named field. + /// + /// # Arguments + /// * `name` - The name of the field. + pub fn for_field(name: impl Into) -> Self { + SlotSelector { + select_key: Value::text(name), + } + } +} + +impl ValueSelector for SlotSelector { + fn select_value<'a>(&self, value: &'a Value) -> Option<&'a Value> { + let SlotSelector { select_key } = self; + match value { + Value::Record(_, items) => items.iter().find_map(|item: &Item| match item { + Item::Slot(key, value) if key == select_key => Some(value), + _ => None, + }), + _ => None, + } + } +} + +/// A selector that chooses an item by index if the value is a record and has a sufficient number of items. If +/// the selected item is a slot, its value is selected. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IndexSelector { + index: usize, +} + +impl IndexSelector { + /// # Arguments + /// * `index` - The index in the record to select. + pub fn new(index: usize) -> Self { + IndexSelector { index } + } +} + +impl ValueSelector for IndexSelector { + fn select_value<'a>(&self, value: &'a Value) -> Option<&'a Value> { + let IndexSelector { index } = self; + match value { + Value::Record(_, items) => items.get(*index).map(|item| match item { + Item::ValueItem(v) => v, + Item::Slot(_, v) => v, + }), + _ => None, + } + } +} + +/// One of an [`AttrSelector`], [`SlotSelector`] or [`IndexSelector`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BasicSelector { + Attr(AttrSelector), + Slot(SlotSelector), + Index(IndexSelector), +} + +impl From for BasicSelector { + fn from(value: AttrSelector) -> Self { + BasicSelector::Attr(value) + } +} + +impl From for BasicSelector { + fn from(value: SlotSelector) -> Self { + BasicSelector::Slot(value) + } +} + +impl From for BasicSelector { + fn from(value: IndexSelector) -> Self { + BasicSelector::Index(value) + } +} + +impl ValueSelector for BasicSelector { + fn select_value<'a>(&self, value: &'a Value) -> Option<&'a Value> { + match self { + BasicSelector::Attr(s) => ValueSelector::select_value(s, value), + BasicSelector::Slot(s) => ValueSelector::select_value(s, value), + BasicSelector::Index(s) => ValueSelector::select_value(s, value), + } + } +} + +/// A selector that applies a sequence of simpler selectors, in order, passing the result of one selector to the next. +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct ChainSelector(Vec); + +impl From for ChainSelector +where + I: IntoIterator, +{ + fn from(value: I) -> Self { + ChainSelector(value.into_iter().collect()) + } +} + +impl ChainSelector { + pub fn new(index: Option, components: &[SelectorComponent<'_>]) -> ChainSelector { + let mut links = vec![]; + if let Some(n) = index { + links.push(BasicSelector::Index(IndexSelector::new(n))); + } + for SelectorComponent { + is_attr, + name, + index, + } in components + { + links.push(if *is_attr { + BasicSelector::Attr(AttrSelector::new(name.to_string())) + } else { + BasicSelector::Slot(SlotSelector::for_field(*name)) + }); + if let Some(n) = index { + links.push(BasicSelector::Index(IndexSelector::new(*n))); + } + } + ChainSelector::from(links) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SelectorComponent<'a> { + pub is_attr: bool, + pub name: &'a str, + pub index: Option, +} + +impl<'a> SelectorComponent<'a> { + pub fn new(is_attr: bool, name: &'a str, index: Option) -> Self { + SelectorComponent { + is_attr, + name, + index, + } + } +} + +impl ValueSelector for ChainSelector { + fn select_value<'a>(&self, value: &'a Value) -> Option<&'a Value> { + let mut v = Some(value); + let ChainSelector(selectors) = self; + for s in selectors { + let selected = if let Some(v) = v { + ValueSelector::select_value(s, v) + } else { + break; + }; + v = selected; + } + v + } +} + +/// A value lane selector generates event handlers from messages to update the state of a map lane. +#[derive(Debug)] +pub struct MapLaneSelector { + name: String, + key_selector: K, + value_selector: V, + required: bool, + remove_when_no_value: bool, +} + +impl Clone for MapLaneSelector +where + K: Clone, + V: Clone, +{ + fn clone(&self) -> Self { + MapLaneSelector { + name: self.name.clone(), + key_selector: self.key_selector.clone(), + value_selector: self.value_selector.clone(), + required: self.required, + remove_when_no_value: self.remove_when_no_value, + } + } +} + +impl MapLaneSelector { + /// # Arguments + /// * `name` - The name of the lane. + /// * `key_selector` - Selects a component from the message for the map key. + /// * `value_selector` - Selects a component from the message for the map value. + /// * `required` - If this is required and the selectors do not return a result, an error will be generated. + /// * `remove_when_no_value` - If a key is selected but no value is selected, the corresponding entry will be + /// removed from the map. + pub fn new( + name: String, + key_selector: K, + value_selector: V, + required: bool, + remove_when_no_value: bool, + ) -> Self { + MapLaneSelector { + name, + key_selector, + value_selector, + required, + remove_when_no_value, + } + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn is_required(&self) -> bool { + self.required + } + + pub fn remove_when_no_value(&self) -> bool { + self.remove_when_no_value + } + + pub fn into_selectors(self) -> (K, V) { + let MapLaneSelector { + key_selector, + value_selector, + .. + } = self; + (key_selector, value_selector) + } +} + +impl SelectHandler for MapLaneSelector +where + K: Selector, + V: Selector, +{ + type Handler = GenericMapLaneOp; + + fn select_handler(&self, from: &mut A) -> Result { + let MapLaneSelector { + name, + key_selector, + value_selector, + required, + remove_when_no_value, + } = self; + + let maybe_key: Option = key_selector.select(from)?; + let maybe_value = value_selector.select(from)?; + let select_lane = MapLaneSelectorFn::new(name.clone()); + + let handler: Option = match (maybe_key, maybe_value) { + (None, _) if *required => { + error!( + name, + "A message did not contain a required map lane update/removal." + ); + return Err(SelectorError::MissingRequiredLane(name.clone())); + } + (Some(key), None) if *remove_when_no_value => { + trace!(name, key = %key, "Removing an entry from a map lane with a key extracted from a message."); + let remove = MapLaneSelectRemove::new(select_lane, key); + Some(MapLaneOp::inject(remove)) + } + (Some(key), Some(value)) => { + trace!(name, key = %key, value = %value, "Updating a map lane with an entry extracted from a message."); + let update = MapLaneSelectUpdate::new(select_lane, key, value); + Some(MapLaneOp::inject(update)) + } + _ => None, + }; + Ok(handler.discard()) + } +} diff --git a/server/swimos_connector/src/selector/mod.rs b/server/swimos_connector/src/selector/mod.rs new file mode 100644 index 000000000..c7db68b4f --- /dev/null +++ b/server/swimos_connector/src/selector/mod.rs @@ -0,0 +1,27 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod base; +mod model; + +pub use base::*; +mod relay; +pub use model::*; +pub use relay::*; + +#[cfg(feature = "pubsub")] +mod pubsub; + +#[cfg(feature = "pubsub")] +pub use pubsub::*; diff --git a/server/swimos_connector/src/selector/model.rs b/server/swimos_connector/src/selector/model.rs new file mode 100644 index 000000000..4c1c098f5 --- /dev/null +++ b/server/swimos_connector/src/selector/model.rs @@ -0,0 +1,64 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::selector::SelectorError; +use crate::{ConnectorAgent, DeserializationError}; +use frunk::{hlist::HList, HCons, HNil}; +use std::fmt::Debug; +use swimos_agent::{ + event_handler::HandlerAction, + reexport::coproduct::{CNil, Coproduct}, +}; +use swimos_model::Value; + +/// A value selector attempts to choose some sub-component of a [`Value`], matching against a +/// pattern, returning nothing if the pattern does not match. +pub trait ValueSelector: Debug { + /// Attempt to select some sub-component of the provided [`Value`]. + fn select_value<'a>(&self, value: &'a Value) -> Option<&'a Value>; +} + +/// A dynamic selector which attempts to choose some sub-component of a [`Value`] from some +/// arguments, matching against a pattern, returning nothing if the pattern does not match. +pub trait Selector { + /// Attempt to select some sub-component of the provided [`Value`] from the arguments this + /// selector accepts. + fn select(&self, from: &mut A) -> Result, DeserializationError>; +} + +impl Selector> for Coproduct +where + L: Selector, + R: Selector, + Tail: HList, +{ + fn select(&self, from: &mut HCons) -> Result, DeserializationError> { + match self { + Coproduct::Inl(l) => l.select(&mut from.head), + Coproduct::Inr(r) => r.select(&mut from.tail), + } + } +} + +impl Selector for CNil { + fn select(&self, _from: &mut HNil) -> Result, DeserializationError> { + Ok(None) + } +} + +pub trait SelectHandler { + type Handler: HandlerAction + 'static; + + fn select_handler(&self, args: &mut A) -> Result; +} diff --git a/server/swimos_connector/src/selector/pubsub/error.rs b/server/swimos_connector/src/selector/pubsub/error.rs new file mode 100644 index 000000000..c8e927a2c --- /dev/null +++ b/server/swimos_connector/src/selector/pubsub/error.rs @@ -0,0 +1,94 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::DeserializationError; +use std::num::ParseIntError; +use swimos_recon::parser::ParseError; +use thiserror::Error; + +/// Errors that can be produced attempting to select components of a Message. +#[derive(Debug, Error)] +pub enum SelectorError { + /// A selector failed to provide a value for a required lane. + #[error("The lane '{0}' is required but did not occur in a message.")] + MissingRequiredLane(String), + /// Deserializing a component of a message failed. + #[error("Deserializing the content of a message failed: {0}")] + DeserializationFailed(#[from] DeserializationError), + #[error("Selector `{0}` failed to select from input")] + Selector(String), + /// A node or lane selector failed to select from a record as it was not a primitive type. + #[error("Invalid record structure. Expected a primitive type")] + InvalidRecord(String), +} + +/// Error type for an invalid lane selector specification. +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum InvalidLaneSpec { + /// The string describing the selector was invalid. + #[error(transparent)] + Selector(#[from] BadSelector), + /// The lane name could not be inferred from the selector and was not provided explicitly. + #[error("No name provided and it cannot be inferred from the selector.")] + NameCannotBeInferred, +} + +/// Error type produced for invalid lane descriptors. +#[derive(Clone, Debug, Error, PartialEq, Eq)] +pub enum InvalidLanes { + /// The specification of a line as not valid + #[error(transparent)] + Spec(#[from] InvalidLaneSpec), + /// A connector has too many lanes. + /// There are lane descriptors with the same name. + #[error("The lane name {0} occurs more than once.")] + NameCollision(String), +} + +/// Error type for an invalid selector descriptor. +#[derive(Clone, Error, Debug, PartialEq, Eq)] +pub enum BadSelector { + /// An empty string does not describe a valid selector. + #[error("Selector strings cannot be empty.")] + EmptySelector, + /// A selector component may not be empty. + #[error("Selector components cannot be empty.")] + EmptyComponent, + /// The root of a selector must be a valid component of a message. + #[error("Invalid root selector (must be one of '$key' or '$payload' with an optional index or '$topic').")] + InvalidRoot, + /// A component of the descriptor did not describe a valid selector. + #[error( + "Invalid component selector (must be an attribute or slot name with an optional index)." + )] + InvalidComponent, + /// The index for an index selector was too large for usize. + #[error("An index specified was not a valid usize.")] + IndexOutOfRange, + /// The topic root cannot have any other components. + #[error("The topic does not have components.")] + TopicWithComponent, + /// A provided lane/node/payload selector was malformed. + #[error("The selector formed an invalid path.")] + InvalidPath, + /// Invalid Recon was specified for a payload selector. + #[error("Failed to parse Recon value for selector payload")] + InvalidRecon(#[from] ParseError), +} + +impl From for BadSelector { + fn from(_value: ParseIntError) -> Self { + BadSelector::IndexOutOfRange + } +} diff --git a/server/swimos_connector/src/selector/pubsub/mod.rs b/server/swimos_connector/src/selector/pubsub/mod.rs new file mode 100644 index 000000000..e9575d4d5 --- /dev/null +++ b/server/swimos_connector/src/selector/pubsub/mod.rs @@ -0,0 +1,351 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod error; +mod relay; +#[cfg(test)] +mod tests; + +use crate::config::{IngressMapLaneSpec, IngressValueLaneSpec}; +use crate::deser::{Deferred, MessagePart}; +use crate::selector::{MapLaneSelector, SelectorComponent, ValueLaneSelector}; +use crate::{ + selector::{ChainSelector, Selector, ValueSelector}, + DeserializationError, +}; +pub use error::*; +use frunk::{Coprod, HList}; +use regex::Regex; +pub use relay::{ + parse_lane_selector, parse_map_selector, parse_node_selector, parse_value_selector, +}; +use std::sync::OnceLock; +use swimos_model::Value; + +/// Canonical selector for pub-sub type connectors. +pub type PubSubSelector = Coprod!(TopicSelector, KeySelector, PayloadSelector); +pub type PubSubSelectorArgs<'a> = HList!(Value, Deferred<'a>, Deferred<'a>); +/// Selector type for Value Lanes. +pub type PubSubValueLaneSelector = ValueLaneSelector; +/// Selector type for Map Lanes. +pub type PubSubMapLaneSelector = MapLaneSelector; + +#[derive(Debug, PartialEq, Clone, Eq, Default)] +pub struct TopicSelector; + +impl Selector for TopicSelector { + fn select(&self, from: &mut Value) -> Result, DeserializationError> { + Ok(Some(from.clone())) + } +} + +#[derive(Debug, PartialEq, Clone, Eq, Default)] +pub struct KeySelector(ChainSelector); + +impl KeySelector { + pub fn new(inner: ChainSelector) -> KeySelector { + KeySelector(inner) + } +} + +impl From for KeySelector +where + I: Into, +{ + fn from(value: I) -> Self { + KeySelector(value.into()) + } +} + +impl Selector> for KeySelector { + fn select(&self, from: &mut Deferred<'_>) -> Result, DeserializationError> { + let KeySelector(chain) = self; + from.with(|val| ValueSelector::select_value(chain, val).cloned()) + } +} + +#[derive(Default, Debug, PartialEq, Clone, Eq)] +pub struct PayloadSelector(ChainSelector); + +impl PayloadSelector { + pub fn new(inner: ChainSelector) -> PayloadSelector { + PayloadSelector(inner) + } +} + +impl From for PayloadSelector +where + I: Into, +{ + fn from(value: I) -> Self { + PayloadSelector::new(value.into()) + } +} + +impl Selector> for PayloadSelector { + fn select(&self, from: &mut Deferred<'_>) -> Result, DeserializationError> { + let PayloadSelector(chain) = self; + from.with(|val| ValueSelector::select_value(chain, val).cloned()) + } +} + +/// Enumeration of the components of a message. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MessageField { + Key, + Payload, + Topic, +} + +impl From for MessageField { + fn from(value: MessagePart) -> Self { + match value { + MessagePart::Key => MessageField::Key, + MessagePart::Payload => MessageField::Payload, + } + } +} + +static INIT_REGEX: OnceLock = OnceLock::new(); +static FIELD_REGEX: OnceLock = OnceLock::new(); + +/// Regular expression matching the base selectors for topic, key and payload components. +fn init_regex() -> &'static Regex { + INIT_REGEX.get_or_init(|| create_init_regex().expect("Invalid regex.")) +} + +/// Regular expression matching the description of a [`BasicSelector`]. +fn field_regex() -> &'static Regex { + FIELD_REGEX.get_or_init(|| create_field_regex().expect("Invalid regex.")) +} + +fn create_init_regex() -> Result { + Regex::new("\\A(\\$(?:[a-z]+))(?:\\[(\\d+)])?\\z") +} + +fn create_field_regex() -> Result { + Regex::new("\\A(\\@?(?:\\w+))(?:\\[(\\d+)])?\\z") +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RawSelectorDescriptor<'a> { + pub part: &'a str, + pub index: Option, + pub components: Vec>, +} + +// can't use FromStr as it doesn't have a lifetime associated with it. +impl<'a> TryFrom<&'a str> for RawSelectorDescriptor<'a> { + type Error = BadSelector; + + fn try_from(descriptor: &'a str) -> Result { + if descriptor.is_empty() { + return Err(BadSelector::EmptySelector); + } + let mut it = descriptor.split('.'); + let (field, index) = match it.next() { + Some(root) if !root.is_empty() => { + if let Some(captures) = init_regex().captures(root) { + let field = match captures.get(1) { + Some(kind) => kind.as_str(), + _ => return Err(BadSelector::InvalidRoot), + }; + let index = if let Some(index_match) = captures.get(2) { + Some(index_match.as_str().parse::()?) + } else { + None + }; + (field, index) + } else { + return Err(BadSelector::InvalidRoot); + } + } + _ => return Err(BadSelector::EmptyComponent), + }; + + let mut components = vec![]; + for part in it { + if part.is_empty() { + return Err(BadSelector::EmptyComponent); + } + if let Some(captures) = field_regex().captures(part) { + let (is_attr, name) = match captures.get(1) { + Some(name) if name.as_str().starts_with('@') => (true, &name.as_str()[1..]), + Some(name) => (false, name.as_str()), + _ => return Err(BadSelector::InvalidComponent), + }; + let index = if let Some(index_match) = captures.get(2) { + Some(index_match.as_str().parse::()?) + } else { + None + }; + components.push(SelectorComponent::new(is_attr, name, index)); + } else { + return Err(BadSelector::InvalidRoot); + } + } + + Ok(RawSelectorDescriptor { + part: field, + index, + components, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SelectorDescriptor<'a> { + Part { + part: MessagePart, + index: Option, + components: Vec>, + }, + Topic, +} + +impl<'a> TryFrom> for SelectorDescriptor<'a> { + type Error = BadSelector; + + fn try_from(value: RawSelectorDescriptor<'a>) -> Result { + let RawSelectorDescriptor { + part, + index, + components, + } = value; + match part { + "$topic" => { + if index.is_none() && components.is_empty() { + Ok(SelectorDescriptor::Topic) + } else { + Err(BadSelector::TopicWithComponent) + } + } + "$key" => Ok(SelectorDescriptor::Part { + part: MessagePart::Key, + index, + components, + }), + "$payload" => Ok(SelectorDescriptor::Part { + part: MessagePart::Payload, + index, + components, + }), + _ => Err(BadSelector::InvalidRoot), + } + } +} + +impl<'a> SelectorDescriptor<'a> { + pub fn field(&self) -> MessageField { + match self { + SelectorDescriptor::Part { part, .. } => (*part).into(), + SelectorDescriptor::Topic => MessageField::Topic, + } + } + + pub fn suggested_name(&self) -> Option<&'a str> { + match self { + SelectorDescriptor::Part { + part, + index, + components, + } => { + if let Some(SelectorComponent { name, index, .. }) = components.last() { + if index.is_none() { + Some(*name) + } else { + None + } + } else if index.is_none() { + Some(match part { + MessagePart::Key => "key", + MessagePart::Payload => "payload", + }) + } else { + None + } + } + SelectorDescriptor::Topic => Some("topic"), + } + } + + pub fn selector(&self) -> Option { + match self { + SelectorDescriptor::Part { + index, components, .. + } => Some(ChainSelector::new(*index, components)), + SelectorDescriptor::Topic => None, + } + } +} + +/// Attempt to parse a descriptor for a selector from a string. +pub fn parse_selector(descriptor: &str) -> Result, BadSelector> { + RawSelectorDescriptor::try_from(descriptor)?.try_into() +} + +impl TryFrom<&IngressValueLaneSpec> for PubSubValueLaneSelector { + type Error = InvalidLaneSpec; + + fn try_from(value: &IngressValueLaneSpec) -> Result { + let IngressValueLaneSpec { + name, + selector, + required, + } = value; + let parsed = parse_selector(selector.as_str())?; + if let Some(lane_name) = name + .as_ref() + .cloned() + .or_else(|| parsed.suggested_name().map(|s| s.to_owned())) + { + Ok(ValueLaneSelector::new(lane_name, parsed.into(), *required)) + } else { + Err(InvalidLaneSpec::NameCannotBeInferred) + } + } +} + +impl<'a> From> for PubSubSelector { + fn from(parsed: SelectorDescriptor<'a>) -> Self { + match (parsed.field(), parsed.selector()) { + (MessageField::Key, Some(selector)) => PubSubSelector::inject(KeySelector(selector)), + (MessageField::Payload, Some(selector)) => { + PubSubSelector::inject(PayloadSelector(selector)) + } + _ => PubSubSelector::inject(TopicSelector), + } + } +} + +impl TryFrom<&IngressMapLaneSpec> for PubSubMapLaneSelector { + type Error = InvalidLaneSpec; + + fn try_from(value: &IngressMapLaneSpec) -> Result { + let IngressMapLaneSpec { + name, + key_selector, + value_selector, + remove_when_no_value, + required, + } = value; + Ok(PubSubMapLaneSelector::new( + name.clone(), + parse_selector(key_selector.as_str())?.into(), + parse_selector(value_selector.as_str())?.into(), + *required, + *remove_when_no_value, + )) + } +} diff --git a/server/swimos_connector/src/selector/pubsub/relay/mod.rs b/server/swimos_connector/src/selector/pubsub/relay/mod.rs new file mode 100644 index 000000000..40770972e --- /dev/null +++ b/server/swimos_connector/src/selector/pubsub/relay/mod.rs @@ -0,0 +1,206 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[cfg(test)] +mod tests; + +use swimos_model::Value; + +use crate::selector::relay::{ + LaneSelector, NodeSelector, PayloadSegment, RelayPayloadSelector, Segment, +}; +use crate::selector::{parse_selector, BadSelector, PubSubSelector}; +use regex::Regex; +use std::sync::OnceLock; +use swimos_recon::parser::{parse_recognize, ParseError}; + +static STATIC_REGEX: OnceLock = OnceLock::new(); +static STATIC_PATH_REGEX: OnceLock = OnceLock::new(); + +fn static_regex() -> &'static Regex { + STATIC_REGEX.get_or_init(|| create_static_regex().expect("Invalid regex.")) +} + +fn create_static_regex() -> Result { + Regex::new(r"^[a-zA-Z0-9_]+$") +} + +fn static_path_regex() -> &'static Regex { + STATIC_PATH_REGEX.get_or_init(|| create_static_path_regex().expect("Invalid regex.")) +} + +fn create_static_path_regex() -> Result { + Regex::new(r"^\/?[a-zA-Z0-9_]+(\/[a-zA-Z0-9_]+)*(\/|$)") +} + +fn parse_segment(pattern: &str) -> Result, BadSelector> { + let mut iter = pattern.chars(); + match iter.next() { + Some('$') => Ok(Segment::Selector(parse_selector(pattern)?.into())), + Some(_) => { + if static_regex().is_match(pattern) { + Ok(Segment::Static(pattern.to_string())) + } else { + Err(BadSelector::InvalidPath) + } + } + _ => Err(BadSelector::InvalidPath), + } +} + +/// Parses a publish-subscribe [`LaneSelector`]. +/// +/// Publish-subscribe node URI selectors define three selector keywords which may be used at any +/// part of the pattern. +/// * `$topic` - selects the segment from the connector's topic. E.g, "/agents/$topic". +/// * `$key` - selects the segment from the message's key. E.g, "/agents/$key". +/// * `$payload` - selects the segment from the message's value. E.g, "/agents/$payload". +/// +/// `$key` and `$payload` selectors yield Recon [`Value`]'s and allow for selecting attributes and +/// items from the key or value of the message. +/// +/// # Static Example +/// "lights". +/// +/// # Dynamic Example +/// "$payload.id". +pub fn parse_lane_selector(pattern: &str) -> Result, BadSelector> { + Ok(LaneSelector::new( + parse_segment(pattern)?, + pattern.to_string(), + )) +} + +/// Parses a publish-subscribe [`NodeSelector`]. +/// +/// Publish-subscribe node URI selectors define three selector keywords which may be used at any +/// part of the pattern. +/// * `$topic` - selects the segment from the connector's topic. E.g, "/agents/$topic". +/// * `$key` - selects the segment from the message's key. E.g, "/agents/$key". +/// * `$payload` - selects the segment from the message's value. E.g, "/agents/$payload". +/// +/// `$key` and `$payload` selectors yield Recon [`Value`]'s and allow for selecting attributes and +/// items from the key or value of the message. +/// +/// # Static Example +/// "/agents/lights". +/// +/// # Dynamic Example +/// "/$topic/$key/$payload.id". +pub fn parse_node_selector(mut pattern: &str) -> Result, BadSelector> { + let input = pattern.to_string(); + if !pattern.starts_with('/') || pattern.len() < 2 { + return Err(BadSelector::InvalidPath); + } + + let mut segments: Vec> = Vec::new(); + + loop { + let mut iter = pattern.chars(); + match iter.next() { + Some('$') => match pattern.split_once('/') { + Some((head, tail)) => { + segments.push(Segment::Selector(parse_selector(head)?.into())); + pattern = tail; + } + None => { + segments.push(Segment::Selector(parse_selector(pattern)?.into())); + break; + } + }, + Some(_) => match static_path_regex().find(pattern) { + Some(matched) => { + segments.push(Segment::Static(matched.as_str().to_string())); + pattern = &pattern[matched.end()..]; + if pattern.is_empty() { + break; + } else if pattern.starts_with('/') { + // Pattern will capture up to a trailing slash but not a double one, so + // guard against the next static segment starting with a slash. + return Err(BadSelector::InvalidPath); + } + } + None => return Err(BadSelector::InvalidPath), + }, + _ => return Err(BadSelector::InvalidPath), + } + } + + Ok(NodeSelector::new(input, segments)) +} + +impl TryFrom> for PayloadSegment { + type Error = ParseError; + + fn try_from(value: Segment) -> Result { + match value { + Segment::Static(path) => Ok(PayloadSegment::Value(parse_recognize::( + path.as_str(), + false, + )?)), + Segment::Selector(s) => Ok(PayloadSegment::Selector(s)), + } + } +} + +/// Parses a Value Lane selector. +/// +/// # Arguments +/// * `pattern` - the selector pattern to parse. This may be defined as a Recon [`Value`] which may +/// be used as the key or value for the messaage to send to the lane. +/// * `required` - whether the selector must succeed. If this is true and the selector fails, then +/// the connector will terminate. +/// +/// Both key and value selectors may define a Recon [`Value`] which may be used as the key or value +/// for the message to send to the lane. +pub fn parse_value_selector( + pattern: &str, + required: bool, +) -> Result, BadSelector> { + Ok(RelayPayloadSelector::value( + parse_segment(pattern)?.try_into()?, + pattern.to_string(), + required, + )) +} + +/// Parses a Value Lane selector. +/// +/// # Arguments +/// * `key_pattern` - the key selector pattern to parse. +/// * `value_pattern` - the key selector pattern to parse. +/// * `required` - whether the selector must succeed. If this is true and the selector fails, then +/// the connector will terminate. +/// * `remove_when_no_value` - if the value selector fails to select, then it will emit a map +/// remove command to remove the corresponding entry. +/// +/// Both key and value selectors may define a Recon [`Value`] which may be used as the key or value +/// for the message to send to the lane. +pub fn parse_map_selector( + key_pattern: &str, + value_pattern: &str, + required: bool, + remove_when_no_value: bool, +) -> Result, BadSelector> { + let key = parse_segment(key_pattern)?.try_into()?; + let value = parse_segment(value_pattern)?.try_into()?; + Ok(RelayPayloadSelector::map( + key, + value, + key_pattern.to_string(), + value_pattern.to_string(), + required, + remove_when_no_value, + )) +} diff --git a/server/swimos_connector/src/selector/pubsub/relay/tests.rs b/server/swimos_connector/src/selector/pubsub/relay/tests.rs new file mode 100644 index 000000000..385fcdb97 --- /dev/null +++ b/server/swimos_connector/src/selector/pubsub/relay/tests.rs @@ -0,0 +1,216 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::selector::pubsub::relay::{ + parse_lane_selector, parse_node_selector, parse_value_selector, LaneSelector, NodeSelector, + PayloadSegment, RelayPayloadSelector, Segment, +}; +use crate::selector::{KeySelector, PayloadSelector, PubSubSelector, TopicSelector}; +use swimos_model::Value; + +#[test] +fn parse_lane() { + fn ok(pattern: &str, expected: LaneSelector) { + match parse_lane_selector(pattern) { + Ok(actual) => { + assert_eq!(expected, actual); + } + Err(e) => { + panic!("Failed to parse lane selector {}: {:?}", pattern, e); + } + } + } + + fn err(pattern: &str) { + if let Ok(actual) = parse_lane_selector(pattern) { + panic!( + "Expected parse error from pattern {}, but got {:?}", + pattern, actual + ); + } + } + + ok( + "lane", + LaneSelector::new(Segment::Static("lane".to_string()), "lane".to_string()), + ); + ok( + "$key", + LaneSelector::new( + Segment::Selector(PubSubSelector::inject(KeySelector::default())), + "$key".to_string(), + ), + ); + ok( + "$payload", + LaneSelector::new( + Segment::Selector(PubSubSelector::inject(PayloadSelector::default())), + "$payload".to_string(), + ), + ); + ok( + "$topic", + LaneSelector::new( + Segment::Selector(PubSubSelector::inject(TopicSelector)), + "$topic".to_string(), + ), + ); + err("$donut"); + err("a string with words"); + err(" "); + err("!!!"); + err("$"); + err("/$"); + err("/::"); + err("/"); + err("//"); + err("//aaa"); + err("::/"); + err(":/"); + err(""); + err(":"); + err("$"); + err("~"); + err("$key.$value"); + err("$key/::$value"); + err("blah/blah"); + err("$key/$value"); + err("$key.field/blah"); + err("blah/$key"); +} + +#[test] +fn parse_node() { + fn ok(pattern: &str, expected: NodeSelector) { + match parse_node_selector(pattern) { + Ok(actual) => { + assert_eq!(expected, actual); + } + Err(e) => { + panic!("Failed to parse node selector {}: {:?}", pattern, e); + } + } + } + + fn err(pattern: &str) { + if let Ok(actual) = parse_node_selector(pattern) { + panic!( + "Expected parse error from pattern {}, but got {:?}", + pattern, actual + ); + } + } + ok( + "/lane", + NodeSelector::new( + "/lane".to_string(), + vec![Segment::Static("/lane".to_string())], + ), + ); + ok( + "/lane/sub/path", + NodeSelector::new( + "/lane/sub/path".to_string(), + vec![Segment::Static("/lane/sub/path".to_string())], + ), + ); + ok( + "/node/$key", + NodeSelector::new( + "/node/$key".to_string(), + vec![ + Segment::Static("/node/".to_string()), + Segment::Selector(PubSubSelector::inject(KeySelector::default())), + ], + ), + ); + ok( + "/node/$payload", + NodeSelector::new( + "/node/$payload".to_string(), + vec![ + Segment::Static("/node/".to_string()), + Segment::Selector(PubSubSelector::inject(PayloadSelector::default())), + ], + ), + ); + ok( + "/node/$topic", + NodeSelector::new( + "/node/$topic".to_string(), + vec![ + Segment::Static("/node/".to_string()), + Segment::Selector(PubSubSelector::inject(TopicSelector)), + ], + ), + ); + err("/"); + err("//"); + err("///"); + err("/a//b"); + err("/a/b//"); + err("/a/b//c/"); + err("/a/b/c//"); + err("/a/b/c/d//"); + err("/$blah"); + err("/$key/$blah/blah/"); + err("/$key/$blah/blah"); +} + +#[test] +fn parse_payload() { + fn value_ok(pattern: &str, expected: RelayPayloadSelector) { + match parse_value_selector(pattern, true) { + Ok(actual) => { + assert_eq!(expected, actual); + } + Err(e) => { + panic!("Failed to parse payload selector {}: {:?}", pattern, e); + } + } + } + + value_ok( + "blah", + RelayPayloadSelector::value( + PayloadSegment::Value(Value::from("blah")), + "blah".to_string(), + true, + ), + ); + value_ok( + "$key", + RelayPayloadSelector::value( + PayloadSegment::Selector(PubSubSelector::inject(KeySelector::default())), + "$key".to_string(), + true, + ), + ); + value_ok( + "$payload", + RelayPayloadSelector::value( + PayloadSegment::Selector(PubSubSelector::inject(PayloadSelector::default())), + "$payload".to_string(), + true, + ), + ); + value_ok( + "$topic", + RelayPayloadSelector::value( + PayloadSegment::Selector(PubSubSelector::inject(TopicSelector)), + "$topic".to_string(), + true, + ), + ); +} diff --git a/server/swimos_connector_kafka/src/selector/tests.rs b/server/swimos_connector/src/selector/pubsub/tests.rs similarity index 61% rename from server/swimos_connector_kafka/src/selector/tests.rs rename to server/swimos_connector/src/selector/pubsub/tests.rs index 16d2c8924..6f395daca 100644 --- a/server/swimos_connector_kafka/src/selector/tests.rs +++ b/server/swimos_connector/src/selector/pubsub/tests.rs @@ -12,28 +12,38 @@ // See the License for the specific language governing permissions and // limitations under the License. -use swimos_model::{Attr, Item, Value}; - -use crate::deser::MessagePart; -use crate::error::DeserializationError; +use super::{create_init_regex, field_regex, init_regex, SelectorDescriptor}; +use super::{MessageField, SelectorComponent}; +use crate::config::{IngressMapLaneSpec, IngressValueLaneSpec}; +use crate::deser::{Deferred, MessageDeserializer, MessagePart, ReconDeserializer}; use crate::selector::{ - BadSelector, InvalidLaneSpec, MessageField, SelectorComponent, SelectorDescriptor, + parse_selector, AttrSelector, BadSelector, BasicSelector, ChainSelector, IdentitySelector, + IndexSelector, InvalidLaneSpec, MapLaneSelector, SelectHandler, SlotSelector, + ValueLaneSelector, ValueSelector, }; -use crate::{IngressMapLaneSpec, IngressValueLaneSpec}; +use swimos_model::{Attr, Item}; -use super::{ - AttrSelector, BasicSelector, ChainSelector, Deferred, IdentitySelector, IndexSelector, - LaneSelector, MapLaneSelector, Selector, SlotSelector, ValueLaneSelector, +use crate::selector::{PubSubSelector, PubSubValueLaneSelector}; +use crate::{ + selector::pubsub::{KeySelector, PayloadSelector, TopicSelector}, + test_support::{fail, run_handler, TestSpawner}, + ConnectorAgent, }; +use bytes::BytesMut; +use frunk::hlist; +use std::ops::Deref; +use swimos_agent::agent_model::{AgentSpec, ItemDescriptor}; +use swimos_api::agent::WarpLaneKind; +use swimos_model::Value; #[test] fn init_regex_creation() { - super::create_init_regex().expect("Creation failed."); + create_init_regex().expect("Creation failed."); } #[test] fn match_key() { - if let Some(captures) = super::init_regex().captures("$key") { + if let Some(captures) = init_regex().captures("$key") { let kind = captures.get(1).expect("Missing capture."); assert!(captures.get(2).is_none()); assert_eq!(kind.as_str(), "$key"); @@ -44,7 +54,7 @@ fn match_key() { #[test] fn match_payload() { - if let Some(captures) = super::init_regex().captures("$payload") { + if let Some(captures) = init_regex().captures("$payload") { let kind = captures.get(1).expect("Missing capture."); assert!(captures.get(2).is_none()); assert_eq!(kind.as_str(), "$payload"); @@ -55,7 +65,7 @@ fn match_payload() { #[test] fn match_topic() { - if let Some(captures) = super::init_regex().captures("$topic") { + if let Some(captures) = init_regex().captures("$topic") { let kind = captures.get(1).expect("Missing capture."); assert!(captures.get(2).is_none()); assert_eq!(kind.as_str(), "$topic"); @@ -66,7 +76,7 @@ fn match_topic() { #[test] fn match_key_indexed() { - if let Some(captures) = super::init_regex().captures("$key[3]") { + if let Some(captures) = init_regex().captures("$key[3]") { let kind = captures.get(1).expect("Missing capture."); let index = captures.get(2).expect("Missing capture."); assert_eq!(kind.as_str(), "$key"); @@ -78,7 +88,7 @@ fn match_key_indexed() { #[test] fn match_payload_indexed() { - if let Some(captures) = super::init_regex().captures("$payload[0]") { + if let Some(captures) = init_regex().captures("$payload[0]") { let kind = captures.get(1).expect("Missing capture."); let index = captures.get(2).expect("Missing capture."); assert_eq!(kind.as_str(), "$payload"); @@ -90,7 +100,7 @@ fn match_payload_indexed() { #[test] fn match_attr() { - if let Some(captures) = super::field_regex().captures("@my_attr") { + if let Some(captures) = field_regex().captures("@my_attr") { let name = captures.get(1).expect("Missing capture."); assert!(captures.get(2).is_none()); assert_eq!(name.as_str(), "@my_attr"); @@ -101,7 +111,7 @@ fn match_attr() { #[test] fn match_attr_indexed() { - if let Some(captures) = super::field_regex().captures("@attr[73]") { + if let Some(captures) = field_regex().captures("@attr[73]") { let name = captures.get(1).expect("Missing capture."); let index = captures.get(2).expect("Missing capture."); assert_eq!(name.as_str(), "@attr"); @@ -113,7 +123,7 @@ fn match_attr_indexed() { #[test] fn match_slot() { - if let Some(captures) = super::field_regex().captures("slot") { + if let Some(captures) = field_regex().captures("slot") { let name = captures.get(1).expect("Missing capture."); assert!(captures.get(2).is_none()); assert_eq!(name.as_str(), "slot"); @@ -124,7 +134,7 @@ fn match_slot() { #[test] fn match_slot_indexed() { - if let Some(captures) = super::field_regex().captures("slot5[123]") { + if let Some(captures) = field_regex().captures("slot5[123]") { let name = captures.get(1).expect("Missing capture."); let index = captures.get(2).expect("Missing capture."); assert_eq!(name.as_str(), "slot5"); @@ -136,7 +146,7 @@ fn match_slot_indexed() { #[test] fn match_slot_non_latin() { - if let Some(captures) = super::field_regex().captures("اسم[123]") { + if let Some(captures) = field_regex().captures("اسم[123]") { let name = captures.get(1).expect("Missing capture."); let index = captures.get(2).expect("Missing capture."); assert_eq!(name.as_str(), "اسم"); @@ -164,16 +174,16 @@ impl<'a> SelectorDescriptor<'a> { #[test] fn parse_simple() { - let key = super::parse_selector("$key").expect("Parse failed."); + let key = parse_selector("$key").expect("Parse failed."); assert_eq!(key, SelectorDescriptor::for_part(MessagePart::Key, None)); - let payload = super::parse_selector("$payload").expect("Parse failed."); + let payload = parse_selector("$payload").expect("Parse failed."); assert_eq!( payload, SelectorDescriptor::for_part(MessagePart::Payload, None) ); - let indexed = super::parse_selector("$key[2]").expect("Parse failed."); + let indexed = parse_selector("$key[2]").expect("Parse failed."); assert_eq!( indexed, SelectorDescriptor::for_part(MessagePart::Key, Some(2)) @@ -182,37 +192,37 @@ fn parse_simple() { #[test] fn parse_topic() { - let topic = super::parse_selector("$topic").expect("Parse failed."); + let topic = parse_selector("$topic").expect("Parse failed."); assert_eq!(topic, SelectorDescriptor::Topic); assert_eq!( - super::parse_selector("$topic[0]"), + parse_selector("$topic[0]"), Err(BadSelector::TopicWithComponent) ); assert_eq!( - super::parse_selector("$topic.slot"), + parse_selector("$topic.slot"), Err(BadSelector::TopicWithComponent) ); } #[test] fn parse_one_component() { - let first = super::parse_selector("$key.@attr").expect("Parse failed."); + let first = parse_selector("$key.@attr").expect("Parse failed."); let mut expected_first = SelectorDescriptor::for_part(MessagePart::Key, None); expected_first.push(SelectorComponent::new(true, "attr", None)); assert_eq!(first, expected_first); - let second = super::parse_selector("$payload.slot").expect("Parse failed."); + let second = parse_selector("$payload.slot").expect("Parse failed."); let mut expected_second = SelectorDescriptor::for_part(MessagePart::Payload, None); expected_second.push(SelectorComponent::new(false, "slot", None)); assert_eq!(second, expected_second); - let third = super::parse_selector("$key.@attr[3]").expect("Parse failed."); + let third = parse_selector("$key.@attr[3]").expect("Parse failed."); let mut expected_third = SelectorDescriptor::for_part(MessagePart::Key, None); expected_third.push(SelectorComponent::new(true, "attr", Some(3))); assert_eq!(third, expected_third); - let fourth = super::parse_selector("$payload[6].slot[8]").expect("Parse failed."); + let fourth = parse_selector("$payload[6].slot[8]").expect("Parse failed."); let mut expected_fourth = SelectorDescriptor::for_part(MessagePart::Payload, Some(6)); expected_fourth.push(SelectorComponent::new(false, "slot", Some(8))); assert_eq!(fourth, expected_fourth); @@ -220,7 +230,7 @@ fn parse_one_component() { #[test] fn multi_component_selector() { - let selector = super::parse_selector("$payload.red.@green[7].blue").expect("Parse failed."); + let selector = parse_selector("$payload.red.@green[7].blue").expect("Parse failed."); let mut expected = SelectorDescriptor::for_part(MessagePart::Payload, None); expected.push(SelectorComponent::new(false, "red", None)); expected.push(SelectorComponent::new(true, "green", Some(7))); @@ -252,7 +262,7 @@ fn test_value() -> Value { fn identity_selector() { let selector = IdentitySelector; let value = test_value(); - let selected = selector.select(&value); + let selected = selector.select_value(&value); assert_eq!(selected, Some(&value)); } @@ -264,9 +274,9 @@ fn attr_selector() { let selector2 = AttrSelector::new("attr2".to_string()); let selector3 = AttrSelector::new("other".to_string()); - assert_eq!(selector1.select(&value), Some(&Value::Extant)); - assert_eq!(selector2.select(&value), Some(&Value::from(5))); - assert!(selector3.select(&value).is_none()); + assert_eq!(selector1.select_value(&value), Some(&Value::Extant)); + assert_eq!(selector2.select_value(&value), Some(&Value::from(5))); + assert!(selector3.select_value(&value).is_none()); } #[test] @@ -277,9 +287,9 @@ fn slot_selector() { let selector2 = SlotSelector::for_field("a"); let selector3 = SlotSelector::for_field("other"); - assert_eq!(selector1.select(&value), Some(&Value::from(true))); - assert!(selector2.select(&value).is_none()); - assert!(selector3.select(&value).is_none()); + assert_eq!(selector1.select_value(&value), Some(&Value::from(true))); + assert!(selector2.select_value(&value).is_none()); + assert!(selector3.select_value(&value).is_none()); } #[test] @@ -290,119 +300,35 @@ fn index_selector() { let selector2 = IndexSelector::new(2); let selector3 = IndexSelector::new(4); - assert_eq!(selector1.select(&value), Some(&Value::from(3))); - assert_eq!(selector2.select(&value), Some(&Value::from(true))); - assert!(selector3.select(&value).is_none()); + assert_eq!(selector1.select_value(&value), Some(&Value::from(3))); + assert_eq!(selector2.select_value(&value), Some(&Value::from(true))); + assert!(selector3.select_value(&value).is_none()); } #[test] fn chain_selector() { let value = test_value(); - let selector1 = ChainSelector::new(vec![]); - let selector2 = ChainSelector::new(vec![BasicSelector::Slot(SlotSelector::for_field("name"))]); - let selector3 = ChainSelector::new(vec![ + let selector1 = ChainSelector::from(vec![]); + let selector2 = ChainSelector::from(vec![BasicSelector::Slot(SlotSelector::for_field("name"))]); + let selector3 = ChainSelector::from(vec![ BasicSelector::Slot(SlotSelector::for_field("inner")), BasicSelector::Slot(SlotSelector::for_field("green")), ]); - let selector4 = ChainSelector::new(vec![ + let selector4 = ChainSelector::from(vec![ BasicSelector::Slot(SlotSelector::for_field("name")), BasicSelector::Slot(SlotSelector::for_field("green")), ]); - assert_eq!(selector1.select(&value), Some(&value)); - assert_eq!(selector2.select(&value), Some(&Value::from(true))); - assert_eq!(selector3.select(&value), Some(&Value::from(5))); - assert!(selector4.select(&value).is_none()); -} - -struct TestDeferred { - value: Value, - called: bool, -} - -impl TestDeferred { - fn new(value: Value) -> Self { - TestDeferred { - value, - called: false, - } - } - - fn was_called(&self) -> bool { - self.called - } -} - -impl Deferred for TestDeferred { - fn get(&mut self) -> Result<&Value, DeserializationError> { - let TestDeferred { value, called } = self; - *called = true; - Ok(value) - } -} - -#[test] -fn select_topic() { - let selector = LaneSelector::Topic; - - let topic = Value::text("topic_name"); - let mut key = TestDeferred::new(Value::Extant); - let mut payload = TestDeferred::new(Value::Extant); - - let selected = selector - .select(&topic, &mut key, &mut payload) - .expect("Failed."); - assert_eq!(selected, Some(&topic)); - assert!(!key.was_called()); - assert!(!payload.was_called()); -} - -#[test] -fn select_key() { - let selector = ChainSelector::new(vec![ - BasicSelector::Slot(SlotSelector::for_field("inner")), - BasicSelector::Slot(SlotSelector::for_field("green")), - ]); - - let lane_selector = LaneSelector::Key(selector); - - let topic = Value::text("topic_name"); - let mut key = TestDeferred::new(test_value()); - let mut payload = TestDeferred::new(Value::Extant); - - let selected = lane_selector - .select(&topic, &mut key, &mut payload) - .expect("Failed."); - assert_eq!(selected, Some(&Value::from(5))); - assert!(key.was_called()); - assert!(!payload.was_called()); -} - -#[test] -fn select_payload() { - let selector = ChainSelector::new(vec![ - BasicSelector::Slot(SlotSelector::for_field("inner")), - BasicSelector::Slot(SlotSelector::for_field("green")), - ]); - - let lane_selector = LaneSelector::Payload(selector); - - let topic = Value::text("topic_name"); - let mut key = TestDeferred::new(Value::Extant); - let mut payload = TestDeferred::new(test_value()); - - let selected = lane_selector - .select(&topic, &mut key, &mut payload) - .expect("Failed."); - assert_eq!(selected, Some(&Value::from(5))); - assert!(!key.was_called()); - assert!(payload.was_called()); + assert_eq!(selector1.select_value(&value), Some(&value)); + assert_eq!(selector2.select_value(&value), Some(&Value::from(true))); + assert_eq!(selector3.select_value(&value), Some(&Value::from(5))); + assert!(selector4.select_value(&value).is_none()); } #[test] fn topic_selector_descriptor() { - let selector = super::parse_selector("$topic").expect("Invalid selector."); + let selector = parse_selector("$topic").expect("Invalid selector."); assert_eq!(selector.field(), MessageField::Topic); assert!(selector.selector().is_none()); assert_eq!(selector.suggested_name(), Some("topic")); @@ -410,27 +336,27 @@ fn topic_selector_descriptor() { #[test] fn key_selector_descriptor() { - let selector = super::parse_selector("$key").expect("Invalid selector."); + let selector = parse_selector("$key").expect("Invalid selector."); assert_eq!(selector.field(), MessageField::Key); - assert_eq!(selector.selector(), Some(ChainSelector::new(vec![]))); + assert_eq!(selector.selector(), Some(ChainSelector::from(vec![]))); assert_eq!(selector.suggested_name(), Some("key")); } #[test] fn payload_selector_descriptor() { - let selector = super::parse_selector("$payload").expect("Invalid selector."); + let selector = parse_selector("$payload").expect("Invalid selector."); assert_eq!(selector.field(), MessageField::Payload); - assert_eq!(selector.selector(), Some(ChainSelector::new(vec![]))); + assert_eq!(selector.selector(), Some(ChainSelector::from(vec![]))); assert_eq!(selector.suggested_name(), Some("payload")); } #[test] fn indexed_selector_descriptor() { - let selector = super::parse_selector("$payload[1]").expect("Invalid selector."); + let selector = parse_selector("$payload[1]").expect("Invalid selector."); assert_eq!(selector.field(), MessageField::Payload); assert_eq!( selector.selector(), - Some(ChainSelector::new(vec![BasicSelector::Index( + Some(ChainSelector::from(vec![BasicSelector::Index( IndexSelector::new(1) )])) ); @@ -439,11 +365,11 @@ fn indexed_selector_descriptor() { #[test] fn attr_selector_descriptor() { - let selector = super::parse_selector("$payload.@attr").expect("Invalid selector."); + let selector = parse_selector("$payload.@attr").expect("Invalid selector."); assert_eq!(selector.field(), MessageField::Payload); assert_eq!( selector.selector(), - Some(ChainSelector::new(vec![BasicSelector::Attr( + Some(ChainSelector::from(vec![BasicSelector::Attr( AttrSelector::new("attr".to_string()) )])) ); @@ -452,11 +378,11 @@ fn attr_selector_descriptor() { #[test] fn slot_selector_descriptor() { - let selector = super::parse_selector("$payload.slot").expect("Invalid selector."); + let selector = parse_selector("$payload.slot").expect("Invalid selector."); assert_eq!(selector.field(), MessageField::Payload); assert_eq!( selector.selector(), - Some(ChainSelector::new(vec![BasicSelector::Slot( + Some(ChainSelector::from(vec![BasicSelector::Slot( SlotSelector::for_field("slot") )])) ); @@ -465,11 +391,11 @@ fn slot_selector_descriptor() { #[test] fn complex_selector_descriptor_named() { - let selector = super::parse_selector("$payload.@attr[3].inner").expect("Invalid selector."); + let selector = parse_selector("$payload.@attr[3].inner").expect("Invalid selector."); assert_eq!(selector.field(), MessageField::Payload); assert_eq!( selector.selector(), - Some(ChainSelector::new(vec![ + Some(ChainSelector::from(vec![ BasicSelector::Attr(AttrSelector::new("attr".to_string())), BasicSelector::Index(IndexSelector::new(3)), BasicSelector::Slot(SlotSelector::for_field("inner")) @@ -480,11 +406,11 @@ fn complex_selector_descriptor_named() { #[test] fn complex_selector_descriptor_unnamed() { - let selector = super::parse_selector("$payload.@attr[3].inner[0]").expect("Invalid selector."); + let selector = parse_selector("$payload.@attr[3].inner[0]").expect("Invalid selector."); assert_eq!(selector.field(), MessageField::Payload); assert_eq!( selector.selector(), - Some(ChainSelector::new(vec![ + Some(ChainSelector::from(vec![ BasicSelector::Attr(AttrSelector::new("attr".to_string())), BasicSelector::Index(IndexSelector::new(3)), BasicSelector::Slot(SlotSelector::for_field("inner")), @@ -498,11 +424,16 @@ fn complex_selector_descriptor_unnamed() { fn value_lane_selector_from_spec_inferred_name() { let spec = IngressValueLaneSpec::new(None, "$key", true); let selector = ValueLaneSelector::try_from(&spec).expect("Bad specification."); - assert_eq!(&selector.name, "key"); - assert!(&selector.required); + assert_eq!(selector.name(), "key"); + assert!(&selector.is_required()); + let delegate_selector = selector + .into_selector() + .uninject::() + .expect("Failed to get inner selector"); + assert_eq!( - selector.selector, - LaneSelector::Key(ChainSelector::default()) + delegate_selector, + KeySelector::new(ChainSelector::default()) ); } @@ -510,11 +441,15 @@ fn value_lane_selector_from_spec_inferred_name() { fn value_lane_selector_from_spec_named() { let spec = IngressValueLaneSpec::new(Some("field"), "$key[0]", false); let selector = ValueLaneSelector::try_from(&spec).expect("Bad specification."); - assert_eq!(&selector.name, "field"); - assert!(!&selector.required); + assert_eq!(selector.name(), "field"); + assert!(!&selector.is_required()); + let delegate_selector = selector + .into_selector() + .uninject::() + .expect("Failed to get inner selector"); assert_eq!( - selector.selector, - LaneSelector::Key(ChainSelector::new(vec![BasicSelector::Index( + delegate_selector, + KeySelector(ChainSelector::from(vec![BasicSelector::Index( IndexSelector::new(0) )])) ); @@ -538,16 +473,22 @@ fn value_lane_selector_from_spec_bad_selector() { fn map_lane_selector_from_spec() { let spec = IngressMapLaneSpec::new("field", "$key", "$payload", true, false); let selector = MapLaneSelector::try_from(&spec).expect("Bad specification."); - assert_eq!(&selector.name, "field"); - assert!(!selector.required); - assert!(selector.remove_when_no_value); - assert_eq!( - selector.key_selector, - LaneSelector::Key(ChainSelector::default()) - ); + assert_eq!(selector.name(), "field"); + assert!(!selector.is_required()); + assert!(selector.remove_when_no_value()); + + let (key_selector, value_selector) = selector.into_selectors(); + let key_delegate = key_selector + .uninject::() + .expect("Failed to get key delegate."); + let value_delegate = value_selector + .uninject::() + .expect("Failed to get key delegate."); + + assert_eq!(key_delegate, KeySelector::new(ChainSelector::default())); assert_eq!( - selector.value_selector, - LaneSelector::Payload(ChainSelector::default()) + value_delegate, + PayloadSelector::new(ChainSelector::default()) ); } @@ -564,3 +505,63 @@ fn map_lane_selector_from_spec_bad_value() { let error = MapLaneSelector::try_from(&spec).expect_err("Should fail."); assert_eq!(error, InvalidLaneSpec::Selector(BadSelector::InvalidRoot)); } + +fn run_selector(selector: PubSubSelector, expected: Value) { + const LANE: &str = "lane"; + + let selector = PubSubValueLaneSelector::new(LANE.to_string(), selector, true); + let topic = Value::from("topic"); + + let deserializer = ReconDeserializer.boxed(); + let key = Deferred::new(b"13".as_slice(), &deserializer); + let value = Deferred::new(b"64.0".as_slice(), &deserializer); + let mut args = hlist![topic, key, value]; + let handler = selector.select_handler(&mut args).unwrap(); + let mut agent = ConnectorAgent::default(); + + agent + .register_dynamic_item( + LANE, + ItemDescriptor::WarpLane { + kind: WarpLaneKind::Value, + flags: Default::default(), + }, + ) + .expect("Failed to register lane"); + + run_handler( + &TestSpawner::default(), + &mut BytesMut::default(), + &agent, + handler, + fail, + ); + + match agent.value_lane(LANE) { + Some(lane) => lane.deref().read(|state| assert_eq!(state, &expected)), + None => { + panic!("Missing lane") + } + }; +} + +#[test] +fn selects_topic() { + run_selector(PubSubSelector::inject(TopicSelector), Value::from("topic")); +} + +#[test] +fn selects_key() { + run_selector( + PubSubSelector::inject(KeySelector::new(ChainSelector::default())), + Value::from(13), + ); +} + +#[test] +fn selects_payload() { + run_selector( + PubSubSelector::inject(PayloadSelector::new(ChainSelector::default())), + Value::from(64f64), + ); +} diff --git a/server/swimos_connector/src/selector/relay.rs b/server/swimos_connector/src/selector/relay.rs new file mode 100644 index 000000000..66dc3739b --- /dev/null +++ b/server/swimos_connector/src/selector/relay.rs @@ -0,0 +1,418 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::selector::{Selector, SelectorError}; +use frunk::Coprod; +use swimos_model::Value; + +use std::slice::Iter; +use swimos_agent::event_handler::{Discard, SendCommand}; +use swimos_agent_protocol::MapMessage; +use swimos_api::address::Address; + +type SendCommandOp = + Coprod!(SendCommand, SendCommand>); +pub type GenericSendCommandOp = Discard>; + +/// A segment selector in a URI. A segment is the part between two consecutive '/' in a URI. This +/// may delegate to a selector to derive the segment from a key, payload or topic, or yield a static +/// segment. +#[derive(Debug, Clone, PartialEq)] +pub enum Segment { + /// Yield a static segment for the URI. + Static(String), + /// Build the URI segment from a selector. + Selector(S), +} + +/// A lane URI selector. +#[derive(Debug, Clone, PartialEq)] +pub struct LaneSelector { + segment: Segment, + pattern: String, +} + +impl LaneSelector { + /// Builds a new [`LaneSelector`]. + /// + /// # Arguments + /// * `segment` - the selector for the lane. + /// * `pattern` - the pattern which the selector represents. Used to build an error message if + /// the selector fails. + pub fn new(segment: Segment, pattern: String) -> LaneSelector { + LaneSelector { segment, pattern } + } + + fn select(&self, args: &mut A) -> Result + where + S: Selector, + { + let LaneSelector { pattern, segment } = self; + + match segment { + Segment::Static(p) => Ok(p.to_string()), + Segment::Selector(selector) => { + let value = selector + .select(args)? + .ok_or(SelectorError::Selector(pattern.to_string()))?; + match value { + Value::BooleanValue(v) => { + if v { + Ok("true".to_string()) + } else { + Ok("false".to_string()) + } + } + Value::Int32Value(v) => Ok(v.to_string()), + Value::Int64Value(v) => Ok(v.to_string()), + Value::UInt32Value(v) => Ok(v.to_string()), + Value::UInt64Value(v) => Ok(v.to_string()), + Value::BigUint(v) => Ok(v.to_string()), + Value::Text(v) => Ok(v.to_string()), + _ => Err(SelectorError::InvalidRecord(self.pattern.to_string())), + } + } + } + } +} + +/// A URI selector for an agent node. [`NodeSelector`] takes a Vec of segments, where each segment +/// represents the portion of the node's URI between consecutive slashes. Each segment can either be +/// a fixed string or a value, determined by the selector type used by this NodeSelector. +/// +/// This type is generally built using the connector's message type implementation. For publish-subscribe +/// type connectors see [`crate::selector::pubsub::parse_node_selector`]. +#[derive(Debug, Clone, PartialEq)] +pub struct NodeSelector { + pattern: String, + segments: Vec>, +} + +impl<'a, S> IntoIterator for &'a NodeSelector { + type Item = &'a Segment; + type IntoIter = Iter<'a, Segment>; + + fn into_iter(self) -> Self::IntoIter { + self.segments.iter() + } +} + +impl NodeSelector { + /// Builds a new [`NodeSelector`]. + /// + /// # Arguments + /// * `segment` - the selector for the node. + /// * `pattern` - the pattern which the selector represents. Used to build an error message if + /// the selector fails. + pub fn new(pattern: String, segments: Vec>) -> NodeSelector { + NodeSelector { pattern, segments } + } + + fn select(&self, args: &mut A) -> Result + where + S: Selector, + { + let mut node_uri = String::new(); + + for elem in self.into_iter() { + match elem { + Segment::Static(p) => node_uri.push_str(p), + Segment::Selector(selector) => { + let value = selector + .select(args)? + .ok_or(SelectorError::Selector(self.pattern.to_string()))?; + match value { + Value::BooleanValue(v) => { + node_uri.push_str(if v { "true" } else { "false" }) + } + Value::Int32Value(v) => node_uri.push_str(&v.to_string()), + Value::Int64Value(v) => node_uri.push_str(&v.to_string()), + Value::UInt32Value(v) => node_uri.push_str(&v.to_string()), + Value::UInt64Value(v) => node_uri.push_str(&v.to_string()), + Value::BigInt(v) => node_uri.push_str(&v.to_string()), + Value::BigUint(v) => node_uri.push_str(&v.to_string()), + Value::Text(v) => node_uri.push_str(v.as_str()), + _ => return Err(SelectorError::InvalidRecord(self.pattern.to_string())), + } + } + } + } + + Ok(node_uri) + } +} + +/// A command message's payload selector. +#[derive(Debug, Clone, PartialEq)] +pub struct RelayPayloadSelector { + /// Abstraction over a value or map command selector. + inner: Inner, + /// Whether the selector must yield a value. If a selector fails to yield a value then an error + /// will be returned. + required: bool, +} + +/// A payload selector model. When called, will either yield a [`Value`] or delegate to `S`. +#[derive(Debug, Clone, PartialEq)] +pub enum PayloadSegment { + /// Yield a value. + Value(Value), + /// Delegate to a [`Selector`]. + Selector(S), +} + +#[derive(Debug, Clone, PartialEq)] +enum Inner { + Value { + pattern: String, + segment: PayloadSegment, + }, + Map { + key_pattern: String, + value_pattern: String, + remove_when_no_value: bool, + key: PayloadSegment, + value: PayloadSegment, + }, +} + +impl RelayPayloadSelector { + /// Builds a new Value [`RelayPayloadSelector`]. + /// + /// # Arguments + /// * `segment` - the selector for the payload. + /// * `pattern` - the pattern that this selector represents. Used to build an error + /// message if the selector fails. + /// * `required` - whether the selector must succeed. If this is true and the selector fails, then + /// the connector will terminate. + pub fn value( + segment: PayloadSegment, + pattern: String, + required: bool, + ) -> RelayPayloadSelector { + RelayPayloadSelector { + inner: Inner::Value { pattern, segment }, + required, + } + } + + /// Builds a new Map [`RelayPayloadSelector`]. + /// + /// # Arguments + /// * `key_segment` - the key selector for the payload. + /// * `value_segment` - the value selector for the payload. + /// * `key_pattern` - the key pattern that this selector represents. Used to build an error message + /// if the key selector fails. + /// * `value_segment` - the value pattern that this selector represents. Used to build an error + /// message if the value selector fails. + /// * `remove_when_no_value` - if the value selector fails to select, then it will emit a map + /// remove command to remove the corresponding entry. + /// * `required` - whether the selector must succeed. If this is true and the selector fails, then + /// the connector will terminate. + pub fn map( + key_segment: PayloadSegment, + value_segment: PayloadSegment, + key_pattern: String, + value_pattern: String, + remove_when_no_value: bool, + required: bool, + ) -> RelayPayloadSelector { + RelayPayloadSelector { + inner: Inner::Map { + key_pattern, + value_pattern, + remove_when_no_value, + key: key_segment, + value: value_segment, + }, + required, + } + } + + fn select( + &self, + node_uri: String, + lane_uri: String, + args: &mut A, + ) -> Result + where + S: Selector, + { + let RelayPayloadSelector { inner, required } = self; + let op = match inner { + Inner::Value { pattern, segment } => { + build_value(*required, pattern.as_str(), segment, args)?.map(|payload| { + SendCommandOp::inject(SendCommand::new( + Address::new(None, node_uri, lane_uri), + payload, + false, + )) + }) + } + Inner::Map { + key_pattern, + value_pattern, + remove_when_no_value, + key, + value, + } => { + let key = build_value(*required, key_pattern.as_str(), key, args)?; + let value = build_value(*required, value_pattern.as_str(), value, args)?; + + match (key, value) { + (Some(key_payload), Some(value_payload)) => { + let op = SendCommandOp::inject(SendCommand::new( + Address::new(None, node_uri, lane_uri), + MapMessage::Update { + key: key_payload, + value: value_payload, + }, + false, + )); + Some(op) + } + (Some(key_payload), None) if *remove_when_no_value => { + let op = SendCommandOp::inject(SendCommand::new( + Address::new(None, node_uri, lane_uri), + MapMessage::Remove { key: key_payload }, + false, + )); + Some(op) + } + _ => None, + } + } + }; + + Ok(Discard::>::new(op)) + } +} + +fn build_value( + required: bool, + pattern: &str, + segment: &PayloadSegment, + args: &mut A, +) -> Result, SelectorError> +where + S: Selector, +{ + let payload = match segment { + PayloadSegment::Value(value) => Ok(Some(value.clone())), + PayloadSegment::Selector(selector) => selector.select(args).map_err(SelectorError::from), + }; + + match payload { + Ok(Some(payload)) => Ok(Some(payload)), + Ok(None) => { + if required { + Err(SelectorError::Selector(pattern.to_string())) + } else { + Ok(None) + } + } + Err(e) => { + if required { + Err(e) + } else { + Ok(None) + } + } + } +} + +/// A collection of relays which are used to derive the commands to send to lanes on agents. +#[derive(Debug, Clone)] +pub struct Relays { + chain: Vec>, +} + +impl Default for Relays { + fn default() -> Self { + Relays { chain: vec![] } + } +} + +impl Relays { + pub fn new(chain: I) -> Relays + where + I: IntoIterator>, + { + Relays { + chain: chain.into_iter().collect(), + } + } + + pub fn len(&self) -> usize { + self.chain.len() + } + + pub fn is_empty(&self) -> bool { + self.chain.is_empty() + } +} + +impl From> for Relays { + fn from(relays: Relay) -> Relays { + Relays { + chain: vec![relays], + } + } +} + +impl<'s, S> IntoIterator for &'s Relays { + type Item = &'s Relay; + type IntoIter = Iter<'s, Relay>; + + fn into_iter(self) -> Self::IntoIter { + self.chain.as_slice().iter() + } +} + +/// A relay which is used to build a command to send to a lane on an agent. +#[derive(Debug, Clone)] +pub struct Relay { + node: NodeSelector, + lane: LaneSelector, + payload: RelayPayloadSelector, +} + +impl Relay { + pub fn new( + node: NodeSelector, + lane: LaneSelector, + payload: RelayPayloadSelector, + ) -> Relay { + Relay { + node, + lane, + payload, + } + } + + pub fn select_handler(&self, args: &mut A) -> Result + where + S: Selector, + { + let Relay { + node, + lane, + payload, + } = self; + + let node_uri = node.select(args)?; + let lane_uri = lane.select(args)?; + payload.select(node_uri, lane_uri, args) + } +} diff --git a/server/swimos_connector_kafka/src/ser/avro/mod.rs b/server/swimos_connector/src/ser/avro/mod.rs similarity index 100% rename from server/swimos_connector_kafka/src/ser/avro/mod.rs rename to server/swimos_connector/src/ser/avro/mod.rs diff --git a/server/swimos_connector_kafka/src/ser/json/mod.rs b/server/swimos_connector/src/ser/json/mod.rs similarity index 100% rename from server/swimos_connector_kafka/src/ser/json/mod.rs rename to server/swimos_connector/src/ser/json/mod.rs diff --git a/server/swimos_connector_kafka/src/ser/mod.rs b/server/swimos_connector/src/ser/mod.rs similarity index 98% rename from server/swimos_connector_kafka/src/ser/mod.rs rename to server/swimos_connector/src/ser/mod.rs index f053c015b..8881473e5 100644 --- a/server/swimos_connector_kafka/src/ser/mod.rs +++ b/server/swimos_connector/src/ser/mod.rs @@ -29,9 +29,10 @@ mod json; #[cfg(feature = "json")] pub use json::JsonSerializer; -use crate::{Endianness, SerializationError}; +use crate::deser::Endianness; +use crate::SerializationError; -/// A serializer that will attempt to produce a component of a Kafka message from a [value](Value). +/// A serializer that will attempt to produce a component of a message from a [value](Value). pub trait MessageSerializer { /// Attempt to serialize the value to a buffer. /// diff --git a/server/swimos_connector/src/test_support.rs b/server/swimos_connector/src/test_support.rs index f7a60537d..edbb80e56 100644 --- a/server/swimos_connector/src/test_support.rs +++ b/server/swimos_connector/src/test_support.rs @@ -12,15 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; - +use crate::ConnectorAgent; +use bytes::BytesMut; use futures::future::BoxFuture; +use futures::stream::FuturesUnordered; +use futures::{Stream, StreamExt}; +use std::collections::HashMap; +use std::pin::Pin; +use std::task::{Context, Poll}; +use swimos_agent::agent_model::downlink::BoxDownlinkChannelFactory; +use swimos_agent::event_handler::{ + ActionContext, DownlinkSpawnOnDone, EventHandler, EventHandlerError, HandlerFuture, + LaneSpawnOnDone, LaneSpawner, LinkSpawner, LocalBoxEventHandler, Spawner, StepResult, +}; use swimos_agent::AgentMetadata; +use swimos_api::address::Address; use swimos_api::agent::{ AgentConfig, AgentContext, DownlinkKind, HttpLaneRequestChannel, LaneConfig, StoreKind, WarpLaneKind, }; -use swimos_api::error::{AgentRuntimeError, DownlinkRuntimeError, OpenStoreError}; +use swimos_api::error::{ + AgentRuntimeError, CommanderRegistrationError, DownlinkRuntimeError, DynamicRegistrationError, + OpenStoreError, +}; +use swimos_model::Text; use swimos_utilities::byte_channel::{ByteReader, ByteWriter}; use swimos_utilities::routing::RouteUri; @@ -79,3 +94,100 @@ pub fn make_meta<'a>( ) -> AgentMetadata<'a> { AgentMetadata::new(uri, route_params, &CONFIG) } + +#[derive(Default)] +pub struct TestSpawner { + futures: FuturesUnordered>, +} + +impl Stream for TestSpawner { + type Item = LocalBoxEventHandler<'static, ConnectorAgent>; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.futures.poll_next_unpin(cx) + } +} + +impl TestSpawner { + pub fn is_empty(&self) -> bool { + self.futures.is_empty() + } +} + +impl Spawner for TestSpawner { + fn spawn_suspend(&self, fut: HandlerFuture) { + self.futures.push(fut); + } + + fn schedule_timer(&self, _at: tokio::time::Instant, _id: u64) { + panic!("Unexpected timer."); + } +} + +impl LinkSpawner for TestSpawner { + fn spawn_downlink( + &self, + _path: Address, + _make_channel: BoxDownlinkChannelFactory, + _on_done: DownlinkSpawnOnDone, + ) { + panic!("Spawning downlinks not supported."); + } + + fn register_commander(&self, _path: Address) -> Result { + panic!("Registering commanders not supported."); + } +} + +impl LaneSpawner for TestSpawner { + fn spawn_warp_lane( + &self, + _name: &str, + _kind: WarpLaneKind, + _on_done: LaneSpawnOnDone, + ) -> Result<(), DynamicRegistrationError> { + panic!("Spawning lanes not supported."); + } +} + +pub fn run_handler( + spawner: &TestSpawner, + command_buffer: &mut BytesMut, + agent: &ConnectorAgent, + mut handler: H, + on_err: F, +) where + H: EventHandler, + F: FnOnce(EventHandlerError), +{ + let uri = make_uri(); + let route_params = HashMap::new(); + let meta = make_meta(&uri, &route_params); + + let mut join_lane_init = HashMap::new(); + + let mut action_context = ActionContext::new( + spawner, + spawner, + spawner, + &mut join_lane_init, + command_buffer, + ); + + loop { + match handler.step(&mut action_context, meta, agent) { + StepResult::Continue { .. } => {} + StepResult::Fail(err) => { + on_err(err); + break; + } + StepResult::Complete { .. } => { + break; + } + } + } +} + +pub fn fail(err: EventHandlerError) { + panic!("{:?}", err) +} diff --git a/server/swimos_connector_fluvio/Cargo.toml b/server/swimos_connector_fluvio/Cargo.toml new file mode 100644 index 000000000..19eeac69c --- /dev/null +++ b/server/swimos_connector_fluvio/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "swimos_connector_fluvio" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true +description = "SwimOS Connector for Fluvio" +repository = "https://github.com/swimos/swim-rust/tree/main/server/swimos_connector_fluvio" + +[features] +default = [] +json = ["swimos_connector/json"] +avro = ["swimos_connector/json"] + +[dependencies] +fluvio = { workspace = true } +futures = { workspace = true } +swimos_api = { workspace = true } +swimos_utilities = { workspace = true } +swimos_model = { workspace = true } +swimos_form = { workspace = true } +swimos_recon = { workspace = true } +tokio = { workspace = true, features = ["sync", "macros", "fs"] } +swimos_agent = { workspace = true } +tracing = { workspace = true } +serde_json = { workspace = true, optional = true } +apache-avro = { workspace = true, optional = true } +swimos_connector = { workspace = true, features = ["pubsub"] } +thiserror = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread"] } +swimos_connector_util = { workspace = true } +rand = { workspace = true } diff --git a/server/swimos_connector_fluvio/src/config.rs b/server/swimos_connector_fluvio/src/config.rs new file mode 100644 index 000000000..ccd28c169 --- /dev/null +++ b/server/swimos_connector_fluvio/src/config.rs @@ -0,0 +1,242 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use fluvio::config::{TlsCerts, TlsConfig, TlsPaths, TlsPolicy as FluvioTlsPolicy}; +use fluvio::Offset; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use swimos_connector::config::format::DataFormat; +use swimos_connector::config::{ + IngressMapLaneSpec, IngressValueLaneSpec, PubSubRelaySpecification, +}; +use swimos_connector::selector::{PubSubSelector, Relays}; +use swimos_form::Form; +use swimos_recon::parser::parse_recognize; + +type BoxError = Box; + +/// Configuration parameters for the Fluvio connector. +#[derive(Clone, Debug, Form, PartialEq, Eq)] +#[form(tag = "fluvio_specification")] +struct FluvioIngressSpecification { + topic: String, + fluvio: FluvioSpecification, + partition: u32, + #[form(name = "offset")] + offset: OffsetSpecification, + value_lanes: Vec, + map_lanes: Vec, + key_deserializer: DataFormat, + payload_deserializer: DataFormat, + relays: Vec, +} + +impl FluvioIngressSpecification { + fn build(self) -> Result { + let FluvioIngressSpecification { + topic, + fluvio, + partition, + offset, + value_lanes, + map_lanes, + key_deserializer, + payload_deserializer, + relays, + } = self; + + Ok(FluvioIngressConfiguration { + topic, + fluvio: fluvio.build()?, + partition, + offset: offset.build()?, + value_lanes, + map_lanes, + key_deserializer, + payload_deserializer, + relays: Relays::try_from(relays)?, + }) + } +} + +#[derive(Clone, Debug, Form, PartialEq, Eq)] +enum OffsetSpecification { + Beginning(Option), + End(Option), + Absolute(i64), +} + +impl OffsetSpecification { + fn build(self) -> Result { + match self { + OffsetSpecification::Beginning(Some(n)) => Ok(Offset::from_beginning(n)), + OffsetSpecification::Beginning(None) => Ok(Offset::beginning()), + OffsetSpecification::End(Some(n)) => Ok(Offset::from_end(n)), + OffsetSpecification::End(None) => Ok(Offset::end()), + OffsetSpecification::Absolute(n) => Ok(Offset::absolute(n)?), + } + } +} + +#[derive(Clone, Debug, Form, PartialEq, Eq)] +#[form(tag = "fluvio")] +struct FluvioSpecification { + addr: String, + use_spu_local_address: Option, + tls: Option, + client_id: Option, +} + +impl FluvioSpecification { + fn build(self) -> Result { + let FluvioSpecification { + addr, + use_spu_local_address, + tls, + client_id, + } = self; + + let tls = match tls { + Some(tls) => tls.build(), + None => FluvioTlsPolicy::default(), + }; + + let mut config = fluvio::FluvioConfig::new(addr).with_tls(tls); + config.client_id = client_id; + + if let Some(use_spu_local_address) = use_spu_local_address { + config.use_spu_local_address = use_spu_local_address; + } + + Ok(config) + } +} + +/// Describes whether or not to use TLS and how. +#[derive(Clone, Debug, Form, PartialEq, Eq)] +enum TlsPolicy { + Disabled, + Anonymous, + Verified(FluvioTlsConfig), +} + +impl TlsPolicy { + fn build(self) -> FluvioTlsPolicy { + match self { + TlsPolicy::Disabled => FluvioTlsPolicy::Disabled, + TlsPolicy::Anonymous => FluvioTlsPolicy::Anonymous, + TlsPolicy::Verified(FluvioTlsConfig::Inline(paths)) => { + let FluvioTlsCerts { + domain, + key, + cert, + ca_cert, + } = paths; + FluvioTlsPolicy::Verified(TlsConfig::Inline(TlsCerts { + domain, + key, + cert, + ca_cert, + })) + } + TlsPolicy::Verified(FluvioTlsConfig::Files(paths)) => { + let FluvioTlsPaths { + domain, + key, + cert, + ca_cert, + } = paths; + FluvioTlsPolicy::Verified(TlsConfig::Files(TlsPaths { + domain, + key: PathBuf::from(key), + cert: PathBuf::from(cert), + ca_cert: PathBuf::from(ca_cert), + })) + } + } + } +} + +/// Describes the TLS configuration either inline or via file paths. +#[derive(Clone, Debug, Form, PartialEq, Eq)] +enum FluvioTlsConfig { + Inline(FluvioTlsCerts), + Files(FluvioTlsPaths), +} + +#[derive(Clone, Debug, Form, PartialEq, Eq)] +struct FluvioTlsCerts { + /// Domain name. + domain: String, + /// Client or Server private key. + key: String, + /// Client or Server certificate. + cert: String, + /// Certificate Authority cert. + ca_cert: String, +} + +/// TLS config with paths to keys and certs. +#[derive(Clone, Debug, Form, PartialEq, Eq)] +struct FluvioTlsPaths { + /// Domain name. + domain: String, + /// Path to client or server private key. + key: String, + /// Path to client or server certificate. + cert: String, + /// Path to Certificate Authority certificate. + ca_cert: String, +} + +#[derive(Clone, Debug)] +pub struct FluvioIngressConfiguration { + /// The topic to consume from. + pub topic: String, + /// Fluvio library configuration. + pub fluvio: fluvio::FluvioConfig, + /// The partition to consume from. + pub partition: u32, + /// The offset to start consuming from. + pub offset: Offset, + /// Specifications for the value lanes to define for the connector. This includes a pattern to + /// define a selector that will pick out values to set to that lane, from a Fluvio message. + pub value_lanes: Vec, + /// Specifications for the map lanes to define for the connector. This includes a pattern to + /// define a selector that will pick out updates to apply to that lane, from a Fluvio message. + pub map_lanes: Vec, + /// Deserialization format to use to interpret the contents of the keys of the Fluvio messages. + pub key_deserializer: DataFormat, + /// Deserialization format to use to interpret the contents of the payloads of the Fluvio + /// messages. + pub payload_deserializer: DataFormat, + /// Collection of relays used for forwarding messages to lanes on agents. + pub relays: Relays, +} + +impl FluvioIngressConfiguration { + pub async fn from_file(path: impl AsRef) -> Result { + let content = tokio::fs::read_to_string(path).await?; + FluvioIngressConfiguration::from_str(&content) + } +} + +impl FromStr for FluvioIngressConfiguration { + type Err = BoxError; + + fn from_str(s: &str) -> Result { + let config = parse_recognize::(s, true)?.build()?; + Ok(config) + } +} diff --git a/server/swimos_connector_fluvio/src/ingress/mod.rs b/server/swimos_connector_fluvio/src/ingress/mod.rs new file mode 100644 index 000000000..e8dd137d5 --- /dev/null +++ b/server/swimos_connector_fluvio/src/ingress/mod.rs @@ -0,0 +1,386 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[cfg(test)] +mod tests; + +use crate::config::FluvioIngressConfiguration; +use crate::FluvioConnectorError; +use fluvio::consumer::{ConsumerConfigExt, Record}; +use fluvio::dataplane::link::ErrorCode; +use fluvio::dataplane::record::ConsumerRecord; +use fluvio::dataplane::types::PartitionId; +use fluvio::{Fluvio, FluvioError}; +use futures::stream::{unfold, BoxStream}; +use futures::StreamExt; +use std::cell::RefCell; +use std::future::{ready, Future}; +use swimos_agent::agent_lifecycle::HandlerContext; +use swimos_agent::event_handler::{EventHandler, UnitHandler}; +use swimos_api::agent::WarpLaneKind; +use swimos_connector::config::format::DataFormat; +use swimos_connector::deser::{BoxMessageDeserializer, MessageView}; +use swimos_connector::ingress::{pubsub::MessageSelector, Lanes}; +use swimos_connector::selector::PubSubSelector; +use swimos_connector::{ + BaseConnector, ConnectorAgent, ConnectorStream, IngressConnector, IngressContext, LoadError, +}; +use swimos_utilities::trigger::Sender; +use tracing::{error, info, trace}; + +/// A Fluivo ingress [connector](`swimos_connector::IngressConnector`) to ingest a stream of Fluvio +/// records into a Swim application. +#[derive(Debug, Clone)] +pub struct FluvioIngressConnector { + configuration: FluvioIngressConfiguration, + lanes: RefCell>, + factory: F, +} + +impl FluvioIngressConnector { + pub fn new(configuration: FluvioIngressConfiguration, factory: F) -> FluvioIngressConnector { + FluvioIngressConnector { + configuration, + lanes: RefCell::new(Default::default()), + factory, + } + } +} + +impl FluvioIngressConnector { + /// Create a [`FluvioIngressConnector`] with the provided configuration. The configuration is + /// only validated when the agent attempts to start so this will never fail. + /// + /// # Arguments + /// * `configuration` - The connector configuration, specifying the connection details for the + /// Fluvio consumer and the lanes that the connector agent should expose. + pub fn for_config( + configuration: FluvioIngressConfiguration, + ) -> FluvioIngressConnector { + FluvioIngressConnector::new(configuration, FluvioIngressConsumer) + } +} + +impl BaseConnector for FluvioIngressConnector { + fn on_start(&self, init_complete: Sender) -> impl EventHandler + '_ { + let handler_context = HandlerContext::::default(); + handler_context.effect(move || { + init_complete.trigger(); + }) + } + + fn on_stop(&self) -> impl EventHandler + '_ { + UnitHandler::default() + } +} + +impl IngressConnector for FluvioIngressConnector +where + F: FluvioConsumer, +{ + type Error = FluvioConnectorError; + + fn create_stream(&self) -> Result, Self::Error> { + let FluvioIngressConnector { + configuration, + lanes, + factory, + } = self; + let FluvioIngressConfiguration { + topic, + key_deserializer, + payload_deserializer, + relays, + .. + } = configuration; + + let key_deser = key_deserializer.clone(); + let value_deser = payload_deserializer.clone(); + let lanes = lanes.take(); + let topic = topic.clone(); + let relays = relays.clone(); + + Ok(unfold( + ConnectorState::Uninit(configuration.clone(), factory.clone()), + move |state: ConnectorState| { + let topic = topic.clone(); + let key_deser = key_deser.clone(); + let value_deser = value_deser.clone(); + let lanes = lanes.clone(); + let relays = relays.clone(); + + let fut = async move { + match state { + ConnectorState::Uninit(config, factory) => match factory.open(config).await + { + Ok(consumer) => { + let (key, value) = + match load_deserializers(key_deser, value_deser).await { + Ok((key, value)) => (key, value), + Err(e) => { + return Some(( + Err(FluvioConnectorError::Configuration(e)), + ConnectorState::Failed, + )) + } + }; + poll_dispatch( + consumer, + topic, + MessageSelector::new(key, value, lanes, relays), + ) + .await + } + Err(e) => Some((Err(e), ConnectorState::Failed)), + }, + ConnectorState::Running { + topic, + consumer, + message_selector, + } => poll_dispatch(consumer, topic, message_selector).await, + ConnectorState::Failed => None, + } + }; + Box::pin(fut) + }, + )) + } + + fn initialize(&self, context: &mut dyn IngressContext) -> Result<(), Self::Error> { + let FluvioIngressConnector { + lanes, + configuration, + .. + } = self; + + let mut guard = lanes.borrow_mut(); + match Lanes::try_from_lane_specs(&configuration.value_lanes, &configuration.map_lanes) { + Ok(lanes_from_conf) => { + for lane_spec in lanes_from_conf.value_lanes() { + context.open_lane(lane_spec.name(), WarpLaneKind::Value); + } + for lane_spec in lanes_from_conf.map_lanes() { + context.open_lane(lane_spec.name(), WarpLaneKind::Map); + } + *guard = lanes_from_conf; + } + Err(err) => { + error!(error = %err, "Failed to create lanes for a Fluvio connector."); + return Err(err.into()); + } + } + Ok(()) + } +} + +enum ConnectorState { + Uninit(FluvioIngressConfiguration, F), + Running { + topic: String, + consumer: C, + message_selector: MessageSelector, + }, + Failed, +} + +trait FluvioConsumer: Clone + Send + 'static { + type Client: FluvioClient; + + fn open( + &self, + config: FluvioIngressConfiguration, + ) -> impl Future> + Send; +} + +trait FluvioRecord: Send { + fn offset(&self) -> i64; + + fn partition_id(&self) -> PartitionId; + + fn key(&self) -> &[u8]; + + fn value(&self) -> &[u8]; +} + +impl FluvioRecord for ConsumerRecord { + fn offset(&self) -> i64 { + self.offset + } + + fn partition_id(&self) -> PartitionId { + self.partition + } + + fn key(&self) -> &[u8] { + self.record.key().map(|k| k.as_ref()).unwrap_or_default() + } + + fn value(&self) -> &[u8] { + self.record.value().as_ref() + } +} + +trait FluvioClient: Send + 'static { + type FluvioRecord: FluvioRecord; + + fn next( + &mut self, + ) -> impl Future>> + Send; + + fn shutdown(&mut self) -> impl Future + Send; +} + +#[derive(Clone)] +pub struct FluvioIngressConsumer; + +impl FluvioConsumer for FluvioIngressConsumer { + type Client = FluvioStreamClient; + + async fn open( + &self, + config: FluvioIngressConfiguration, + ) -> Result { + let FluvioIngressConfiguration { + topic, + fluvio, + partition, + offset, + .. + } = config; + + match Fluvio::connect_with_config(&fluvio).await { + Ok(handle) => { + let consumer_config = match ConsumerConfigExt::builder() + .topic(topic) + .offset_start(offset) + .partition(partition) + .build() + { + Ok(config) => config, + Err(error) => { + error!(?error, "Failed to build consumer config"); + return Err(FluvioConnectorError::Message(error.to_string())); + } + }; + + match handle.consumer_with_config(consumer_config).await { + Ok(consumer) => { + info!("Fluvio consumer successfully opened"); + Ok(FluvioStreamClient { + fluvio: Some(handle), + stream: consumer.boxed(), + }) + } + Err(error) => { + error!(?error, "Failed to create Fluvio consumer"); + Err(FluvioConnectorError::Message(error.to_string())) + } + } + } + Err(error) => { + error!(?error, "Failed to connect to Fluvio cluster"); + Err(FluvioConnectorError::Message(error.to_string())) + } + } + } +} + +pub struct FluvioStreamClient { + fluvio: Option, + stream: BoxStream<'static, Result>, +} + +impl FluvioClient for FluvioStreamClient { + type FluvioRecord = ConsumerRecord; + + fn next( + &mut self, + ) -> impl Future>> + Send { + let stream = &mut self.stream; + async move { + match stream.next().await { + Some(Ok(record)) => Some(Ok(record)), + Some(Err(code)) => { + error!(%code, "Fluvio consumer failed to read"); + Some(Err(FluvioConnectorError::Native(FluvioError::Other( + code.to_string(), + )))) + } + None => None, + } + } + } + + fn shutdown(&mut self) -> impl Future + Send { + // drop the Fluvio instance to initiate a shutdown. + // we have to keep it around until a shutdown is requested to prevent an early shutdown. + let _ = self.fluvio.take(); + ready(()) + } +} + +async fn load_deserializers( + key: DataFormat, + value: DataFormat, +) -> Result<(BoxMessageDeserializer, BoxMessageDeserializer), LoadError> { + let key = key.load_deserializer().await?; + let value = value.load_deserializer().await?; + Ok((key, value)) +} + +async fn poll_dispatch( + mut consumer: C, + topic: String, + message_selector: MessageSelector, +) -> Option<( + Result + Send + 'static, FluvioConnectorError>, + ConnectorState, +)> +where + C: FluvioClient, +{ + match consumer.next().await { + Some(Ok(record)) => { + let offset = record.offset(); + let partition = record.partition_id(); + + trace!(?offset, ?partition, topic=%topic, "Handling record"); + + let view = MessageView { + topic: topic.as_str(), + key: record.key(), + payload: record.value(), + }; + + let handle_result = message_selector.handle_message(&view).map_err(Into::into); + if let Err(err) = &handle_result { + error!(error = %err, "Failed to handle message"); + } + + Some(( + handle_result, + ConnectorState::Running { + topic, + consumer, + message_selector, + }, + )) + } + Some(Err(e)) => { + consumer.shutdown().await; + Some((Err(e), ConnectorState::Failed)) + } + None => None, + } +} diff --git a/server/swimos_connector_fluvio/src/ingress/tests.rs b/server/swimos_connector_fluvio/src/ingress/tests.rs new file mode 100644 index 000000000..4f451b9dd --- /dev/null +++ b/server/swimos_connector_fluvio/src/ingress/tests.rs @@ -0,0 +1,333 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::ingress::{FluvioClient, FluvioConsumer, FluvioRecord}; +use crate::{ErrorCode, FluvioConnectorError, FluvioIngressConfiguration, FluvioIngressConnector}; +use fluvio::dataplane::types::PartitionId; +use fluvio::Offset; +use futures::future::join; +use futures::TryStreamExt; +use rand::Rng; +use std::collections::{HashMap, VecDeque}; +use std::future::{ready, Future}; +use std::time::Duration; +use swimos_agent::agent_model::AgentSpec; +use swimos_agent::agent_model::{ItemDescriptor, ItemFlags}; +use swimos_api::agent::WarpLaneKind; +use swimos_connector::config::format::DataFormat; +use swimos_connector::config::{IngressMapLaneSpec, IngressValueLaneSpec}; +use swimos_connector::{BaseConnector, ConnectorAgent, IngressConnector, IngressContext}; +use swimos_connector_util::run_handler_with_futures; +use swimos_model::{Item, Value}; +use swimos_recon::print_recon_compact; +use swimos_utilities::trigger; + +const TEST_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Default)] +struct TestIngressContext { + requests: Vec<(String, WarpLaneKind)>, +} + +impl IngressContext for TestIngressContext { + fn open_lane(&mut self, name: &str, kind: WarpLaneKind) { + self.requests.push((name.to_string(), kind)); + } +} + +fn make_config() -> FluvioIngressConfiguration { + FluvioIngressConfiguration { + topic: "topic".to_string(), + fluvio: fluvio::FluvioConfig::new("127.0.0.1:9000"), + partition: 0, + offset: Offset::beginning(), + value_lanes: vec![IngressValueLaneSpec::new(None, "$key", true)], + map_lanes: vec![IngressMapLaneSpec::new( + "map", + "$payload.key", + "$payload.value", + true, + true, + )], + key_deserializer: DataFormat::Recon, + payload_deserializer: DataFormat::Recon, + relays: Default::default(), + } +} + +#[derive(Debug, Clone)] +struct TestRecord { + offset: i64, + partition_id: PartitionId, + key: Vec, + key_v: Value, + value: Vec, + value_v: Value, +} + +impl FluvioRecord for TestRecord { + fn offset(&self) -> i64 { + self.offset + } + + fn partition_id(&self) -> PartitionId { + self.partition_id + } + + fn key(&self) -> &[u8] { + self.key.as_slice() + } + + fn value(&self) -> &[u8] { + self.value.as_slice() + } +} + +#[derive(Clone)] +struct FnProducer(F); + +impl FnProducer { + fn new(funky: F) -> Self + where + F: Fn() -> VecDeque> + Send + 'static, + { + FnProducer(funky) + } +} + +impl FluvioConsumer for FnProducer +where + F: Fn() -> VecDeque> + Send + Clone + 'static, +{ + type Client = VecClient; + + fn open( + &self, + _config: FluvioIngressConfiguration, + ) -> impl Future> + Send { + ready(Ok(VecClient(self.0()))) + } +} + +#[derive(Default)] +struct VecClient(VecDeque>); + +impl FluvioClient for VecClient { + type FluvioRecord = TestRecord; + + fn next( + &mut self, + ) -> impl Future>> + Send { + ready(self.0.pop_front()) + } + + fn shutdown(&mut self) -> impl Future + Send { + ready(()) + } +} + +#[test] +fn connector_initialize() { + let mut context = TestIngressContext::default(); + let config = make_config(); + let factory = FnProducer::new(VecDeque::default); + let connector = FluvioIngressConnector::new(config, factory); + assert!(connector.initialize(&mut context).is_ok()); + let requests = context.requests; + assert_eq!(requests.len(), 2); + + let lane_map = requests.into_iter().collect::>(); + let lanes_expected = [ + ("key".to_string(), WarpLaneKind::Value), + ("map".to_string(), WarpLaneKind::Map), + ] + .into_iter() + .collect::>(); + assert_eq!(lane_map, lanes_expected); +} + +fn make_key_value(key: impl Into, value: impl Into) -> Value { + Value::record(vec![Item::slot("key", key), Item::slot("value", value)]) +} + +fn generate_messages(n: usize) -> Vec { + let mut rng = rand::thread_rng(); + let mut messages = Vec::with_capacity(n); + + for i in 0..n { + let key = Value::from(rng.r#gen::()); + let payload_value = rng.r#gen::(); + let payload = make_key_value(key.clone(), payload_value); + + let message = TestRecord { + offset: i as i64, + partition_id: 0, + key: format!("{}", print_recon_compact(&key)).as_bytes().to_vec(), + key_v: key, + value: format!("{}", print_recon_compact(&payload)) + .as_bytes() + .to_vec(), + value_v: Value::from(payload_value), + }; + messages.push(message); + } + messages +} + +#[tokio::test] +async fn connector_on_start() { + tokio::time::timeout(TEST_TIMEOUT, async { + let num_messages = 3; + let messages = generate_messages(num_messages); + let config = make_config(); + let factory = FnProducer::new(move || { + messages + .clone() + .into_iter() + .map(Ok) + .collect::>() + }); + let connector = FluvioIngressConnector::new(config, factory); + + let (tx, rx) = trigger::trigger(); + let handler = connector.on_start(tx); + + let agent = ConnectorAgent::default(); + let start_task = run_handler_with_futures(&agent, handler); + + let (modified, result) = join(start_task, rx).await; + assert!(result.is_ok()); + assert!(modified.is_empty()); + }) + .await + .expect("Test timed out."); +} + +async fn init_agent(connector: &FluvioIngressConnector, agent: &ConnectorAgent) +where + F: FluvioConsumer, +{ + let mut context = TestIngressContext::default(); + assert!(connector.initialize(&mut context).is_ok()); + + for (name, kind) in context.requests { + assert!(agent + .register_dynamic_item( + &name, + ItemDescriptor::WarpLane { + kind, + flags: ItemFlags::TRANSIENT + } + ) + .is_ok()); + } + + let (tx, rx) = trigger::trigger(); + let handler = connector.on_start(tx); + let start_task = run_handler_with_futures(agent, handler); + let (_, result) = join(start_task, rx).await; + + assert!(result.is_ok()); +} + +#[derive(Default)] +struct MessageChecker { + expected_map: HashMap, +} + +impl MessageChecker { + fn check_message(&mut self, agent: &mut ConnectorAgent, message: &TestRecord) { + let MessageChecker { expected_map } = self; + let TestRecord { key_v, value_v, .. } = message; + let guard = agent.value_lane("key").expect("Lane missing."); + guard.read(|v| { + assert_eq!(v, key_v); + }); + drop(guard); + let guard = agent.map_lane("map").expect("Lane missing."); + expected_map.insert(key_v.clone(), value_v.clone()); + guard.get_map(|map| { + assert_eq!(map, expected_map); + }); + } +} + +#[tokio::test] +async fn connector_stream() { + tokio::time::timeout(TEST_TIMEOUT, async { + let num_messages = 3; + let messages = generate_messages(num_messages); + let config = make_config(); + let producer_messages = messages.clone(); + let factory = FnProducer::new(move || { + producer_messages + .clone() + .into_iter() + .map(Ok) + .collect::>() + }); + let connector = FluvioIngressConnector::new(config, factory); + + let mut agent = ConnectorAgent::default(); + init_agent(&connector, &agent).await; + + let mut stream = connector.create_stream().expect("Connector failed."); + + let mut checker = MessageChecker::default(); + for message in &messages { + let handler = stream + .try_next() + .await + .expect("Consumer failed.") + .expect("Consumer terminated."); + run_handler_with_futures(&agent, handler).await; + checker.check_message(&mut agent, message); + } + }) + .await + .expect("Test timed out."); +} + +#[tokio::test] +async fn failed_connector_stream_start() { + tokio::time::timeout(TEST_TIMEOUT, async { + let config = make_config(); + let factory = FnProducer::new(move || { + VecDeque::from([Err(FluvioConnectorError::Code(ErrorCode::TopicError))]) + }); + let connector = FluvioIngressConnector::new(config, factory); + let agent = ConnectorAgent::default(); + init_agent(&connector, &agent).await; + + let item = connector + .create_stream() + .expect("Opening a Fluvio connector should be infallible") + .try_next() + .await; + + match item { + Ok(Some(_)) | Ok(None) => { + panic!("Connector should have yielded an error") + } + Err(e) => { + assert!(matches!( + e, + FluvioConnectorError::Code(ErrorCode::TopicError) + )); + } + } + }) + .await + .expect("Test timed out."); +} diff --git a/server/swimos_connector_fluvio/src/lib.rs b/server/swimos_connector_fluvio/src/lib.rs new file mode 100644 index 000000000..57f010849 --- /dev/null +++ b/server/swimos_connector_fluvio/src/lib.rs @@ -0,0 +1,53 @@ +// Copyright 2015-2024 Swim Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod config; +mod ingress; + +pub use config::FluvioIngressConfiguration; +pub use fluvio::dataplane::link::ErrorCode; +pub use fluvio::FluvioError; +pub use ingress::{FluvioIngressConnector, FluvioIngressConsumer}; +use swimos_connector::selector::{InvalidLanes, SelectorError}; +pub use swimos_connector::{ + config::{IngressMapLaneSpec, IngressValueLaneSpec}, + deser::Endianness, + DeserializationError, LoadError, SerializationError, +}; + +/// Errors that can be produced by the Fluvio connector. +#[derive(thiserror::Error, Debug)] +pub enum FluvioConnectorError { + /// Fluvio Library Error. + #[error("Fluvio client error: {0}")] + Native(FluvioError), + /// Fluvio error code. + #[error("Fluvio dataplane error: {0}")] + Code(ErrorCode), + /// Failed to load the deserializers required to interpret the Fluvio messages. + #[error("Failed to load deserializer: {0}")] + Configuration(#[from] LoadError), + /// Attempting to select the required components of a Fluvio message failed. + #[error("Failed to select from a message: {0}")] + Lane(#[from] SelectorError), + /// String error message. + #[error("{0}")] + Message(String), + /// The specification of at least one lane is invalid. + #[error(transparent)] + Lanes(#[from] InvalidLanes), + /// A selector failed to select a value from a Fluvio record. + #[error(transparent)] + Selector(SelectorError), +} diff --git a/server/swimos_connector_kafka/Cargo.toml b/server/swimos_connector_kafka/Cargo.toml index 8b114424b..01052ec54 100644 --- a/server/swimos_connector_kafka/Cargo.toml +++ b/server/swimos_connector_kafka/Cargo.toml @@ -10,8 +10,8 @@ homepage.workspace = true [features] default = [] -json = ["dep:serde_json"] -avro = ["dep:apache-avro", "dep:chrono"] +json = ["swimos_connector/json"] +avro = ["swimos_connector/json", "dep:apache-avro"] [dependencies] futures = { workspace = true } @@ -25,16 +25,14 @@ swimos_agent = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } rdkafka = { workspace = true, features = ["cmake-build", "tokio"] } -serde_json = { workspace = true, optional = true } -apache-avro = { workspace = true, optional = true} -chrono = { workspace = true, optional = true } +apache-avro = { workspace = true, optional = true } thiserror = { workspace = true } -swimos_connector = { workspace = true } -regex = { workspace = true } +swimos_connector = { workspace = true, features = ["pubsub"] } frunk = { workspace = true } bytes = { workspace = true } [dev-dependencies] +swimos_connector_util = { workspace = true } tokio = { workspace = true, features = ["rt"] } uuid = { workspace = true, features = ["v4"] } bytes = { workspace = true } diff --git a/server/swimos_connector_kafka/src/config/ingress.rs b/server/swimos_connector_kafka/src/config/ingress.rs index 520ed7b86..7f48fc0cc 100644 --- a/server/swimos_connector_kafka/src/config/ingress.rs +++ b/server/swimos_connector_kafka/src/config/ingress.rs @@ -12,14 +12,60 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; - use super::{DataFormat, KafkaLogLevel}; +use std::collections::HashMap; +use std::path::Path; +use std::str::FromStr; +use swimos_connector::config::{ + IngressMapLaneSpec, IngressValueLaneSpec, PubSubRelaySpecification, +}; +use swimos_connector::selector::{BadSelector, PubSubSelector, Relays}; use swimos_form::Form; +use swimos_recon::parser::parse_recognize; + +type BoxError = Box; /// Configuration parameters for the Kafka ingress connector. #[derive(Clone, Debug, Form, PartialEq, Eq)] #[form(tag = "kafka")] +struct KafkaIngressSpecification { + properties: HashMap, + log_level: KafkaLogLevel, + value_lanes: Vec, + map_lanes: Vec, + key_deserializer: DataFormat, + payload_deserializer: DataFormat, + topics: Vec, + relays: Vec, +} + +impl KafkaIngressSpecification { + pub fn build(self) -> Result { + let KafkaIngressSpecification { + properties, + log_level, + value_lanes, + map_lanes, + key_deserializer, + payload_deserializer, + topics, + relays, + } = self; + + Ok(KafkaIngressConfiguration { + properties, + log_level, + value_lanes, + map_lanes, + key_deserializer, + payload_deserializer, + topics, + relays: Relays::try_from(relays)?, + }) + } +} + +#[derive(Clone, Debug)] pub struct KafkaIngressConfiguration { /// Properties to configure the Kafka consumer. pub properties: HashMap, @@ -37,76 +83,22 @@ pub struct KafkaIngressConfiguration { pub payload_deserializer: DataFormat, /// A list of Kafka topics to subscribe to. pub topics: Vec, + /// Collection of relays used for forwarding messages to lanes on agents. + pub relays: Relays, } -/// Specification of a value lane for the Kafka connector. -#[derive(Clone, Debug, Form, PartialEq, Eq)] -#[form(tag = "ValueLaneSpec")] -pub struct IngressValueLaneSpec { - /// A name to use for the lane. If not specified, the connector will attempt to infer one from the selector. - pub name: Option, - /// String representation of a selector to extract values for the lane from Kafka messages. - pub selector: String, - /// Whether the lane is required. If this is `true` and the selector returns nothing for a Kafka Message, the - /// connector will fail with an error. - pub required: bool, -} - -impl IngressValueLaneSpec { - /// # Arguments - /// * `name` - A name to use for the lane. If not specified the connector will attempt to infer a name from the selector. - /// * `selector` - String representation of the selector to extract values from the Kafka message. - /// * `required` - Whether the lane is required. If this is `true` and the selector returns nothing for a Kafka Message, the - /// connector will fail with an error. - pub fn new>(name: Option, selector: S, required: bool) -> Self { - IngressValueLaneSpec { - name: name.map(Into::into), - selector: selector.into(), - required, - } +impl KafkaIngressConfiguration { + pub async fn from_file(path: impl AsRef) -> Result { + let content = tokio::fs::read_to_string(path).await?; + KafkaIngressConfiguration::from_str(&content) } } -/// Specification of a value lane for the Kafka connector. -#[derive(Clone, Debug, Form, PartialEq, Eq)] -#[form(tag = "MapLaneSpec")] -pub struct IngressMapLaneSpec { - /// The name of the lane. - pub name: String, - /// String representation of a selector to extract the map keys from the Kafka messages. - pub key_selector: String, - /// String representation of a selector to extract the map values from the Kafka messages. - pub value_selector: String, - /// Whether to remove an entry from the map if the value selector does not return a value. Otherwise, missing - /// values will be treated as a failed extraction from the message. - pub remove_when_no_value: bool, - /// Whether the lane is required. If this is `true` and the selector returns nothing for a Kafka Message, the - /// connector will fail with an error. - pub required: bool, -} +impl FromStr for KafkaIngressConfiguration { + type Err = BoxError; -impl IngressMapLaneSpec { - /// # Arguments - /// * `name` - The name of the lane. - /// * `key_selector` - String representation of a selector to extract the map keys from the Kafka messages. - /// * `value_selector` - String representation of a selector to extract the map values from the Kafka messages. - /// * `remove_when_no_value` - Whether to remove an entry from the map if the value selector does not return a value. Otherwise, missing - /// values will be treated as a failed extraction from the message. - /// * `required` - Whether the lane is required. If this is `true` and the selector returns nothing for a Kafka Message, the - /// connector will fail with an error. - pub fn new>( - name: S, - key_selector: S, - value_selector: S, - remove_when_no_value: bool, - required: bool, - ) -> Self { - IngressMapLaneSpec { - name: name.into(), - key_selector: key_selector.into(), - value_selector: value_selector.into(), - remove_when_no_value, - required, - } + fn from_str(s: &str) -> Result { + let config = parse_recognize::(s, true)?.build()?; + Ok(config) } } diff --git a/server/swimos_connector_kafka/src/config/mod.rs b/server/swimos_connector_kafka/src/config/mod.rs index 7ed5a7319..269775445 100644 --- a/server/swimos_connector_kafka/src/config/mod.rs +++ b/server/swimos_connector_kafka/src/config/mod.rs @@ -15,15 +15,14 @@ use rdkafka::config::RDKafkaLogLevel; mod egress; -mod format; mod ingress; pub use egress::{ DownlinkAddress, EgressDownlinkSpec, EgressLaneSpec, ExtractionSpec, KafkaEgressConfiguration, TopicSpecifier, }; -pub use format::DataFormat; -pub use ingress::{IngressMapLaneSpec, IngressValueLaneSpec, KafkaIngressConfiguration}; +pub use ingress::KafkaIngressConfiguration; +pub use swimos_connector::config::format::DataFormat; use swimos_form::Form; /// Enumeration of logging levels supported by the underlying Kafka consumer. diff --git a/server/swimos_connector_kafka/src/connector/egress/mod.rs b/server/swimos_connector_kafka/src/connector/egress/mod.rs index 128b27ed9..faac66f23 100644 --- a/server/swimos_connector_kafka/src/connector/egress/mod.rs +++ b/server/swimos_connector_kafka/src/connector/egress/mod.rs @@ -14,30 +14,27 @@ use std::{cell::RefCell, collections::HashMap, sync::Arc, time::Duration}; +use super::ConnHandlerContext; +use crate::selector::message::{MessageSelector, MessageSelectors}; +use crate::{ + config::KafkaEgressConfiguration, + facade::{KafkaFactory, KafkaProducer, ProduceResult, ProducerFactory}, + KafkaSenderError, +}; use bytes::BytesMut; use futures::{channel::oneshot, FutureExt}; use swimos_agent::event_handler::{ EventHandler, HandlerActionExt, TryHandlerActionExt, UnitHandler, }; use swimos_api::{address::Address, agent::WarpLaneKind}; +use swimos_connector::ser::SharedMessageSerializer; use swimos_connector::{ BaseConnector, ConnectorAgent, ConnectorFuture, EgressConnector, EgressConnectorSender, - EgressContext, MessageSource, SendResult, + EgressContext, LoadError, MessageSource, SendResult, SerializationError, }; use swimos_model::Value; use swimos_utilities::trigger; -use crate::{ - config::KafkaEgressConfiguration, - error::SerializationError, - facade::{KafkaFactory, KafkaProducer, ProduceResult, ProducerFactory}, - selector::{MessageSelector, MessageSelectors}, - ser::SharedMessageSerializer, - KafkaSenderError, LoadError, -}; - -use super::ConnHandlerContext; - #[cfg(test)] mod tests; diff --git a/server/swimos_connector_kafka/src/connector/egress/tests/end_to_end.rs b/server/swimos_connector_kafka/src/connector/egress/tests/end_to_end.rs index 45372a405..174e982f0 100644 --- a/server/swimos_connector_kafka/src/connector/egress/tests/end_to_end.rs +++ b/server/swimos_connector_kafka/src/connector/egress/tests/end_to_end.rs @@ -14,22 +14,23 @@ use std::{collections::HashMap, time::SystemTime}; +use crate::config::TopicSpecifier; +use crate::{ + connector::test_util::create_kafka_props, DataFormat, EgressLaneSpec, ExtractionSpec, + KafkaEgressConfiguration, KafkaEgressConnector, KafkaLogLevel, +}; use futures::{future::join, TryFutureExt}; use rand::Rng; +use swimos_connector::deser::Endianness; use swimos_connector::{ BaseConnector, ConnectorAgent, EgressConnector, EgressConnectorSender, MessageSource, SendResult, }; +use swimos_connector_util::run_handler_with_futures; use swimos_model::{Item, Value}; use swimos_utilities::trigger; use tokio::time::sleep; -use crate::config::TopicSpecifier; -use crate::{ - connector::test_util::{create_kafka_props, run_handler_with_futures}, - DataFormat, EgressLaneSpec, Endianness, ExtractionSpec, KafkaEgressConfiguration, - KafkaEgressConnector, KafkaLogLevel, -}; const LANE: &str = "lane"; fn make_config() -> KafkaEgressConfiguration { diff --git a/server/swimos_connector_kafka/src/connector/egress/tests/integration.rs b/server/swimos_connector_kafka/src/connector/egress/tests/integration.rs index d08108053..464adf030 100644 --- a/server/swimos_connector_kafka/src/connector/egress/tests/integration.rs +++ b/server/swimos_connector_kafka/src/connector/egress/tests/integration.rs @@ -33,18 +33,16 @@ use swimos_connector::{ BaseConnector, ConnectorAgent, EgressConnector, EgressConnectorSender, EgressContext, MessageSource, SendResult, }; +use swimos_connector_util::{run_handler_with_futures, run_handler_with_futures_dl}; use swimos_model::{Item, Value}; use swimos_recon::print_recon_compact; use swimos_utilities::trigger; +use crate::selector::message::MessageSelector; use crate::{ config::{EgressDownlinkSpec, EgressLaneSpec, KafkaEgressConfiguration, TopicSpecifier}, - connector::{ - egress::{ConnectorState, KafkaEgressConnector}, - test_util::{run_handler_with_futures, run_handler_with_futures_dl}, - }, + connector::egress::{ConnectorState, KafkaEgressConnector}, facade::{KafkaProducer, ProduceResult, ProducerFactory}, - selector::MessageSelector, DataFormat, DownlinkAddress, ExtractionSpec, KafkaLogLevel, }; diff --git a/server/swimos_connector_kafka/src/connector/egress/tests/mod.rs b/server/swimos_connector_kafka/src/connector/egress/tests/mod.rs index e86eb1568..f5827351e 100644 --- a/server/swimos_connector_kafka/src/connector/egress/tests/mod.rs +++ b/server/swimos_connector_kafka/src/connector/egress/tests/mod.rs @@ -17,13 +17,12 @@ use std::collections::HashMap; use swimos_api::{address::Address, agent::WarpLaneKind}; use swimos_connector::EgressContext; +use super::open_downlinks; use crate::{ config::{EgressDownlinkSpec, KafkaEgressConfiguration, TopicSpecifier}, DataFormat, DownlinkAddress, ExtractionSpec, KafkaLogLevel, }; -use super::open_downlinks; - #[cfg(feature = "json")] mod end_to_end; mod integration; diff --git a/server/swimos_connector_kafka/src/connector/ingress/mod.rs b/server/swimos_connector_kafka/src/connector/ingress/mod.rs index 9a14aa77a..2d11738d0 100644 --- a/server/swimos_connector_kafka/src/connector/ingress/mod.rs +++ b/server/swimos_connector_kafka/src/connector/ingress/mod.rs @@ -16,27 +16,24 @@ mod tests; use std::cell::RefCell; -use std::collections::HashSet; -use crate::config::KafkaIngressConfiguration; -use crate::deser::{BoxMessageDeserializer, MessagePart, MessageView}; -use crate::error::{KafkaConnectorError, LaneSelectorError}; +use super::ConnHandlerContext; +use crate::error::KafkaConnectorError; use crate::facade::{ConsumerFactory, KafkaConsumer, KafkaFactory, KafkaMessage}; -use crate::selector::{Computed, MapLaneSelector, ValueLaneSelector}; -use crate::{IngressMapLaneSpec, IngressValueLaneSpec, InvalidLanes}; +use crate::KafkaIngressConfiguration; use futures::{stream::unfold, Future}; -use swimos_agent::event_handler::{EventHandler, HandlerActionExt, Sequentially, UnitHandler}; +use swimos_agent::agent_lifecycle::HandlerContext; +use swimos_agent::event_handler::{EventHandler, HandlerActionExt, UnitHandler}; use swimos_api::agent::WarpLaneKind; -use swimos_model::Value; -use swimos_utilities::trigger; -use tokio::sync::mpsc; -use tracing::{debug, error, info, trace}; - +use swimos_connector::deser::MessageView; +use swimos_connector::ingress::{pubsub::MessageSelector, Lanes}; +use swimos_connector::selector::{PubSubSelector, SelectorError}; use swimos_connector::{ BaseConnector, ConnectorAgent, ConnectorStream, IngressConnector, IngressContext, }; - -use super::ConnHandlerContext; +use swimos_utilities::trigger; +use tokio::sync::mpsc; +use tracing::{debug, error, info}; /// A [connector](IngressConnector) to ingest a stream of Kafka messages into a Swim application. This should be used to /// provide a lifecycle for a [connector agent](ConnectorAgent). @@ -54,7 +51,7 @@ use super::ConnHandlerContext; pub struct KafkaIngressConnector { factory: F, configuration: KafkaIngressConfiguration, - lanes: RefCell, + lanes: RefCell>, } impl KafkaIngressConnector { @@ -73,7 +70,7 @@ impl KafkaIngressConnector { /// /// # Arguments /// * `configuration` - The connector configuration, specifying the connection details for the Kafka consumer - /// an the lanes that the connector agent should expose. + /// and the lanes that the connector agent should expose. pub fn for_config(configuration: KafkaIngressConfiguration) -> Self { Self::new(KafkaFactory, configuration) } @@ -120,12 +117,14 @@ where let key_deser_cpy = configuration.key_deserializer.clone(); let payload_deser_cpy = configuration.payload_deserializer.clone(); + let relays = configuration.relays.clone(); let lanes = lanes.take(); + let consumer_task = Box::pin(async move { debug!(key = ?key_deser_cpy, payload = ?payload_deser_cpy, "Attempting to load message deserializers."); let key_deser = key_deser_cpy.load_deserializer().await?; let payload_deser = payload_deser_cpy.load_deserializer().await?; - let selector = MessageSelector::new(key_deser, payload_deser, lanes); + let selector = MessageSelector::new(key_deser, payload_deser, lanes, relays); let state = MessageState::new(consumer, selector, message_to_handler, tx); state.consume_messages(None).await }); @@ -141,12 +140,12 @@ where .. } = self; let mut guard = lanes.borrow_mut(); - match Lanes::try_from(configuration) { + match Lanes::try_from_lane_specs(&configuration.value_lanes, &configuration.map_lanes) { Ok(lanes_from_conf) => { - for lane_spec in &lanes_from_conf.value_lanes { + for lane_spec in lanes_from_conf.value_lanes() { context.open_lane(lane_spec.name(), WarpLaneKind::Value); } - for lane_spec in &lanes_from_conf.map_lanes { + for lane_spec in lanes_from_conf.map_lanes() { context.open_lane(lane_spec.name(), WarpLaneKind::Map); } *guard = lanes_from_conf; @@ -164,8 +163,13 @@ fn message_to_handler<'a>( selector: &'a MessageSelector, message: &'a MessageView<'a>, trigger_tx: trigger::Sender, -) -> Result + Send + 'static, LaneSelectorError> { - selector.handle_message(message, trigger_tx) +) -> Result + Send + 'static, SelectorError> { + let handler_context = HandlerContext::default(); + selector.handle_message(message).map(|handler| { + handler.followed_by(handler_context.effect(move || { + let _ = trigger_tx.trigger(); + })) + }) } // Consumes the Kafka messages and converts them into event handlers. @@ -183,7 +187,7 @@ where &'a MessageSelector, &'a MessageView<'a>, trigger::Sender, - ) -> Result + ) -> Result + Send + 'static, H: EventHandler + Send + 'static, @@ -230,7 +234,7 @@ where let message = consumer.recv().await?; let view = message.view(); let (trigger_tx, trigger_rx) = trigger::trigger(); - // We need to keep a borrow on the receiver in order to be bale to commit it. However, we don't want + // We need to keep a borrow on the receiver in order to be able to commit it. However, we don't want // to do this until after we know the handler has been run. The handler cannot be executed within the // as it requires access to the agent state. Therefore, we send it out via an MPSC channel and // wait for a signal to indicate that it has been handled. @@ -307,129 +311,3 @@ where } } } - -// Information about the lanes of the connector. These are computed from the configuration in the `on_start` handler -// and stored in the lifecycle to be used to start the consumer stream. -#[derive(Debug, Default, Clone)] -struct Lanes { - value_lanes: Vec, - map_lanes: Vec, -} - -impl TryFrom<&KafkaIngressConfiguration> for Lanes { - type Error = InvalidLanes; - - fn try_from(value: &KafkaIngressConfiguration) -> Result { - let KafkaIngressConfiguration { - value_lanes, - map_lanes, - .. - } = value; - Lanes::try_from_lane_specs(value_lanes, map_lanes) - } -} - -impl Lanes { - fn try_from_lane_specs( - value_lanes: &[IngressValueLaneSpec], - map_lanes: &[IngressMapLaneSpec], - ) -> Result { - let value_selectors = value_lanes - .iter() - .map(ValueLaneSelector::try_from) - .collect::, _>>()?; - let map_selectors = map_lanes - .iter() - .map(MapLaneSelector::try_from) - .collect::, _>>()?; - check_selectors(&value_selectors, &map_selectors)?; - Ok(Lanes { - value_lanes: value_selectors, - map_lanes: map_selectors, - }) - } -} - -fn check_selectors( - value_selectors: &[ValueLaneSelector], - map_selectors: &[MapLaneSelector], -) -> Result<(), InvalidLanes> { - let mut names = HashSet::new(); - for value_selector in value_selectors { - let name = value_selector.name(); - if names.contains(name) { - return Err(InvalidLanes::NameCollision(name.to_string())); - } else { - names.insert(name); - } - } - for map_selector in map_selectors { - let name = map_selector.name(); - if names.contains(name) { - return Err(InvalidLanes::NameCollision(name.to_string())); - } else { - names.insert(name); - } - } - Ok(()) -} - -// Uses the information about the lanes of the agent to convert Kafka messages into event handlers that update the lanes. -struct MessageSelector { - key_deserializer: BoxMessageDeserializer, - value_deserializer: BoxMessageDeserializer, - lanes: Lanes, -} - -impl MessageSelector { - pub fn new( - key_deserializer: BoxMessageDeserializer, - value_deserializer: BoxMessageDeserializer, - lanes: Lanes, - ) -> Self { - MessageSelector { - key_deserializer, - value_deserializer, - lanes, - } - } - - fn handle_message<'a>( - &self, - message: &'a MessageView<'a>, - on_done: trigger::Sender, - ) -> Result + Send + 'static, LaneSelectorError> { - let MessageSelector { - key_deserializer, - value_deserializer, - lanes, - } = self; - let Lanes { - value_lanes, - map_lanes, - .. - } = lanes; - trace!(topic = { message.topic() }, "Handling a Kafka message."); - let mut value_lane_handlers = Vec::with_capacity(value_lanes.len()); - let mut map_lane_handlers = Vec::with_capacity(map_lanes.len()); - { - let topic = Value::text(message.topic()); - let mut key = Computed::new(|| key_deserializer.deserialize(message, MessagePart::Key)); - let mut value = - Computed::new(|| value_deserializer.deserialize(message, MessagePart::Payload)); - - for value_lane in value_lanes { - value_lane_handlers.push(value_lane.select_handler(&topic, &mut key, &mut value)?); - } - for map_lane in map_lanes { - map_lane_handlers.push(map_lane.select_handler(&topic, &mut key, &mut value)?); - } - } - let handler_context = ConnHandlerContext::default(); - Ok(Sequentially::new(value_lane_handlers) - .followed_by(Sequentially::new(map_lane_handlers)) - .followed_by(handler_context.effect(move || { - let _ = on_done.trigger(); - }))) - } -} diff --git a/server/swimos_connector_kafka/src/connector/ingress/tests/end_to_end.rs b/server/swimos_connector_kafka/src/connector/ingress/tests/end_to_end.rs index c1aafb5c9..095dfa383 100644 --- a/server/swimos_connector_kafka/src/connector/ingress/tests/end_to_end.rs +++ b/server/swimos_connector_kafka/src/connector/ingress/tests/end_to_end.rs @@ -13,15 +13,16 @@ // limitations under the License. use crate::{ - connector::test_util::create_kafka_props, DataFormat, Endianness, IngressMapLaneSpec, - IngressValueLaneSpec, KafkaIngressConfiguration, KafkaIngressConnector, KafkaLogLevel, + connector::test_util::create_kafka_props, DataFormat, KafkaIngressConfiguration, + KafkaIngressConnector, KafkaLogLevel, }; use futures::{future::join, TryStreamExt}; +use swimos_connector::config::{IngressMapLaneSpec, IngressValueLaneSpec}; +use swimos_connector::deser::Endianness; use swimos_connector::{BaseConnector, ConnectorAgent, IngressConnector}; +use swimos_connector_util::{run_handler, run_handler_with_futures, TestSpawner}; use swimos_utilities::trigger; -use crate::connector::test_util::{run_handler, run_handler_with_futures, TestSpawner}; - fn make_config() -> KafkaIngressConfiguration { KafkaIngressConfiguration { properties: create_kafka_props(), @@ -37,6 +38,7 @@ fn make_config() -> KafkaIngressConfiguration { key_deserializer: DataFormat::Int32(Endianness::BigEndian), payload_deserializer: DataFormat::Json, topics: vec!["cellular-integer-json".to_string()], + relays: Default::default(), } } diff --git a/server/swimos_connector_kafka/src/connector/ingress/tests/integration.rs b/server/swimos_connector_kafka/src/connector/ingress/tests/integration.rs index 9e4f3a433..cbaf8cba7 100644 --- a/server/swimos_connector_kafka/src/connector/ingress/tests/integration.rs +++ b/server/swimos_connector_kafka/src/connector/ingress/tests/integration.rs @@ -18,30 +18,53 @@ use std::{ time::Duration, }; +use crate::{ + config::KafkaLogLevel, + connector::ingress::{message_to_handler, Lanes, MessageSelector, MessageState, MessageTasks}, + error::KafkaConnectorError, + facade::{ConsumerFactory, KafkaConsumer, KafkaMessage}, + DataFormat, KafkaIngressConfiguration, KafkaIngressConnector, +}; use futures::{future::join, TryStreamExt}; use parking_lot::Mutex; use rand::{rngs::ThreadRng, Rng}; use rdkafka::error::KafkaError; use swimos_agent::agent_model::{AgentSpec, ItemDescriptor, ItemFlags}; use swimos_api::agent::WarpLaneKind; +use swimos_connector::config::{IngressMapLaneSpec, IngressValueLaneSpec}; +use swimos_connector::deser::{MessageDeserializer, MessageView, ReconDeserializer}; use swimos_connector::{BaseConnector, ConnectorAgent, IngressConnector, IngressContext}; +use swimos_connector_util::run_handler_with_futures; use swimos_model::{Item, Value}; use swimos_recon::print_recon_compact; use swimos_utilities::trigger; use tokio::sync::mpsc; -use crate::{ - config::KafkaLogLevel, - connector::ingress::{message_to_handler, Lanes, MessageSelector, MessageState, MessageTasks}, - deser::{MessageDeserializer, MessageView, ReconDeserializer}, - error::KafkaConnectorError, - facade::{ConsumerFactory, KafkaConsumer, KafkaMessage}, - DataFormat, IngressMapLaneSpec, IngressValueLaneSpec, KafkaIngressConfiguration, - KafkaIngressConnector, -}; - -use super::setup_agent; -use crate::connector::test_util::run_handler_with_futures; +fn setup_agent() -> (ConnectorAgent, HashMap) { + let agent = ConnectorAgent::default(); + let mut ids = HashMap::new(); + let id1 = agent + .register_dynamic_item( + "key", + ItemDescriptor::WarpLane { + kind: WarpLaneKind::Value, + flags: ItemFlags::TRANSIENT, + }, + ) + .expect("Registration failed."); + let id2 = agent + .register_dynamic_item( + "map", + ItemDescriptor::WarpLane { + kind: WarpLaneKind::Map, + flags: ItemFlags::TRANSIENT, + }, + ) + .expect("Registration failed."); + ids.insert("key".to_string(), id1); + ids.insert("map".to_string(), id2); + (agent, ids) +} fn props() -> HashMap { [("key".to_string(), "value".to_string())] @@ -64,6 +87,7 @@ fn make_config() -> KafkaIngressConfiguration { key_deserializer: DataFormat::Recon, payload_deserializer: DataFormat::Recon, topics: vec!["topic".to_string()], + relays: Default::default(), } } @@ -276,8 +300,12 @@ async fn message_state() { let lanes = Lanes::try_from_lane_specs(&value_specs, &map_specs).expect("Invalid specifications."); - let selector = - MessageSelector::new(ReconDeserializer.boxed(), ReconDeserializer.boxed(), lanes); + let selector = MessageSelector::new( + ReconDeserializer.boxed(), + ReconDeserializer.boxed(), + lanes, + Default::default(), + ); let (tx, mut rx) = mpsc::channel(1); let message_state = MessageState::new(mock_consumer, selector, message_to_handler, tx); @@ -349,8 +377,12 @@ async fn message_tasks_stream() { let lanes = Lanes::try_from_lane_specs(&value_specs, &map_specs).expect("Invalid specifications."); - let selector = - MessageSelector::new(ReconDeserializer.boxed(), ReconDeserializer.boxed(), lanes); + let selector = MessageSelector::new( + ReconDeserializer.boxed(), + ReconDeserializer.boxed(), + lanes, + Default::default(), + ); let (tx, rx) = mpsc::channel(1); let message_state = MessageState::new(mock_consumer, selector, message_to_handler, tx); diff --git a/server/swimos_connector_kafka/src/connector/ingress/tests/mod.rs b/server/swimos_connector_kafka/src/connector/ingress/tests/mod.rs index 94c3624d2..8afd88102 100644 --- a/server/swimos_connector_kafka/src/connector/ingress/tests/mod.rs +++ b/server/swimos_connector_kafka/src/connector/ingress/tests/mod.rs @@ -15,448 +15,3 @@ #[cfg(feature = "json")] mod end_to_end; mod integration; - -use std::{ - collections::{HashMap, HashSet}, - time::Duration, -}; - -use futures::future::join; -use swimos_agent::agent_model::{AgentSpec, ItemDescriptor, ItemFlags, WarpLaneKind}; -use swimos_connector::ConnectorAgent; -use swimos_model::{Item, Value}; -use swimos_recon::print_recon_compact; -use swimos_utilities::trigger; -use tokio::time::timeout; - -use crate::{ - connector::{ - ingress::{InvalidLanes, MessageSelector}, - test_util::{run_handler, run_handler_with_futures, TestSpawner}, - }, - deser::{MessageDeserializer, MessageView, ReconDeserializer}, - error::{DeserializationError, LaneSelectorError}, - selector::{ - BasicSelector, ChainSelector, Deferred, LaneSelector, MapLaneSelector, SlotSelector, - ValueLaneSelector, - }, - IngressMapLaneSpec, IngressValueLaneSpec, -}; - -use super::Lanes; - -#[test] -fn lanes_from_spec() { - let value_lanes = vec![ - IngressValueLaneSpec::new(None, "$key", true), - IngressValueLaneSpec::new(Some("name"), "$payload.field", false), - ]; - let map_lanes = vec![IngressMapLaneSpec::new( - "map", "$key", "$payload", true, false, - )]; - let lanes = - Lanes::try_from_lane_specs(&value_lanes, &map_lanes).expect("Invalid specification."); - - let value_lanes = lanes - .value_lanes - .iter() - .map(|l| l.name()) - .collect::>(); - let map_lanes = lanes.map_lanes.iter().map(|l| l.name()).collect::>(); - - assert_eq!(&value_lanes, &["key", "name"]); - assert_eq!(&map_lanes, &["map"]); -} - -#[test] -fn value_lane_collision() { - let value_lanes = vec![ - IngressValueLaneSpec::new(None, "$key", true), - IngressValueLaneSpec::new(Some("key"), "$payload.field", false), - ]; - let map_lanes = vec![]; - let err = Lanes::try_from_lane_specs(&value_lanes, &map_lanes).expect_err("Should fail."); - assert_eq!(err, InvalidLanes::NameCollision("key".to_string())) -} - -#[test] -fn map_lane_collision() { - let value_lanes = vec![]; - let map_lanes = vec![ - IngressMapLaneSpec::new("map", "$key", "$payload", true, false), - IngressMapLaneSpec::new("map", "$key[0]", "$payload", true, true), - ]; - let err = Lanes::try_from_lane_specs(&value_lanes, &map_lanes).expect_err("Should fail."); - assert_eq!(err, InvalidLanes::NameCollision("map".to_string())) -} - -#[test] -fn value_map_lane_collision() { - let value_lanes = vec![IngressValueLaneSpec::new( - Some("field"), - "$payload.field", - false, - )]; - let map_lanes = vec![IngressMapLaneSpec::new( - "field", "$key", "$payload", true, false, - )]; - let err = Lanes::try_from_lane_specs(&value_lanes, &map_lanes).expect_err("Should fail."); - assert_eq!(err, InvalidLanes::NameCollision("field".to_string())) -} - -const TEST_TIMEOUT: Duration = Duration::from_secs(5); - -fn setup_agent() -> (ConnectorAgent, HashMap) { - let agent = ConnectorAgent::default(); - let mut ids = HashMap::new(); - let id1 = agent - .register_dynamic_item( - "key", - ItemDescriptor::WarpLane { - kind: WarpLaneKind::Value, - flags: ItemFlags::TRANSIENT, - }, - ) - .expect("Registration failed."); - let id2 = agent - .register_dynamic_item( - "map", - ItemDescriptor::WarpLane { - kind: WarpLaneKind::Map, - flags: ItemFlags::TRANSIENT, - }, - ) - .expect("Registration failed."); - ids.insert("key".to_string(), id1); - ids.insert("map".to_string(), id2); - (agent, ids) -} - -struct TestDeferred { - value: Value, -} - -impl From for TestDeferred { - fn from(value: Value) -> Self { - TestDeferred { value } - } -} - -impl Deferred for TestDeferred { - fn get(&mut self) -> Result<&Value, DeserializationError> { - Ok(&self.value) - } -} - -fn make_key_value(key: impl Into, value: impl Into) -> Value { - Value::record(vec![Item::slot("key", key), Item::slot("value", value)]) -} - -fn make_key_only(key: impl Into) -> Value { - Value::record(vec![Item::slot("key", key)]) -} - -#[test] -fn value_lane_selector_handler() { - let (mut agent, ids) = setup_agent(); - - let selector = ValueLaneSelector::new( - "key".to_string(), - LaneSelector::Key(ChainSelector::default()), - true, - ); - - let topic = Value::text("topic_name"); - let mut key = TestDeferred::from(Value::from(3)); - let mut value = TestDeferred::from(make_key_value("a", 7)); - - let handler = selector - .select_handler(&topic, &mut key, &mut value) - .expect("Selector failed."); - let spawner = TestSpawner::default(); - let modified = run_handler(&agent, &spawner, handler); - - assert_eq!(modified, [ids["key"]].into_iter().collect::>()); - let lane = agent.value_lane("key").expect("Lane missing."); - lane.read(|v| assert_eq!(v, &Value::from(3))); -} - -#[test] -fn value_lane_selector_handler_optional_field() { - let (agent, _) = setup_agent(); - - let selector = LaneSelector::Payload(ChainSelector::new(vec![BasicSelector::Slot( - SlotSelector::for_field("other"), - )])); - - let selector = ValueLaneSelector::new("other".to_string(), selector, false); - - let topic = Value::text("topic_name"); - let mut key = TestDeferred::from(Value::from(3)); - let mut value = TestDeferred::from(make_key_value("a", 7)); - - let handler = selector - .select_handler(&topic, &mut key, &mut value) - .expect("Selector failed."); - let spawner = TestSpawner::default(); - let modified = run_handler(&agent, &spawner, handler); - - assert!(modified.is_empty()); -} - -#[test] -fn value_lane_selector_handler_missing_field() { - let selector = LaneSelector::Payload(ChainSelector::new(vec![BasicSelector::Slot( - SlotSelector::for_field("other"), - )])); - - let selector = ValueLaneSelector::new("other".to_string(), selector, true); - - let topic = Value::text("topic_name"); - let mut key = TestDeferred::from(Value::from(3)); - let mut value = TestDeferred::from(make_key_value("a", 7)); - - let error = selector - .select_handler(&topic, &mut key, &mut value) - .expect_err("Should fail."); - assert!(matches!(error, LaneSelectorError::MissingRequiredLane(name) if &name == "other")); -} - -#[test] -fn map_lane_selector_handler() { - let (mut agent, ids) = setup_agent(); - - let key = LaneSelector::Payload(ChainSelector::new(vec![BasicSelector::Slot( - SlotSelector::for_field("key"), - )])); - let value = LaneSelector::Payload(ChainSelector::new(vec![BasicSelector::Slot( - SlotSelector::for_field("value"), - )])); - - let selector = MapLaneSelector::new("map".to_string(), key, value, true, false); - - let topic = Value::text("topic_name"); - let mut key = TestDeferred::from(Value::from(3)); - let mut value = TestDeferred::from(make_key_value("a", 7)); - - let handler = selector - .select_handler(&topic, &mut key, &mut value) - .expect("Selector failed."); - let spawner = TestSpawner::default(); - let modified = run_handler(&agent, &spawner, handler); - - assert_eq!(modified, [ids["map"]].into_iter().collect::>()); - let lane = agent.map_lane("map").expect("Lane missing."); - lane.get_map(|m| { - let expected = [(Value::text("a"), Value::from(7))] - .into_iter() - .collect::>(); - assert_eq!(m, &expected); - }); -} - -#[test] -fn map_lane_selector_handler_optional_field() { - let (agent, _) = setup_agent(); - - let key = LaneSelector::Payload(ChainSelector::new(vec![BasicSelector::Slot( - SlotSelector::for_field("key"), - )])); - let value = LaneSelector::Payload(ChainSelector::new(vec![BasicSelector::Slot( - SlotSelector::for_field("value"), - )])); - - let selector = MapLaneSelector::new("map".to_string(), key, value, false, false); - - let topic = Value::text("topic_name"); - let mut key = TestDeferred::from(Value::from(3)); - let mut value = TestDeferred::from(Value::Extant); - - let handler = selector - .select_handler(&topic, &mut key, &mut value) - .expect("Selector failed."); - let spawner = TestSpawner::default(); - let modified = run_handler(&agent, &spawner, handler); - - assert!(modified.is_empty()); -} - -#[test] -fn map_lane_selector_handler_missing_field() { - let key = LaneSelector::Payload(ChainSelector::new(vec![BasicSelector::Slot( - SlotSelector::for_field("key"), - )])); - let value = LaneSelector::Payload(ChainSelector::new(vec![BasicSelector::Slot( - SlotSelector::for_field("value"), - )])); - - let selector = MapLaneSelector::new("map".to_string(), key, value, true, false); - - let topic = Value::text("topic_name"); - let mut key = TestDeferred::from(Value::from(3)); - let mut value = TestDeferred::from(Value::Extant); - - let error = selector - .select_handler(&topic, &mut key, &mut value) - .expect_err("Should fail."); - assert!(matches!(error, LaneSelectorError::MissingRequiredLane(name) if &name == "map")); -} - -#[test] -fn map_lane_selector_remove() { - let (mut agent, ids) = setup_agent(); - - let key = LaneSelector::Payload(ChainSelector::new(vec![BasicSelector::Slot( - SlotSelector::for_field("key"), - )])); - let value = LaneSelector::Payload(ChainSelector::new(vec![BasicSelector::Slot( - SlotSelector::for_field("value"), - )])); - - let selector = MapLaneSelector::new("map".to_string(), key, value, true, true); - - let topic = Value::text("topic_name"); - let mut key = TestDeferred::from(Value::from(3)); - let mut value = TestDeferred::from(make_key_value("a", 7)); - - let update_handler = selector - .select_handler(&topic, &mut key, &mut value) - .expect("Selector failed."); - let spawner = TestSpawner::default(); - let modified = run_handler(&agent, &spawner, update_handler); - - assert_eq!(modified, [ids["map"]].into_iter().collect::>()); - let lane = agent.map_lane("map").expect("Lane missing."); - lane.get_map(|m| { - let expected = [(Value::text("a"), Value::from(7))] - .into_iter() - .collect::>(); - assert_eq!(m, &expected); - }); - - drop(lane); - - let mut value2 = TestDeferred::from(make_key_only("a")); - let remove_handler = selector - .select_handler(&topic, &mut key, &mut value2) - .expect("Selector failed."); - let modified = run_handler(&agent, &spawner, remove_handler); - - assert_eq!(modified, [ids["map"]].into_iter().collect::>()); - let lane = agent.map_lane("map").expect("Lane missing."); - lane.get_map(|m| { - assert!(m.is_empty()); - }); -} - -#[tokio::test] -async fn handle_message() { - let value_specs = vec![IngressValueLaneSpec::new(None, "$key", true)]; - let map_specs = vec![IngressMapLaneSpec::new( - "map", - "$payload.key", - "$payload.value", - true, - true, - )]; - let lanes = - Lanes::try_from_lane_specs(&value_specs, &map_specs).expect("Invalid specifications."); - - let (agent, ids) = setup_agent(); - - let selector = - MessageSelector::new(ReconDeserializer.boxed(), ReconDeserializer.boxed(), lanes); - - let key = Value::from(3); - let payload = make_key_value("ab", 67); - let key_str = format!("{}", print_recon_compact(&key)); - let payload_str = format!("{}", print_recon_compact(&payload)); - - let message = MessageView { - topic: "topic_name", - key: key_str.as_bytes(), - payload: payload_str.as_bytes(), - }; - - let (tx, rx) = trigger::trigger(); - - let handler = selector - .handle_message(&message, tx) - .expect("Selector failed."); - - let handler_task = run_handler_with_futures(&agent, handler); - - let (modified, done_result) = timeout(TEST_TIMEOUT, join(handler_task, rx)) - .await - .expect("Test timed out."); - - assert!(done_result.is_ok()); - assert_eq!(modified, ids.values().copied().collect::>()); -} - -#[tokio::test] -async fn handle_message_missing_field() { - let value_specs = vec![IngressValueLaneSpec::new(None, "$key", true)]; - let map_specs = vec![IngressMapLaneSpec::new( - "map", - "$payload.key", - "$payload.value", - true, - true, - )]; - let lanes = - Lanes::try_from_lane_specs(&value_specs, &map_specs).expect("Invalid specifications."); - - let selector = - MessageSelector::new(ReconDeserializer.boxed(), ReconDeserializer.boxed(), lanes); - - let key = Value::from(3); - let payload = Value::text("word"); - let key_str = format!("{}", print_recon_compact(&key)); - let payload_str = format!("{}", print_recon_compact(&payload)); - - let message = MessageView { - topic: "topic_name", - key: key_str.as_bytes(), - payload: payload_str.as_bytes(), - }; - - let (tx, _rx) = trigger::trigger(); - - let result = selector.handle_message(&message, tx); - assert!(matches!(result, Err(LaneSelectorError::MissingRequiredLane(name)) if name == "map")); -} - -#[tokio::test] -async fn handle_message_bad_data() { - let value_specs = vec![IngressValueLaneSpec::new(None, "$key", true)]; - let map_specs = vec![IngressMapLaneSpec::new( - "map", - "$payload.key", - "$payload.value", - true, - true, - )]; - let lanes = - Lanes::try_from_lane_specs(&value_specs, &map_specs).expect("Invalid specifications."); - - let selector = - MessageSelector::new(ReconDeserializer.boxed(), ReconDeserializer.boxed(), lanes); - - let key = Value::from(3); - let key_str = format!("{}", print_recon_compact(&key)); - - let message = MessageView { - topic: "topic_name", - key: key_str.as_bytes(), - payload: b"^*$&@*@", - }; - - let (tx, _rx) = trigger::trigger(); - - let result = selector.handle_message(&message, tx); - assert!(matches!( - result, - Err(LaneSelectorError::DeserializationFailed(_)) - )); -} diff --git a/server/swimos_connector_kafka/src/connector/test_util.rs b/server/swimos_connector_kafka/src/connector/test_util.rs index 69bf61ef9..48a17e02f 100644 --- a/server/swimos_connector_kafka/src/connector/test_util.rs +++ b/server/swimos_connector_kafka/src/connector/test_util.rs @@ -12,254 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{ - cell::RefCell, - collections::{HashMap, HashSet}, -}; - -use bytes::BytesMut; -use futures::{stream::FuturesUnordered, StreamExt}; -use swimos_agent::{ - agent_model::{ - downlink::BoxDownlinkChannelFactory, AgentSpec, ItemDescriptor, ItemFlags, WarpLaneKind, - }, - event_handler::{ - ActionContext, DownlinkSpawnOnDone, EventHandler, HandlerFuture, LaneSpawnOnDone, - LaneSpawner, LinkSpawner, Spawner, StepResult, - }, - AgentMetadata, -}; -use swimos_api::{ - address::Address, - agent::AgentConfig, - error::{CommanderRegistrationError, DynamicRegistrationError}, -}; -use swimos_connector::ConnectorAgent; -use swimos_model::Text; -use swimos_utilities::routing::RouteUri; - -pub struct LaneRequest { - name: String, - is_map: bool, - on_done: LaneSpawnOnDone, -} - -impl std::fmt::Debug for LaneRequest { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LaneRequest") - .field("name", &self.name) - .field("is_map", &self.is_map) - .field("on_done", &"...") - .finish() - } -} - -#[derive(Default, Debug)] -pub struct TestSpawner { - allow_downlinks: bool, - downlinks: RefCell>, - suspended: FuturesUnordered>, - lane_requests: RefCell>, -} - -impl TestSpawner { - pub fn with_downlinks() -> TestSpawner { - TestSpawner { - allow_downlinks: true, - ..Default::default() - } - } - - pub fn take_downlinks(&self) -> Vec { - let mut guard = self.downlinks.borrow_mut(); - std::mem::take(&mut *guard) - } -} - -impl Spawner for TestSpawner { - fn spawn_suspend(&self, fut: HandlerFuture) { - self.suspended.push(fut); - } - - fn schedule_timer(&self, _at: tokio::time::Instant, _id: u64) { - panic!("Unexpected timer"); - } -} - -pub struct DownlinkRequest { - pub path: Address, - _make_channel: BoxDownlinkChannelFactory, - _on_done: DownlinkSpawnOnDone, -} - -impl std::fmt::Debug for DownlinkRequest { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("DownlinkRequest") - .field("path", &self.path) - .field("make_channel", &"...") - .field("_on_done", &"...") - .finish() - } -} - -impl LinkSpawner for TestSpawner { - fn spawn_downlink( - &self, - path: Address, - _make_channel: BoxDownlinkChannelFactory, - _on_done: DownlinkSpawnOnDone, - ) { - if self.allow_downlinks { - self.downlinks.borrow_mut().push(DownlinkRequest { - path: path.into(), - _make_channel, - _on_done, - }) - } else { - panic!("Opening downlinks not supported."); - } - } - - fn register_commander(&self, _path: Address) -> Result { - panic!("Registering commanders not supported."); - } -} - -impl LaneSpawner for TestSpawner { - fn spawn_warp_lane( - &self, - name: &str, - kind: WarpLaneKind, - on_done: LaneSpawnOnDone, - ) -> Result<(), DynamicRegistrationError> { - let is_map = match kind { - WarpLaneKind::Map => true, - WarpLaneKind::Value => false, - _ => panic!("Unexpected lane kind: {}", kind), - }; - self.lane_requests.borrow_mut().push(LaneRequest { - name: name.to_string(), - is_map, - on_done, - }); - Ok(()) - } -} - -const CONFIG: AgentConfig = AgentConfig::DEFAULT; -const NODE_URI: &str = "/node"; - -fn make_uri() -> RouteUri { - RouteUri::try_from(NODE_URI).expect("Bad URI.") -} - -fn make_meta<'a>( - uri: &'a RouteUri, - route_params: &'a HashMap, -) -> AgentMetadata<'a> { - AgentMetadata::new(uri, route_params, &CONFIG) -} - -pub async fn run_handler_with_futures_dl>( - agent: &ConnectorAgent, - handler: H, -) -> (HashSet, Vec) { - let mut spawner = TestSpawner::with_downlinks(); - let modified = run_handler_with_futures_inner(agent, handler, &mut spawner).await; - (modified, spawner.take_downlinks()) -} - -pub async fn run_handler_with_futures>( - agent: &ConnectorAgent, - handler: H, -) -> HashSet { - run_handler_with_futures_inner(agent, handler, &mut TestSpawner::default()).await -} - -async fn run_handler_with_futures_inner>( - agent: &ConnectorAgent, - handler: H, - spawner: &mut TestSpawner, -) -> HashSet { - let mut modified = run_handler(agent, spawner, handler); - let mut handlers = vec![]; - let reg = move |req: LaneRequest| { - let LaneRequest { - name, - is_map, - on_done, - } = req; - let kind = if is_map { - WarpLaneKind::Map - } else { - WarpLaneKind::Value - }; - let descriptor = ItemDescriptor::WarpLane { - kind, - flags: ItemFlags::TRANSIENT, - }; - let result = agent.register_dynamic_item(&name, descriptor); - on_done(result.map_err(Into::into)) - }; - for request in std::mem::take::>(spawner.lane_requests.borrow_mut().as_mut()) { - handlers.push(reg(request)); - } - - while !(handlers.is_empty() && spawner.suspended.is_empty()) { - let m = if let Some(h) = handlers.pop() { - run_handler(agent, spawner, h) - } else { - let h = spawner.suspended.next().await.expect("No handler."); - run_handler(agent, spawner, h) - }; - modified.extend(m); - for request in - std::mem::take::>(spawner.lane_requests.borrow_mut().as_mut()) - { - handlers.push(reg(request)); - } - } - modified -} - -pub fn run_handler>( - agent: &ConnectorAgent, - spawner: &TestSpawner, - mut handler: H, -) -> HashSet { - let route_params = HashMap::new(); - let uri = make_uri(); - let meta = make_meta(&uri, &route_params); - let mut join_lane_init = HashMap::new(); - let mut command_buffer = BytesMut::new(); - - let mut action_context = ActionContext::new( - spawner, - spawner, - spawner, - &mut join_lane_init, - &mut command_buffer, - ); - - let mut modified = HashSet::new(); - - loop { - match handler.step(&mut action_context, meta, agent) { - StepResult::Continue { modified_item } => { - if let Some(m) = modified_item { - modified.insert(m.id()); - } - } - StepResult::Fail(err) => panic!("Handler Failed: {}", err), - StepResult::Complete { modified_item, .. } => { - if let Some(m) = modified_item { - modified.insert(m.id()); - } - break modified; - } - } - } -} +use std::collections::HashMap; #[cfg(feature = "json")] pub fn create_kafka_props() -> HashMap { diff --git a/server/swimos_connector_kafka/src/error.rs b/server/swimos_connector_kafka/src/error.rs index 46665f3cd..6529e1c4e 100644 --- a/server/swimos_connector_kafka/src/error.rs +++ b/server/swimos_connector_kafka/src/error.rs @@ -12,42 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::num::ParseIntError; - use rdkafka::error::KafkaError; use swimos_api::address::Address; -use swimos_model::{Value, ValueKind}; +use swimos_connector::selector::{BadSelector, InvalidLanes, SelectorError}; +use swimos_connector::{LoadError, SerializationError}; use thiserror::Error; -/// An error type that boxes any type of error that could be returned by a message deserializer. -#[derive(Debug, Error)] -#[error(transparent)] -pub struct DeserializationError(Box); - -impl DeserializationError { - pub fn new(error: E) -> Self - where - E: std::error::Error + Send + 'static, - { - DeserializationError(Box::new(error)) - } -} - -/// An error type that can be produced when attempting to load a serializer or deserializer. -#[derive(Debug, Error)] -pub enum LoadError { - /// Attempting to read a required resource (for example, a file) failed. - #[error(transparent)] - Io(#[from] std::io::Error), - /// A required resource was invalid. - #[error(transparent)] - InvalidDescriptor(#[from] Box), - #[error("The configuration provided for the serializer or deserializer is invalid: {0}")] - InvalidConfiguration(String), - #[error("Loading of the configuration was cancelled.")] - Cancelled, -} - /// Errors that can be produced by the Kafka connector. #[derive(Debug, Error)] pub enum KafkaConnectorError { @@ -62,77 +32,13 @@ pub enum KafkaConnectorError { Kafka(#[from] KafkaError), /// Attempting to select the required components of a Kafka message failed. #[error(transparent)] - Lane(#[from] LaneSelectorError), + Lane(#[from] SelectorError), #[error("A message was not handled properly and so could not be committed.")] MessageNotHandled, } -/// Errors that can be produced attempting to select components of a Kafka Message. -#[derive(Debug, Error)] -pub enum LaneSelectorError { - /// A selector failed to provide a value for a required lane. - #[error("The lane '{0}' is required but did not occur in a message.")] - MissingRequiredLane(String), - /// Deserializing a component of a Kafka message failed. - #[error("Deserializing the content of a Kafka message failed: {0}")] - DeserializationFailed(#[from] DeserializationError), -} - -/// Error type for an invalid lane selector specification. -#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] -pub enum InvalidLaneSpec { - /// The string describing the selector was invalid. - #[error(transparent)] - Selector(#[from] BadSelector), - /// The lane name could not be inferred from the selector and was not provided explicitly. - #[error("No name provided and it cannot be inferred from the selector.")] - NameCannotBeInferred, -} - -/// Error type produced for invalid lane descriptors. -#[derive(Clone, Debug, Error, PartialEq, Eq)] -pub enum InvalidLanes { - /// The specification of a lane as not valid - #[error(transparent)] - Spec(#[from] InvalidLaneSpec), - /// There are lane descriptors with the same name. - #[error("The lane name {0} occurs more than once.")] - NameCollision(String), -} - -/// Error type for an invalid selector descriptor. -#[derive(Clone, Copy, Error, Debug, PartialEq, Eq)] -pub enum BadSelector { - /// An empty string does not describe a valid selector. - #[error("Selector strings cannot be empty.")] - EmptySelector, - /// A selector component pay not be empty. - #[error("Selector components cannot be empty.")] - EmptyComponent, - /// The root of a selector must be a valid component of a Kafka message. - #[error("Invalid root selector (must be one of '$key' or '$payload' with an optional index or '$topic').")] - InvalidRoot, - /// A component of the descriptor did not describe a valid selector. - #[error( - "Invalid component selector (must be an attribute or slot name with an optional index)." - )] - InvalidComponent, - /// The index for an index selector was too large for usize. - #[error("An index specified was not a valid usize.")] - IndexOutOfRange, - /// The topic root cannot have any other components. - #[error("The topic does not have components.")] - TopicWithComponent, -} - -impl From for BadSelector { - fn from(_value: ParseIntError) -> Self { - BadSelector::IndexOutOfRange - } -} - /// Error type for an invalid egress extractor specification. -#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] +#[derive(Clone, Debug, Error, PartialEq, Eq)] pub enum InvalidExtractor { /// A string describing a selector was invalid. #[error(transparent)] @@ -160,23 +66,6 @@ pub enum InvalidExtractors { #[error("Connector agent initialized twice.")] pub struct DoubleInitialization; -/// An error type that boxes any type of error that could be returned by a message serializer. -#[derive(Debug, Error)] -pub enum SerializationError { - /// The value is not supported by the serialization format. - #[error("The serializations scheme does not support values of kind: {0}")] - InvalidKind(ValueKind), - /// An integer in a value was out of range for the serialization format. - #[error("Integer value {0} out of range for the serialization scheme.")] - IntegerOutOfRange(Value), - /// An floating point number in a value was out of range for the serialization format. - #[error("Float value {0} out of range for the serialization scheme.")] - FloatOutOfRange(f64), - /// The serializer failed with an error. - #[error("A value serializer failed: {0}")] - SerializerFailed(Box), -} - #[derive(Debug, Error)] pub enum KafkaSenderError { #[error(transparent)] diff --git a/server/swimos_connector_kafka/src/facade.rs b/server/swimos_connector_kafka/src/facade.rs index a54feaed4..a84f0ad07 100644 --- a/server/swimos_connector_kafka/src/facade.rs +++ b/server/swimos_connector_kafka/src/facade.rs @@ -18,6 +18,7 @@ use std::{ task::{Context, Poll}, }; +use crate::config::KafkaLogLevel; use futures::{Future, FutureExt}; use rdkafka::{ config::RDKafkaLogLevel, @@ -28,10 +29,9 @@ use rdkafka::{ types::RDKafkaErrorCode, ClientConfig, ClientContext, Message, Statistics, TopicPartitionList, }; +use swimos_connector::deser::MessageView; use tracing::{debug, error, info, warn}; -use crate::{config::KafkaLogLevel, deser::MessageView}; - pub trait KafkaMessage { fn view(&self) -> MessageView<'_>; } diff --git a/server/swimos_connector_kafka/src/lib.rs b/server/swimos_connector_kafka/src/lib.rs index 48762ed2b..7868dd25e 100644 --- a/server/swimos_connector_kafka/src/lib.rs +++ b/server/swimos_connector_kafka/src/lib.rs @@ -14,21 +14,16 @@ mod config; mod connector; -mod deser; mod error; mod facade; mod selector; -mod ser; pub use config::{ DataFormat, DownlinkAddress, EgressDownlinkSpec, EgressLaneSpec, ExtractionSpec, - IngressMapLaneSpec, IngressValueLaneSpec, KafkaEgressConfiguration, KafkaIngressConfiguration, - KafkaLogLevel, TopicSpecifier, + KafkaEgressConfiguration, KafkaIngressConfiguration, KafkaLogLevel, TopicSpecifier, }; pub use connector::{KafkaEgressConnector, KafkaIngressConnector}; -pub use deser::Endianness; pub use error::{ - BadSelector, DeserializationError, DoubleInitialization, InvalidExtractor, InvalidExtractors, - InvalidLaneSpec, InvalidLanes, KafkaConnectorError, KafkaSenderError, LaneSelectorError, - LoadError, SerializationError, + DoubleInitialization, InvalidExtractor, InvalidExtractors, KafkaConnectorError, + KafkaSenderError, }; diff --git a/server/swimos_connector_kafka/src/selector/message/mod.rs b/server/swimos_connector_kafka/src/selector/message/mod.rs index fb9accf4d..92e704e09 100644 --- a/server/swimos_connector_kafka/src/selector/message/mod.rs +++ b/server/swimos_connector_kafka/src/selector/message/mod.rs @@ -18,6 +18,9 @@ use std::{ }; use swimos_api::address::Address; +use swimos_connector::selector::{ + BadSelector, ChainSelector, RawSelectorDescriptor, SelectorComponent, ValueSelector, +}; use swimos_model::Value; use crate::{ @@ -25,12 +28,7 @@ use crate::{ EgressDownlinkSpec, EgressLaneSpec, ExtractionSpec, KafkaEgressConfiguration, TopicSpecifier, }, - selector::make_chain_selector, - BadSelector, InvalidExtractor, InvalidExtractors, -}; - -use super::{ - parse_raw_selector, ChainSelector, RawSelectorDescriptor, Selector, SelectorComponent, + InvalidExtractor, InvalidExtractors, }; #[cfg(test)] @@ -96,8 +94,8 @@ impl FieldSelector { pub fn select<'a>(&self, key: Option<&'a Value>, value: &'a Value) -> Option<&'a Value> { let FieldSelector { part, selector } = self; match part { - KeyOrValue::Key => key.and_then(|k| selector.select(k)), - KeyOrValue::Value => selector.select(value), + KeyOrValue::Key => key.and_then(|k| selector.select_value(k)), + KeyOrValue::Value => selector.select_value(value), } } } @@ -138,7 +136,7 @@ impl<'a> From> for FieldSelector { index, components, } = value; - FieldSelector::new(part, make_chain_selector(index, &components)) + FieldSelector::new(part, ChainSelector::new(index, &components)) } } @@ -196,7 +194,7 @@ impl<'a> TryFrom> for FieldSelectorSpec<'a> { /// Attempt to parse a field selector from a string. fn parse_field_selector(descriptor: &str) -> Result, BadSelector> { - parse_raw_selector(descriptor)?.try_into() + RawSelectorDescriptor::try_from(descriptor)?.try_into() } impl MessageSelector { diff --git a/server/swimos_connector_kafka/src/selector/message/tests.rs b/server/swimos_connector_kafka/src/selector/message/tests.rs index 83f4304ae..97631984c 100644 --- a/server/swimos_connector_kafka/src/selector/message/tests.rs +++ b/server/swimos_connector_kafka/src/selector/message/tests.rs @@ -15,15 +15,16 @@ use std::collections::HashMap; use swimos_api::address::Address; +use swimos_connector::deser::Endianness; use swimos_model::{Item, Value}; use crate::{ config::{EgressDownlinkSpec, EgressLaneSpec, KafkaEgressConfiguration, TopicSpecifier}, - selector::{BasicSelector, ChainSelector, SlotSelector}, - DataFormat, DownlinkAddress, Endianness, ExtractionSpec, InvalidExtractors, KafkaLogLevel, + DataFormat, DownlinkAddress, ExtractionSpec, InvalidExtractors, KafkaLogLevel, }; use super::{FieldSelector, KeyOrValue, MessageSelector, MessageSelectors, TopicSelector}; +use swimos_connector::selector::{BasicSelector, ChainSelector, SlotSelector}; const FIXED_TOPIC: &str = "fixed"; const OTHER_TOPIC: &str = "other"; @@ -108,7 +109,7 @@ fn expected_extractors() -> MessageSelectors { None, Some(FieldSelector::new( KeyOrValue::Value, - ChainSelector::new(vec![BasicSelector::Slot(SlotSelector::for_field("field"))]), + ChainSelector::from(vec![BasicSelector::Slot(SlotSelector::for_field("field"))]), )), ), )] @@ -133,11 +134,11 @@ fn expected_extractors() -> MessageSelectors { TopicSelector::Fixed(FIXED_TOPIC.to_string()), Some(FieldSelector::new( KeyOrValue::Value, - ChainSelector::new(vec![BasicSelector::Slot(SlotSelector::for_field("key"))]), + ChainSelector::from(vec![BasicSelector::Slot(SlotSelector::for_field("key"))]), )), Some(FieldSelector::new( KeyOrValue::Value, - ChainSelector::new(vec![BasicSelector::Slot(SlotSelector::for_field( + ChainSelector::from(vec![BasicSelector::Slot(SlotSelector::for_field( "payload", ))]), )), @@ -201,7 +202,7 @@ fn field_selector_key() { fn field_selector_value() { let selector = FieldSelector::new( KeyOrValue::Value, - ChainSelector::new(vec![BasicSelector::Slot(SlotSelector::for_field( + ChainSelector::from(vec![BasicSelector::Slot(SlotSelector::for_field( "record_payload", ))]), ); @@ -243,7 +244,7 @@ fn topic_selector_key() { fn topic_selector_value() { let selector = TopicSelector::Selector(FieldSelector::new( KeyOrValue::Value, - ChainSelector::new(vec![BasicSelector::Slot(SlotSelector::for_field( + ChainSelector::from(vec![BasicSelector::Slot(SlotSelector::for_field( "record_topic", ))]), )); @@ -264,7 +265,7 @@ fn message_selector() { )), Some(FieldSelector::new( KeyOrValue::Value, - ChainSelector::new(vec![BasicSelector::Slot(SlotSelector::for_field( + ChainSelector::from(vec![BasicSelector::Slot(SlotSelector::for_field( "record_payload", ))]), )), @@ -288,7 +289,7 @@ fn message_selector_no_key_selector() { None, Some(FieldSelector::new( KeyOrValue::Value, - ChainSelector::new(vec![BasicSelector::Slot(SlotSelector::for_field( + ChainSelector::from(vec![BasicSelector::Slot(SlotSelector::for_field( "record_payload", ))]), )), diff --git a/server/swimos_connector_kafka/src/selector/mod.rs b/server/swimos_connector_kafka/src/selector/mod.rs index 45e187d23..1401287ad 100644 --- a/server/swimos_connector_kafka/src/selector/mod.rs +++ b/server/swimos_connector_kafka/src/selector/mod.rs @@ -12,712 +12,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod message; -#[cfg(test)] -mod tests; - -use std::{fmt::Debug, sync::OnceLock}; - -use frunk::Coprod; -use regex::Regex; -use swimos_agent::event_handler::{Discard, HandlerActionExt}; -use swimos_connector::{ConnectorAgent, MapLaneSelectorFn, ValueLaneSelectorFn}; -use swimos_model::{Attr, Item, Text, Value}; -use tracing::{error, trace}; - -use crate::{ - config::{IngressMapLaneSpec, IngressValueLaneSpec}, - deser::MessagePart, - error::{DeserializationError, LaneSelectorError}, - BadSelector, InvalidLaneSpec, -}; -use swimos_agent::lanes::{MapLaneSelectRemove, MapLaneSelectUpdate, ValueLaneSelectSet}; - -pub use message::{MessageSelector, MessageSelectors}; - -/// Enumeration of the components of a Kafka message. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum MessageField { - Key, - Payload, - Topic, -} - -impl From for MessageField { - fn from(value: MessagePart) -> Self { - match value { - MessagePart::Key => MessageField::Key, - MessagePart::Payload => MessageField::Payload, - } - } -} - -/// A lazy loader for a component of a Kafka messages. This ensures that deserializers are only run if a selector -/// refers to a component. -pub trait Deferred { - /// Get the deserialized component (computing it on the first call). - fn get(&mut self) -> Result<&Value, DeserializationError>; -} - -/// A selector attempts to choose some sub-component of a [`Value`], matching against a pattern, returning -/// nothing if the pattern does not match. -pub trait Selector: Debug { - /// Attempt to select some sub-component of the provided [`Value`]. - fn select<'a>(&self, value: &'a Value) -> Option<&'a Value>; -} - -/// Canonical implementation of [`Deferred`]. -pub struct Computed { - inner: Option, - f: F, -} - -impl Computed -where - F: Fn() -> Result, -{ - pub fn new(f: F) -> Self { - Computed { inner: None, f } - } -} - -impl Deferred for Computed -where - F: Fn() -> Result, -{ - fn get(&mut self) -> Result<&Value, DeserializationError> { - let Computed { inner, f } = self; - if let Some(v) = inner { - Ok(v) - } else { - Ok(inner.insert(f()?)) - } - } -} - -/// A lane selector attempts to extract a value from a Kafka message to use as a new value for a value lane -/// or an update for a map lane. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum LaneSelector { - /// Select the topic from the message. - Topic, - /// Use a selector to select a sub-component of the key of the message. - Key(ChainSelector), - /// Use a selector to select a sub-component of the payload of the message. - Payload(ChainSelector), -} - -impl<'a> From> for LaneSelector { - fn from(value: SelectorDescriptor<'a>) -> Self { - match (value.field(), value.selector()) { - (MessageField::Key, Some(selector)) => LaneSelector::Key(selector), - (MessageField::Payload, Some(selector)) => LaneSelector::Payload(selector), - _ => LaneSelector::Topic, - } - } -} - -impl LaneSelector { - /// Attempt to select a sub-component from a Kafka message. - /// - /// # Arguments - /// * `topic` - The topic of the message. - /// * `key` - Lazily deserialized message key. - /// * `payload` - Lazily deserialized message payload. - pub fn select<'a, K, V>( - &self, - topic: &'a Value, - key: &'a mut K, - payload: &'a mut V, - ) -> Result, DeserializationError> - where - K: Deferred + 'a, - V: Deferred + 'a, - { - Ok(match self { - LaneSelector::Topic => Some(topic), - LaneSelector::Key(selector) => selector.select(key.get()?), - LaneSelector::Payload(selector) => selector.select(payload.get()?), - }) - } -} - -/// Trivial selector that chooses the entire input value. -#[derive(Debug, Clone, Copy, Default)] -pub struct IdentitySelector; - -impl Selector for IdentitySelector { - fn select<'a>(&self, value: &'a Value) -> Option<&'a Value> { - Some(value) - } -} - -/// A selector that chooses the value of a named attribute if the value is a record and that attribute exists. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct AttrSelector { - select_name: String, -} - -impl AttrSelector { - /// # Arguments - /// * `name` - The name of the attribute. - fn new(name: String) -> Self { - AttrSelector { select_name: name } - } -} - -impl Selector for AttrSelector { - fn select<'a>(&self, value: &'a Value) -> Option<&'a Value> { - let AttrSelector { select_name } = self; - match value { - Value::Record(attrs, _) => attrs.iter().find_map(|Attr { name, value }: &Attr| { - if name.as_str() == select_name.as_str() { - Some(value) - } else { - None - } - }), - _ => None, - } - } -} - -/// A selector that chooses the value of a slot if the value is a record and that slot exists. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SlotSelector { - select_key: Value, -} - -impl SlotSelector { - /// Construct a slot selector for a named field. - /// - /// # Arguments - /// * `name` - The name of the field. - pub fn for_field(name: impl Into) -> Self { - SlotSelector { - select_key: Value::text(name), - } - } -} - -impl Selector for SlotSelector { - fn select<'a>(&self, value: &'a Value) -> Option<&'a Value> { - let SlotSelector { select_key } = self; - match value { - Value::Record(_, items) => items.iter().find_map(|item: &Item| match item { - Item::Slot(key, value) if key == select_key => Some(value), - _ => None, - }), - _ => None, - } - } -} - -/// A selector that chooses an item by index if the value is a record and has a sufficient number of items. If -/// the selected item is a slot, its value is selected. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct IndexSelector { - index: usize, -} - -impl IndexSelector { - /// # Arguments - /// * `index` - The index in the record to select. - pub fn new(index: usize) -> Self { - IndexSelector { index } - } -} - -impl Selector for IndexSelector { - fn select<'a>(&self, value: &'a Value) -> Option<&'a Value> { - let IndexSelector { index } = self; - match value { - Value::Record(_, items) => items.get(*index).map(|item| match item { - Item::ValueItem(v) => v, - Item::Slot(_, v) => v, - }), - _ => None, - } - } -} - -/// One of an [`AttrSelector`], [`SlotSelector`] or [`IndexSelector`]. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BasicSelector { - Attr(AttrSelector), - Slot(SlotSelector), - Index(IndexSelector), -} - -impl From for BasicSelector { - fn from(value: AttrSelector) -> Self { - BasicSelector::Attr(value) - } -} - -impl From for BasicSelector { - fn from(value: SlotSelector) -> Self { - BasicSelector::Slot(value) - } -} - -impl From for BasicSelector { - fn from(value: IndexSelector) -> Self { - BasicSelector::Index(value) - } -} - -impl Selector for BasicSelector { - fn select<'a>(&self, value: &'a Value) -> Option<&'a Value> { - match self { - BasicSelector::Attr(s) => s.select(value), - BasicSelector::Slot(s) => s.select(value), - BasicSelector::Index(s) => s.select(value), - } - } -} - -/// A selector that applies a sequence of simpler selectors, in order, passing the result of one selector to the next. -#[derive(Default, Debug, Clone, PartialEq, Eq)] -pub struct ChainSelector(Vec); - -impl ChainSelector { - /// # Arguments - /// * `selector` - A chain of simpler selectors. - pub fn new(selectors: Vec) -> Self { - ChainSelector(selectors) - } -} - -impl Selector for ChainSelector { - fn select<'a>(&self, value: &'a Value) -> Option<&'a Value> { - let mut v = Some(value); - let ChainSelector(selectors) = self; - for s in selectors { - let selected = if let Some(v) = v { - s.select(v) - } else { - break; - }; - v = selected; - } - v - } -} - -static INIT_REGEX: OnceLock = OnceLock::new(); -static FIELD_REGEX: OnceLock = OnceLock::new(); - -/// Regular expression matching the base selectors for topic, key and payload components. -fn init_regex() -> &'static Regex { - INIT_REGEX.get_or_init(|| create_init_regex().expect("Invalid regex.")) -} - -/// Regular expression matching the description of a [`BasicSelector`]. -fn field_regex() -> &'static Regex { - FIELD_REGEX.get_or_init(|| create_field_regex().expect("Invalid regex.")) -} - -fn create_init_regex() -> Result { - Regex::new("\\A(\\$(?:[a-z]+))(?:\\[(\\d+)])?\\z") -} - -fn create_field_regex() -> Result { - Regex::new("\\A(\\@?(?:\\w+))(?:\\[(\\d+)])?\\z") -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct SelectorComponent<'a> { - is_attr: bool, - name: &'a str, - index: Option, -} - -impl<'a> SelectorComponent<'a> { - pub fn new(is_attr: bool, name: &'a str, index: Option) -> Self { - SelectorComponent { - is_attr, - name, - index, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct RawSelectorDescriptor<'a> { - part: &'a str, - index: Option, - components: Vec>, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum SelectorDescriptor<'a> { - Part { - part: MessagePart, - index: Option, - components: Vec>, - }, - Topic, -} - -impl<'a> TryFrom> for SelectorDescriptor<'a> { - type Error = BadSelector; - - fn try_from(value: RawSelectorDescriptor<'a>) -> Result { - let RawSelectorDescriptor { - part, - index, - components, - } = value; - match part { - "$topic" => { - if index.is_none() && components.is_empty() { - Ok(SelectorDescriptor::Topic) - } else { - Err(BadSelector::TopicWithComponent) - } - } - "$key" => Ok(SelectorDescriptor::Part { - part: MessagePart::Key, - index, - components, - }), - "$payload" => Ok(SelectorDescriptor::Part { - part: MessagePart::Payload, - index, - components, - }), - _ => Err(BadSelector::InvalidRoot), - } - } -} - -fn make_chain_selector( - index: Option, - components: &[SelectorComponent<'_>], -) -> ChainSelector { - let mut links = vec![]; - if let Some(n) = index { - links.push(BasicSelector::Index(IndexSelector::new(n))); - } - for SelectorComponent { - is_attr, - name, - index, - } in components - { - links.push(if *is_attr { - BasicSelector::Attr(AttrSelector::new(name.to_string())) - } else { - BasicSelector::Slot(SlotSelector::for_field(*name)) - }); - if let Some(n) = index { - links.push(BasicSelector::Index(IndexSelector::new(*n))); - } - } - ChainSelector::new(links) -} - -impl<'a> SelectorDescriptor<'a> { - pub fn field(&self) -> MessageField { - match self { - SelectorDescriptor::Part { part, .. } => (*part).into(), - SelectorDescriptor::Topic => MessageField::Topic, - } - } - - pub fn suggested_name(&self) -> Option<&'a str> { - match self { - SelectorDescriptor::Part { - part, - index, - components, - } => { - if let Some(SelectorComponent { name, index, .. }) = components.last() { - if index.is_none() { - Some(*name) - } else { - None - } - } else if index.is_none() { - Some(match part { - MessagePart::Key => "key", - MessagePart::Payload => "payload", - }) - } else { - None - } - } - SelectorDescriptor::Topic => Some("topic"), - } - } - - pub fn selector(&self) -> Option { - match self { - SelectorDescriptor::Part { - index, components, .. - } => Some(make_chain_selector(*index, components)), - SelectorDescriptor::Topic => None, - } - } -} - -/// Attempt to parse a descriptor for a selector from a string. -fn parse_selector(descriptor: &str) -> Result, BadSelector> { - parse_raw_selector(descriptor)?.try_into() -} - -/// Attempt to parse a descriptor for a selector from a string. -fn parse_raw_selector(descriptor: &str) -> Result, BadSelector> { - if descriptor.is_empty() { - return Err(BadSelector::EmptySelector); - } - let mut it = descriptor.split('.'); - let (field, index) = match it.next() { - Some(root) if !root.is_empty() => { - if let Some(captures) = init_regex().captures(root) { - let field = match captures.get(1) { - Some(kind) => kind.as_str(), - _ => return Err(BadSelector::InvalidRoot), - }; - let index = if let Some(index_match) = captures.get(2) { - Some(index_match.as_str().parse::()?) - } else { - None - }; - (field, index) - } else { - return Err(BadSelector::InvalidRoot); - } - } - _ => return Err(BadSelector::EmptyComponent), - }; - - let mut components = vec![]; - for part in it { - if part.is_empty() { - return Err(BadSelector::EmptyComponent); - } - if let Some(captures) = field_regex().captures(part) { - let (is_attr, name) = match captures.get(1) { - Some(name) if name.as_str().starts_with('@') => (true, &name.as_str()[1..]), - Some(name) => (false, name.as_str()), - _ => return Err(BadSelector::InvalidComponent), - }; - let index = if let Some(index_match) = captures.get(2) { - Some(index_match.as_str().parse::()?) - } else { - None - }; - components.push(SelectorComponent::new(is_attr, name, index)); - } else { - return Err(BadSelector::InvalidRoot); - } - } - - Ok(RawSelectorDescriptor { - part: field, - index, - components, - }) -} - -/// A value lane selector generates event handlers from Kafka messages to update the state of a value lane. -#[derive(Debug, Clone)] -pub struct ValueLaneSelector { - name: String, - selector: LaneSelector, - required: bool, -} - -impl ValueLaneSelector { - /// # Arguments - /// * `name` - The name of the lane. - /// * `selector` - Selects a component from the message. - /// * `required` - If this is required and the selector does not return a result, an error will be generated. - pub fn new(name: String, selector: LaneSelector, required: bool) -> Self { - ValueLaneSelector { - name, - selector, - required, - } - } -} - -/// A value lane selector generates event handlers from Kafka messages to update the state of a map lane. -#[derive(Debug, Clone)] -pub struct MapLaneSelector { - name: String, - key_selector: LaneSelector, - value_selector: LaneSelector, - required: bool, - remove_when_no_value: bool, -} - -impl MapLaneSelector { - /// # Arguments - /// * `name` - The name of the lane. - /// * `key_selector` - Selects a component from the message for the map key. - /// * `value_selector` - Selects a component from the message for the map value. - /// * `required` - If this is required and the selectors do not return a result, an error will be generated. - /// * `remove_when_no_value` - If a key is selected but no value is selected, the corresponding entry will be - /// removed from the map. - pub fn new( - name: String, - key_selector: LaneSelector, - value_selector: LaneSelector, - required: bool, - remove_when_no_value: bool, - ) -> Self { - MapLaneSelector { - name, - key_selector, - value_selector, - required, - remove_when_no_value, - } - } -} - -impl TryFrom<&IngressValueLaneSpec> for ValueLaneSelector { - type Error = InvalidLaneSpec; - - fn try_from(value: &IngressValueLaneSpec) -> Result { - let IngressValueLaneSpec { - name, - selector, - required, - } = value; - let parsed = parse_selector(selector.as_str())?; - if let Some(lane_name) = name - .as_ref() - .cloned() - .or_else(|| parsed.suggested_name().map(|s| s.to_owned())) - { - Ok(ValueLaneSelector::new(lane_name, parsed.into(), *required)) - } else { - Err(InvalidLaneSpec::NameCannotBeInferred) - } - } -} - -impl TryFrom<&IngressMapLaneSpec> for MapLaneSelector { - type Error = InvalidLaneSpec; - - fn try_from(value: &IngressMapLaneSpec) -> Result { - let IngressMapLaneSpec { - name, - key_selector, - value_selector, - remove_when_no_value, - required, - } = value; - let key = LaneSelector::from(parse_selector(key_selector.as_str())?); - let value = LaneSelector::from(parse_selector(value_selector.as_str())?); - Ok(MapLaneSelector::new( - name.clone(), - key, - value, - *required, - *remove_when_no_value, - )) - } -} - -type GenericValueLaneSet = - Discard>>; - -impl ValueLaneSelector { - pub fn name(&self) -> &str { - &self.name - } - - pub fn select_handler( - &self, - topic: &Value, - key: &mut K, - value: &mut V, - ) -> Result - where - K: Deferred, - V: Deferred, - { - let ValueLaneSelector { - name, - selector, - required, - } = self; - let maybe_value = selector.select(topic, key, value)?; - let handler = match maybe_value { - Some(value) => { - trace!(name, value = %value, "Setting a value extracted from a Kafka message to a value lane."); - let select_lane = ValueLaneSelectorFn::new(name.clone()); - Some(ValueLaneSelectSet::new(select_lane, value.clone())) - } - None => { - if *required { - error!(name, "A Kafka message did not contain a required value."); - return Err(LaneSelectorError::MissingRequiredLane(name.clone())); - } else { - None - } - } - }; - Ok(handler.discard()) - } -} - -type MapLaneUpdate = MapLaneSelectUpdate; -type MapLaneRemove = MapLaneSelectRemove; -type MapLaneOp = Coprod!(MapLaneUpdate, MapLaneRemove); - -type GenericMapLaneOp = Discard>; - -impl MapLaneSelector { - pub fn name(&self) -> &str { - &self.name - } - - pub fn select_handler( - &self, - topic: &Value, - key: &mut K, - value: &mut V, - ) -> Result - where - K: Deferred, - V: Deferred, - { - let MapLaneSelector { - name, - key_selector, - value_selector, - required, - remove_when_no_value, - } = self; - let maybe_key: Option = key_selector.select(topic, key, value)?.cloned(); - let maybe_value = value_selector.select(topic, key, value)?; - let select_lane = MapLaneSelectorFn::new(name.clone()); - let handler: Option = match (maybe_key, maybe_value) { - (None, _) if *required => { - error!( - name, - "A Kafka message did not contain a required map lane update/removal." - ); - return Err(LaneSelectorError::MissingRequiredLane(name.clone())); - } - (Some(key), None) if *remove_when_no_value => { - trace!(name, key = %key, "Removing an entry from a map lane with a key extracted from a Kafka message."); - let remove = MapLaneSelectRemove::new(select_lane, key); - Some(MapLaneOp::inject(remove)) - } - (Some(key), Some(value)) => { - trace!(name, key = %key, value = %value, "Updating a map lane with an entry extracted from a Kafka message."); - let update = MapLaneSelectUpdate::new(select_lane, key, value.clone()); - Some(MapLaneOp::inject(update)) - } - _ => None, - }; - Ok(handler.discard()) - } -} +pub mod message; diff --git a/server/swimos_connector_util/Cargo.toml b/server/swimos_connector_util/Cargo.toml new file mode 100644 index 000000000..584cc35a5 --- /dev/null +++ b/server/swimos_connector_util/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "swimos_connector_util" +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +homepage.workspace = true + +[dependencies] +swimos_agent = { workspace = true } +swimos_api = { workspace = true } +futures = { workspace = true } +bytes = { workspace = true } +swimos_form = { workspace = true } +swimos_utilities = { workspace = true, features = ["text"] } +swimos_model = { workspace = true } +tokio = { workspace = true, features = ["time"] } diff --git a/server/swimos_connector_util/src/lib.rs b/server/swimos_connector_util/src/lib.rs new file mode 100644 index 000000000..7507c73a6 --- /dev/null +++ b/server/swimos_connector_util/src/lib.rs @@ -0,0 +1,266 @@ +use std::{ + cell::RefCell, + collections::{HashMap, HashSet}, +}; + +use bytes::BytesMut; +use futures::{stream::FuturesUnordered, StreamExt}; +use swimos_agent::{ + agent_model::{ + downlink::BoxDownlinkChannelFactory, AgentSpec, ItemDescriptor, ItemFlags, WarpLaneKind, + }, + event_handler::{ + ActionContext, DownlinkSpawnOnDone, EventHandler, HandlerFuture, LaneSpawnOnDone, + LaneSpawner, LinkSpawner, Spawner, StepResult, + }, + AgentMetadata, +}; +use swimos_api::{ + address::Address, + agent::AgentConfig, + error::{CommanderRegistrationError, DynamicRegistrationError}, +}; +use swimos_model::Text; +use swimos_utilities::routing::RouteUri; + +pub struct LaneRequest { + name: String, + is_map: bool, + on_done: LaneSpawnOnDone, +} + +impl std::fmt::Debug for LaneRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LaneRequest") + .field("name", &self.name) + .field("is_map", &self.is_map) + .field("on_done", &"...") + .finish() + } +} + +#[derive(Debug)] +pub struct TestSpawner { + allow_downlinks: bool, + downlinks: RefCell>>, + suspended: FuturesUnordered>, + lane_requests: RefCell>>, +} + +impl Default for TestSpawner { + fn default() -> Self { + TestSpawner { + allow_downlinks: false, + downlinks: RefCell::new(vec![]), + suspended: Default::default(), + lane_requests: RefCell::new(vec![]), + } + } +} + +impl TestSpawner { + pub fn with_downlinks() -> TestSpawner { + TestSpawner { + allow_downlinks: true, + ..Default::default() + } + } + + pub fn take_downlinks(&self) -> Vec> { + let mut guard = self.downlinks.borrow_mut(); + std::mem::take(&mut *guard) + } +} + +impl Spawner for TestSpawner { + fn spawn_suspend(&self, fut: HandlerFuture) { + self.suspended.push(fut); + } + + fn schedule_timer(&self, _at: tokio::time::Instant, _id: u64) { + panic!("Unexpected timer"); + } +} + +pub struct DownlinkRequest { + pub path: Address, + _make_channel: BoxDownlinkChannelFactory, + _on_done: DownlinkSpawnOnDone, +} + +impl std::fmt::Debug for DownlinkRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DownlinkRequest") + .field("path", &self.path) + .field("make_channel", &"...") + .field("_on_done", &"...") + .finish() + } +} + +impl LinkSpawner for TestSpawner { + fn spawn_downlink( + &self, + path: Address, + _make_channel: BoxDownlinkChannelFactory, + _on_done: DownlinkSpawnOnDone, + ) { + if self.allow_downlinks { + self.downlinks.borrow_mut().push(DownlinkRequest { + path: path.into(), + _make_channel, + _on_done, + }) + } else { + panic!("Opening downlinks not supported."); + } + } + + fn register_commander(&self, _path: Address) -> Result { + panic!("Registering commanders not supported."); + } +} + +impl LaneSpawner for TestSpawner { + fn spawn_warp_lane( + &self, + name: &str, + kind: WarpLaneKind, + on_done: LaneSpawnOnDone, + ) -> Result<(), DynamicRegistrationError> { + let is_map = match kind { + WarpLaneKind::Map => true, + WarpLaneKind::Value => false, + _ => panic!("Unexpected lane kind: {}", kind), + }; + self.lane_requests.borrow_mut().push(LaneRequest { + name: name.to_string(), + is_map, + on_done, + }); + Ok(()) + } +} + +const CONFIG: AgentConfig = AgentConfig::DEFAULT; +const NODE_URI: &str = "/node"; + +fn make_uri() -> RouteUri { + RouteUri::try_from(NODE_URI).expect("Bad URI.") +} + +fn make_meta<'a>( + uri: &'a RouteUri, + route_params: &'a HashMap, +) -> AgentMetadata<'a> { + AgentMetadata::new(uri, route_params, &CONFIG) +} + +pub async fn run_handler_with_futures_dl>( + agent: &A, + handler: H, +) -> (HashSet, Vec>) +where + A: AgentSpec, +{ + let mut spawner = TestSpawner::with_downlinks(); + let modified = run_handler_with_futures_inner(agent, handler, &mut spawner).await; + (modified, spawner.take_downlinks()) +} + +pub async fn run_handler_with_futures>(agent: &A, handler: H) -> HashSet +where + A: AgentSpec, +{ + run_handler_with_futures_inner(agent, handler, &mut TestSpawner::default()).await +} + +async fn run_handler_with_futures_inner>( + agent: &A, + handler: H, + spawner: &mut TestSpawner, +) -> HashSet +where + A: AgentSpec, +{ + let mut modified = run_handler(agent, spawner, handler); + let mut handlers = vec![]; + let reg = move |req: LaneRequest| { + let LaneRequest { + name, + is_map, + on_done, + } = req; + let kind = if is_map { + WarpLaneKind::Map + } else { + WarpLaneKind::Value + }; + let descriptor = ItemDescriptor::WarpLane { + kind, + flags: ItemFlags::TRANSIENT, + }; + let result = agent.register_dynamic_item(&name, descriptor); + on_done(result.map_err(Into::into)) + }; + for request in + std::mem::take::>>(spawner.lane_requests.borrow_mut().as_mut()) + { + handlers.push(reg(request)); + } + + while !(handlers.is_empty() && spawner.suspended.is_empty()) { + let m = if let Some(h) = handlers.pop() { + run_handler(agent, spawner, h) + } else { + let h = spawner.suspended.next().await.expect("No handler."); + run_handler(agent, spawner, h) + }; + modified.extend(m); + for request in + std::mem::take::>>(spawner.lane_requests.borrow_mut().as_mut()) + { + handlers.push(reg(request)); + } + } + modified +} + +pub fn run_handler>( + agent: &A, + spawner: &TestSpawner, + mut handler: H, +) -> HashSet { + let route_params = HashMap::new(); + let uri = make_uri(); + let meta = make_meta(&uri, &route_params); + let mut join_lane_init = HashMap::new(); + let mut command_buffer = BytesMut::new(); + + let mut action_context = ActionContext::new( + spawner, + spawner, + spawner, + &mut join_lane_init, + &mut command_buffer, + ); + + let mut modified = HashSet::new(); + + loop { + match handler.step(&mut action_context, meta, agent) { + StepResult::Continue { modified_item } => { + if let Some(m) = modified_item { + modified.insert(m.id()); + } + } + StepResult::Fail(err) => panic!("Handler Failed: {}", err), + StepResult::Complete { modified_item, .. } => { + if let Some(m) = modified_item { + modified.insert(m.id()); + } + break modified; + } + } + } +}