Replace `serde_cbor` with `ciborium` due to security vulnerability (#3855) (3871e9ae) · Commits · Public Repositories / Smithy Rust

codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/rustlang/CargoDependency.kt

+1 −1

Original line number	Diff line number	Diff line
		@@ -310,7 +310,7 @@ data class CargoDependency(
		val Hound: CargoDependency = CargoDependency("hound", CratesIo("3.4.0"), DependencyScope.Dev)
		val PrettyAssertions: CargoDependency =
		CargoDependency("pretty_assertions", CratesIo("1.3.0"), DependencyScope.Dev)
		val SerdeCbor: CargoDependency = CargoDependency("serde_cbor", CratesIo("0.11"), DependencyScope.Dev)
		val Ciborium: CargoDependency = CargoDependency("ciborium", CratesIo("0.2"), DependencyScope.Dev)
		val SerdeJson: CargoDependency = CargoDependency("serde_json", CratesIo("1.0.0"), DependencyScope.Dev)
		val Smol: CargoDependency = CargoDependency("smol", CratesIo("1.2.0"), DependencyScope.Dev)
		val TempFile: CargoDependency = CargoDependency("tempfile", CratesIo("3.2.0"), DependencyScope.Dev)

codegen-serde/src/test/kotlin/software/amazon/smithy/rust/codegen/serde/SerdeDecoratorTest.kt

+4 −2

Original line number	Diff line number	Diff line
		@@ -341,7 +341,7 @@ class SerdeDecoratorTest {
		arrayOf(
		"crate" to RustType.Opaque(ctx.moduleUseName()),
		"serde_json" to CargoDependency("serde_json", CratesIo("1")).toDevDependency().toType(),
		"serde_cbor" to CargoDependency("serde_cbor", CratesIo("0.11.2")).toDevDependency().toType(),
		"ciborium" to CargoDependency.Ciborium.toType(),
		// we need the derive feature
		"serde" to CargoDependency.Serde.toDevDependency().toType(),
		)
		@@ -437,7 +437,9 @@ class SerdeDecoratorTest {
		use #{crate}::serde::*;
		let input = StreamingInput::builder().data(ByteStream::from_static(b"123")).build().unwrap();
		let settings = SerializationSettings::default();
		let serialized = #{serde_cbor}::to_vec(&input.serialize_ref(&settings)).expect("failed to serialize");
		let mut serialized = Vec::new();
		#{ciborium}::ser::into_writer(&input.serialize_ref(&settings), &mut serialized)
		.expect("failed to serialize input into CBOR format using `ciborium`");
		assert_eq!(serialized, b"\xa1ddataC123");
		""",
		*codegenScope,

codegen-server/src/test/kotlin/software/amazon/smithy/rust/codegen/server/smithy/protocols/serialize/CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest.kt

+10 −9

Original line number	Diff line number	Diff line
		@@ -62,7 +62,7 @@ import java.util.logging.Logger
		/**
		* This lives in `codegen-server` because we want to run a full integration test for convenience,
		* but there's really nothing server-specific here. We're just testing that the CBOR (de)serializers work like
		* the ones generated by `serde_cbor`. This is a good exhaustive litmus test for correctness, since `serde_cbor`
		* the ones generated by `ciborium`. This is a good exhaustive litmus test for correctness, since `ciborium`
		* is battle-tested.
		*/
		internal class CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest {
		@@ -175,7 +175,7 @@ internal class CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest {
		}

		@Test
		fun `serde_cbor round trip`() {
		fun `ciborium round trip`() {
		val addDeriveSerdeSerializeDeserializeDecorator =
		object : ServerCodegenDecorator {
		override val name: String = "Add `#[derive(serde::Serialize, serde::Deserialize)]`"
		@@ -240,7 +240,7 @@ internal class CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest {
		val codegenScope =
		arrayOf(
		"AssertEq" to RuntimeType.PrettyAssertions.resolve("assert_eq!"),
		"SerdeCbor" to CargoDependency.SerdeCbor.toType(),
		"ciborium" to CargoDependency.Ciborium.toType(),
		)

		val instantiator = ServerInstantiator(codegenContext, ignoreMissingMembers = true, withinTest = true)
		@@ -278,14 +278,14 @@ internal class CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest {
		if (expectFail.contains(test.id)) {
		writeWithNoFormatting("#[should_panic]")
		}
		unitTest("we_serialize_and_serde_cbor_deserializes_${test.id.toSnakeCase()}_${test.kind.toString().toSnakeCase()}") {
		unitTest("we_serialize_and_ciborium_deserializes_${test.id.toSnakeCase()}_${test.kind.toString().toSnakeCase()}") {
		rustTemplate(
		"""
		let expected = #{InstantiateShape:W};
		let bytes = #{SerializeFn}(&expected)
		.expect("our generated CBOR serializer failed");
		let actual = #{SerdeCbor}::from_slice(&bytes)
		.expect("serde_cbor failed deserializing from bytes");
		let actual = #{ciborium}::from_reader(::std::io::Cursor::new(&bytes))
		.expect("failed to deserialize bytes with `ciborium`");
		#{AssertEq}(expected, actual);
		""",
		"InstantiateShape" to instantiator.generate(targetShape, params),
		@@ -330,12 +330,13 @@ internal class CborSerializerAndParserGeneratorSerdeRoundTripIntegrationTest {
		if (expectFail.contains(test.id)) {
		writeWithNoFormatting("#[should_panic]")
		}
		unitTest("serde_cbor_serializes_and_we_deserialize_${test.id.toSnakeCase()}_${test.kind.toString().toSnakeCase()}") {
		unitTest("ciborium_serializes_and_we_deserialize_${test.id.toSnakeCase()}_${test.kind.toString().toSnakeCase()}") {
		rustTemplate(
		"""
		let expected = #{InstantiateShape:W};
		let bytes: Vec<u8> = #{SerdeCbor}::to_vec(&expected)
		.expect("serde_cbor failed serializing to `Vec<u8>`");
		let mut bytes = Vec::new();
		#{ciborium}::into_writer(&expected, &mut bytes)
		.expect("failed to serialize to `Vec<u8>` with `ciborium`");
		let input = #{InputBuilder}::default();
		let input = #{DeserializeFn}(&bytes, input)
		.expect("our generated CBOR deserializer failed");

rust-runtime/aws-smithy-protocol-test/Cargo.toml

+2 −2

Original line number	Diff line number	Diff line
		[package]
		name = "aws-smithy-protocol-test"
		version = "0.62.0"
		version = "0.63.0"
		authors = ["AWS Rust SDK Team <aws-sdk-rust@amazon.com>", "Russell Cohen <rcoh@amazon.com>"]
		description = "A collection of library functions to validate HTTP requests against Smithy protocol tests."
		edition = "2021"
		@@ -12,7 +12,7 @@ repository = "https://github.com/smithy-lang/smithy-rs"
		assert-json-diff = "1.1"
		base64-simd = "0.8"
		cbor-diag = "0.1.12"
		serde_cbor = "0.11"
		ciborium = "0.2"
		http = "0.2.1"
		pretty_assertions = "1.3"
		regex-lite = "0.1.5"

rust-runtime/aws-smithy-protocol-test/src/lib.rs

+126 −4

Original line number	Diff line number	Diff line
		@@ -415,6 +415,88 @@ fn try_json_eq(expected: &str, actual: &str) -> Result<(), ProtocolTestFailure>
		}
		}

		/// Compares two `ciborium::value::Value` instances for semantic equality.
		///
		/// This function recursively compares two CBOR values, correctly handling arrays and maps
		/// according to the CBOR specification. Arrays are compared element-wise in order,
		/// while maps are compared without considering the order of key-value pairs.
		fn cbor_values_equal(
		a: &ciborium::value::Value,
		b: &ciborium::value::Value,
		) -> Result<bool, ProtocolTestFailure> {
		match (a, b) {
		(ciborium::value::Value::Array(a_array), ciborium::value::Value::Array(b_array)) => {
		// Both arrays should be equal in size.
		if a_array.len() != b_array.len() {
		return Ok(false);
		}
		// Compare arrays element-wise.
		for (a_elem, b_elem) in a_array.iter().zip(b_array.iter()) {
		if !cbor_values_equal(a_elem, b_elem)? {
		return Ok(false);
		}
		}
		Ok(true)
		}

		// Convert `ciborium::value::Value::Map` to a `HashMap`, and then compare the values of
		// each key in `a` with those in `b`.
		(ciborium::value::Value::Map(a_map), ciborium::value::Value::Map(b_map)) => {
		if a_map.len() != b_map.len() {
		return Ok(false);
		}

		let b_hashmap = ciborium_map_to_hashmap(b_map)?;
		// Each key in `a` should exist in `b`, and the values should match.
		for a_key_value in a_map.iter() {
		let (a_key, a_value) = get_text_key_value(a_key_value)?;
		match b_hashmap.get(a_key) {
		Some(b_value) => {
		if !cbor_values_equal(a_value, b_value)? {
		return Ok(false);
		}
		}
		None => return Ok(false),
		}
		}
		Ok(true)
		}

		(ciborium::value::Value::Float(a_float), ciborium::value::Value::Float(b_float)) => {
		Ok(a_float == b_float \|\| (a_float.is_nan() && b_float.is_nan()))
		}

		_ => Ok(a == b),
		}
		}

		/// Converts a `ciborium::value::Value::Map` into a `HashMap<&String, &ciborium::value::Value>`.
		///
		/// CBOR maps (`Value::Map`) are internally represented as vectors of key-value pairs,
		/// and direct comparison is affected by the order of these pairs.
		/// Since the CBOR specification treats maps as unordered collections,
		/// this function transforms the vector into a `HashMap`, for order-independent comparisons
		/// between maps.
		fn ciborium_map_to_hashmap(
		cbor_map: &[(ciborium::value::Value, ciborium::value::Value)],
		) -> Result<std::collections::HashMap<&String, &ciborium::value::Value>, ProtocolTestFailure> {
		cbor_map.iter().map(get_text_key_value).collect()
		}

		/// Extracts a string key and its associated value from a CBOR key-value pair.
		/// Returns a `ProtocolTestFailure::InvalidBodyFormat` error if the key is not a text value.
		fn get_text_key_value(
		(key, value): &(ciborium::value::Value, ciborium::value::Value),
		) -> Result<(&String, &ciborium::value::Value), ProtocolTestFailure> {
		match key {
		ciborium::value::Value::Text(key_str) => Ok((key_str, value)),
		_ => Err(ProtocolTestFailure::InvalidBodyFormat {
		expected: "a text key as map entry".to_string(),
		found: format!("{:?}", key),
		}),
		}
		}

		fn try_cbor_eq<T: AsRef<[u8]> + Debug>(
		actual_body: T,
		expected_body: &str,
		@@ -422,16 +504,16 @@ fn try_cbor_eq<T: AsRef<[u8]> + Debug>(
		let decoded = base64_simd::STANDARD
		.decode_to_vec(expected_body)
		.expect("smithy protocol test `body` property is not properly base64 encoded");
		let expected_cbor_value: serde_cbor::Value =
		serde_cbor::from_slice(decoded.as_slice()).expect("expected value must be valid CBOR");
		let actual_cbor_value: serde_cbor::Value = serde_cbor::from_slice(actual_body.as_ref())
		let expected_cbor_value: ciborium::value::Value =
		ciborium::de::from_reader(decoded.as_slice()).expect("expected value must be valid CBOR");
		let actual_cbor_value: ciborium::value::Value = ciborium::de::from_reader(actual_body.as_ref())
		.map_err(\|e\| ProtocolTestFailure::InvalidBodyFormat {
		expected: "cbor".to_owned(),
		found: format!("{} {:?}", e, actual_body),
		})?;
		let actual_body_base64 = base64_simd::STANDARD.encode_to_string(&actual_body);

		if expected_cbor_value != actual_cbor_value {
		if !cbor_values_equal(&expected_cbor_value, &actual_cbor_value)? {
		let expected_body_annotated_hex: String = cbor_diag::parse_bytes(&decoded)
		.expect("smithy protocol test `body` property is not valid CBOR")
		.to_hex();
		@@ -599,6 +681,46 @@ mod tests {
		validate_body(actual, expected, MediaType::Json).expect_err("bodies do not match");
		}

		#[test]
		fn test_validate_cbor_body() {
		let base64_encode = \|v: &[u8]\| base64_simd::STANDARD.encode_to_string(v);

		// The following is the CBOR representation of `{"abc": 5 }`.
		let actual = [0xbf, 0x63, 0x61, 0x62, 0x63, 0x05, 0xff];
		// The following is the base64-encoded CBOR representation of `{"abc": 5 }` using a definite length map.
		let expected_base64 = base64_encode(&[0xA1, 0x63, 0x61, 0x62, 0x63, 0x05]);

		validate_body(actual, expected_base64.as_str(), MediaType::Cbor)
		.expect("unexpected mismatch between CBOR definite and indefinite map encodings");

		// The following is the CBOR representation of `{"a":1, "b":2}`.
		let actual = [0xBF, 0x61, 0x61, 0x01, 0x61, 0x62, 0x02, 0xFF];
		// The following is the base64-encoded CBOR representation of `{"b":2, "a":1}`.
		let expected_base64 = base64_encode(&[0xBF, 0x61, 0x62, 0x02, 0x61, 0x61, 0x01, 0xFF]);
		validate_body(actual, expected_base64.as_str(), MediaType::Cbor)
		.expect("different ordering in CBOR decoded maps do not match");

		// The following is the CBOR representation of `{"a":[1,2,{"b":3, "c":4}]}`.
		let actual = [
		0xBF, 0x61, 0x61, 0x9F, 0x01, 0x02, 0xBF, 0x61, 0x62, 0x03, 0x61, 0x63, 0x04, 0xFF,
		0xFF, 0xFF,
		];
		// The following is the base64-encoded CBOR representation of `{"a":[1,2,{"c":4, "b":3}]}`.
		let expected_base64 = base64_encode(&[
		0xBF, 0x61, 0x61, 0x9F, 0x01, 0x02, 0xBF, 0x61, 0x63, 0x04, 0x61, 0x62, 0x03, 0xFF,
		0xFF, 0xFF,
		]);
		validate_body(actual, expected_base64.as_str(), MediaType::Cbor)
		.expect("different ordering in CBOR decoded maps do not match");

		// The following is the CBOR representation of `{"a":[1,2]}`.
		let actual = [0xBF, 0x61, 0x61, 0x9F, 0x01, 0x02, 0xFF, 0xFF];
		// The following is the CBOR representation of `{"a":[2,1]}`.
		let expected_base64 = base64_encode(&[0xBF, 0x61, 0x61, 0x9F, 0x02, 0x01, 0xFF, 0xFF]);
		validate_body(actual, expected_base64.as_str(), MediaType::Cbor)
		.expect_err("arrays in CBOR should follow strict ordering");
		}

		#[test]
		fn test_validate_xml_body() {
		let expected = r#"<a>