Unverified Commit 9c95803a authored by Zelda Hessler's avatar Zelda Hessler Committed by GitHub
Browse files

add support for adaptive retries when orchestrator mode is enabled (#2800) 



## Motivation and Context
<!--- Why is this change required? What problem does it solve? -->
<!--- If it fixes an open issue, please link to the issue here -->
#2190  

## Description
<!--- Describe your changes in detail -->
add support for adaptive retries

## Testing
<!--- Please describe in detail how you tested your changes -->
<!--- Include details of your testing environment, and the tests you ran
to -->
<!--- see how your change affects other areas of the code, etc. -->
I wrote some tests and I'm open to suggestions for more.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._

---------

Co-authored-by: default avatarJohn DiSanti <jdisanti@amazon.com>
parent c8ba2d5b

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/IntegrationTestDependencies.kt

+20 −0
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ package software.amazon.smithy.rustsdk 
import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext

import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator

import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency

import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.Approx

import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.AsyncStd

import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.AsyncStream

import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.BytesUtils
@@ -26,6 +27,8 @@ import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Compani 
import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.TracingAppender

import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.TracingSubscriber

import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.TracingTest

import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.smithyRuntime

import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency.Companion.smithyRuntimeApi

import software.amazon.smithy.rust.codegen.core.rustlang.DependencyScope

import software.amazon.smithy.rust.codegen.core.rustlang.Writable

import software.amazon.smithy.rust.codegen.core.rustlang.writable
@@ -73,6 +76,7 @@ class IntegrationTestDependencies( 
    private val hasTests: Boolean,

    private val hasBenches: Boolean,

) : LibRsCustomization() {

    private val runtimeConfig = codegenContext.runtimeConfig

    override fun section(section: LibRsSection) = when (section) {

        is LibRsSection.Body -> testDependenciesOnly {

            if (hasTests) {
@@ -80,14 +84,22 @@ class IntegrationTestDependencies( 
                    .copy(features = setOf("test-util"), scope = DependencyScope.Dev)

                val smithyAsync = CargoDependency.smithyAsync(codegenContext.runtimeConfig)

                    .copy(features = setOf("test-util"), scope = DependencyScope.Dev)

                val smithyTypes = CargoDependency.smithyTypes(codegenContext.runtimeConfig)

                    .copy(features = setOf("test-util"), scope = DependencyScope.Dev)

                addDependency(smithyClient)

                addDependency(smithyAsync)

                addDependency(smithyTypes)

                addDependency(CargoDependency.smithyProtocolTestHelpers(codegenContext.runtimeConfig))

                addDependency(SerdeJson)

                addDependency(Tokio)

                addDependency(FuturesUtil)

                addDependency(Tracing.toDevDependency())

                addDependency(TracingSubscriber)



                if (codegenContext.smithyRuntimeMode.generateOrchestrator) {

                    addDependency(smithyRuntime(runtimeConfig).copy(features = setOf("test-util"), scope = DependencyScope.Dev))

                    addDependency(smithyRuntimeApi(runtimeConfig).copy(features = setOf("test-util"), scope = DependencyScope.Dev))

                }

            }

            if (hasBenches) {

                addDependency(Criterion)
@@ -103,6 +115,7 @@ class IntegrationTestDependencies( 
    private fun serviceSpecificCustomizations(): List<LibRsCustomization> = when (moduleName) {

        "transcribestreaming" -> listOf(TranscribeTestDependencies())

        "s3" -> listOf(S3TestDependencies(codegenContext))

        "dynamodb" -> listOf(DynamoDbTestDependencies())

        else -> emptyList()

    }

}
@@ -116,6 +129,13 @@ class TranscribeTestDependencies : LibRsCustomization() { 
        }

}



class DynamoDbTestDependencies : LibRsCustomization() {

    override fun section(section: LibRsSection): Writable =

        writable {

            addDependency(Approx)

        }

}



class S3TestDependencies(private val codegenContext: ClientCodegenContext) : LibRsCustomization() {

    override fun section(section: LibRsSection): Writable =

        writable {

aws/sdk/integration-tests/dynamodb/Cargo.toml

+7 −3
Original line number Diff line number Diff line
@@ -11,14 +11,18 @@ publish = false 
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html



[dependencies]

approx = "0.5.1"

aws-config = { path = "../../build/aws-sdk/sdk/aws-config" }

aws-credential-types = { path = "../../build/aws-sdk/sdk/aws-credential-types", features = ["test-util"] }

aws-http = { path = "../../build/aws-sdk/sdk/aws-http" }

aws-sdk-dynamodb = { path = "../../build/aws-sdk/sdk/dynamodb" }

aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async" }

aws-smithy-client = { path = "../../build/aws-sdk/sdk/aws-smithy-client", features = ["test-util", "rustls"] }

aws-smithy-http = { path = "../../build/aws-sdk/sdk/aws-smithy-http" }

aws-smithy-types = { path = "../../build/aws-sdk/sdk/aws-smithy-types" }

aws-smithy-protocol-test = { path = "../../build/aws-sdk/sdk/aws-smithy-protocol-test" }

aws-smithy-async = { path = "../../build/aws-sdk/sdk/aws-smithy-async" }

aws-smithy-runtime = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime", features = ["test-util"]}

aws-smithy-runtime-api = { path = "../../build/aws-sdk/sdk/aws-smithy-runtime-api", features = ["test-util"]}

aws-smithy-types = { path = "../../build/aws-sdk/sdk/aws-smithy-types", features = ["test-util"]}

aws-types = { path = "../../build/aws-sdk/sdk/aws-types" }

bytes = "1.0.0"

criterion = { version = "0.4.0" }
@@ -26,8 +30,8 @@ futures-util = { version = "0.3.16", default-features = false } 
http = "0.2.0"

serde_json = "1.0.0"

tokio = { version = "1.23.1", features = ["full", "test-util"] }

tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }

tokio-stream = "0.1.5"

tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }



[[bench]]

name = "deserialization_bench"

aws/sdk/integration-tests/dynamodb/tests/retries-with-client-rate-limiting.rs

0 → 100644
+173 −0
Original line number Diff line number Diff line 
/*

 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.

 * SPDX-License-Identifier: Apache-2.0

 */



#[cfg(aws_sdk_orchestrator_mode)]

mod test {

    use aws_sdk_dynamodb::config::{Credentials, Region, SharedAsyncSleep};

    use aws_sdk_dynamodb::{config::retry::RetryConfig, error::ProvideErrorMetadata};

    use aws_smithy_async::rt::sleep::TokioSleep;

    use aws_smithy_async::test_util::instant_time_and_sleep;

    use aws_smithy_async::time::SharedTimeSource;

    use aws_smithy_async::time::SystemTimeSource;

    use aws_smithy_client::test_connection::TestConnection;

    use aws_smithy_http::body::SdkBody;

    use aws_smithy_runtime::client::retries::RetryPartition;

    use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse};

    use aws_smithy_types::timeout::TimeoutConfigBuilder;

    use std::time::{Duration, Instant, SystemTime};



    fn req() -> HttpRequest {

        http::Request::builder()

            .body(SdkBody::from("request body"))

            .unwrap()

    }



    fn ok() -> HttpResponse {

        http::Response::builder()

            .status(200)

            .header("server", "Server")

            .header("content-type", "application/x-amz-json-1.0")

            .header("content-length", "23")

            .header("connection", "keep-alive")

            .header("x-amz-crc32", "2335643545")

            .body(SdkBody::from("{ \"TableNames\": [ \"Test\" ] }"))

            .unwrap()

    }



    fn err() -> HttpResponse {

        http::Response::builder()

            .status(500)

            .body(SdkBody::from("{ \"message\": \"The request has failed because of an unknown error, exception or failure.\", \"code\": \"InternalServerError\" }"))

            .unwrap()

    }



    fn throttling_err() -> HttpResponse {

        http::Response::builder()

            .status(400)

            .body(SdkBody::from("{ \"message\": \"The request was denied due to request throttling.\", \"code\": \"ThrottlingException\" }"))

            .unwrap()

    }



    #[tokio::test]

    async fn test_adaptive_retries_with_no_throttling_errors() {

        let (time_source, sleep_impl) = instant_time_and_sleep(SystemTime::UNIX_EPOCH);



        let events = vec![

            // First operation

            (req(), err()),

            (req(), err()),

            (req(), ok()),

            // Second operation

            (req(), err()),

            (req(), ok()),

            // Third operation will fail, only errors

            (req(), err()),

            (req(), err()),

            (req(), err()),

            (req(), err()),

        ];



        let conn = TestConnection::new(events);

        let config = aws_sdk_dynamodb::Config::builder()

            .credentials_provider(Credentials::for_tests())

            .region(Region::new("us-east-1"))

            .retry_config(

                RetryConfig::adaptive()

                    .with_max_attempts(4)

                    .with_use_static_exponential_base(true),

            )

            .time_source(SharedTimeSource::new(time_source))

            .sleep_impl(SharedAsyncSleep::new(sleep_impl.clone()))

            .retry_partition(RetryPartition::new(

                "test_adaptive_retries_with_no_throttling_errors",

            ))

            .http_connector(conn.clone())

            .build();

        let expected_table_names = vec!["Test".to_owned()];



        // We create a new client each time to ensure that the cross-client retry state is working.

        let client = aws_sdk_dynamodb::Client::from_conf(config.clone());

        let res = client.list_tables().send().await.unwrap();

        assert_eq!(sleep_impl.total_duration(), Duration::from_secs(3));

        assert_eq!(res.table_names(), Some(expected_table_names.as_slice()));

        // Three requests should have been made, two failing & one success

        assert_eq!(conn.requests().len(), 3);



        let client = aws_sdk_dynamodb::Client::from_conf(config.clone());

        let res = client.list_tables().send().await.unwrap();

        assert_eq!(sleep_impl.total_duration(), Duration::from_secs(3 + 1));

        assert_eq!(res.table_names(), Some(expected_table_names.as_slice()));

        // Two requests should have been made, one failing & one success (plus previous requests)

        assert_eq!(conn.requests().len(), 5);



        let client = aws_sdk_dynamodb::Client::from_conf(config);

        let err = client.list_tables().send().await.unwrap_err();

        assert_eq!(sleep_impl.total_duration(), Duration::from_secs(3 + 1 + 7),);

        assert_eq!(err.code(), Some("InternalServerError"));

        // four requests should have been made, all failing (plus previous requests)

        assert_eq!(conn.requests().len(), 9);

    }



    #[tokio::test]

    async fn test_adaptive_retries_with_throttling_errors_times_out() {

        tracing_subscriber::fmt::init();

        let events = vec![

            // First operation

            (req(), err()),

            (req(), ok()),

            // Second operation

            (req(), err()),

            (req(), throttling_err()),

            (req(), ok()),

        ];



        let conn = TestConnection::new(events);

        let config = aws_sdk_dynamodb::Config::builder()

            .credentials_provider(Credentials::for_tests())

            .region(Region::new("us-east-1"))

            .retry_config(

                RetryConfig::adaptive()

                    .with_max_attempts(4)

                    .with_initial_backoff(Duration::from_millis(50))

                    .with_use_static_exponential_base(true),

            )

            .timeout_config(

                TimeoutConfigBuilder::new()

                    .operation_attempt_timeout(Duration::from_millis(100))

                    .build(),

            )

            .time_source(SharedTimeSource::new(SystemTimeSource::new()))

            .sleep_impl(SharedAsyncSleep::new(TokioSleep::new()))

            .http_connector(conn.clone())

            .retry_partition(RetryPartition::new(

                "test_adaptive_retries_with_throttling_errors_times_out",

            ))

            .build();



        let expected_table_names = vec!["Test".to_owned()];

        let start = Instant::now();



        // We create a new client each time to ensure that the cross-client retry state is working.

        let client = aws_sdk_dynamodb::Client::from_conf(config.clone());

        let res = client.list_tables().send().await.unwrap();

        assert_eq!(res.table_names(), Some(expected_table_names.as_slice()));

        // Three requests should have been made, two failing & one success

        assert_eq!(conn.requests().len(), 2);



        let client = aws_sdk_dynamodb::Client::from_conf(config);

        let err = client.list_tables().send().await.unwrap_err();

        assert_eq!(err.to_string(), "request has timed out".to_owned());

        // two requests should have been made, both failing (plus previous requests)

        assert_eq!(conn.requests().len(), 2 + 2);



        let since = start.elapsed();

        // At least 300 milliseconds must pass:

        // - 50ms for the first retry on attempt 1

        // - 50ms for the second retry on attempt 3

        // - 100ms for the throttling delay triggered by attempt 4, which required a delay longer than the attempt timeout.

        // - 100ms for the 5th attempt, which would have succeeded, but required a delay longer than the attempt timeout.

        assert!(since.as_secs_f64() > 0.3);

    }

}

codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customizations/ResiliencyConfigCustomization.kt

+114 −2
Original line number Diff line number Diff line
@@ -7,8 +7,12 @@ package software.amazon.smithy.rust.codegen.client.smithy.customizations 


import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext

import software.amazon.smithy.rust.codegen.client.smithy.ClientRustModule

import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginCustomization

import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginSection

import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization

import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ServiceConfig

import software.amazon.smithy.rust.codegen.core.rustlang.Attribute

import software.amazon.smithy.rust.codegen.core.rustlang.Writable

import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate

import software.amazon.smithy.rust.codegen.core.rustlang.writable

import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig
@@ -16,12 +20,14 @@ import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType 
import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType.Companion.preludeScope

import software.amazon.smithy.rust.codegen.core.smithy.RustCrate



class ResiliencyConfigCustomization(codegenContext: ClientCodegenContext) : ConfigCustomization() {

class ResiliencyConfigCustomization(private val codegenContext: ClientCodegenContext) : ConfigCustomization() {

    private val runtimeConfig = codegenContext.runtimeConfig

    private val runtimeMode = codegenContext.smithyRuntimeMode

    private val retryConfig = RuntimeType.smithyTypes(runtimeConfig).resolve("retry")

    private val sleepModule = RuntimeType.smithyAsync(runtimeConfig).resolve("rt::sleep")

    private val timeoutModule = RuntimeType.smithyTypes(runtimeConfig).resolve("timeout")

    private val smithyRuntimeCrate = RuntimeType.smithyRuntime(runtimeConfig)

    private val retries = smithyRuntimeCrate.resolve("client::retries")

    private val moduleUseName = codegenContext.moduleUseName()

    private val codegenScope = arrayOf(

        *preludeScope,
@@ -29,8 +35,17 @@ class ResiliencyConfigCustomization(codegenContext: ClientCodegenContext) : Conf 
        "RetryConfig" to retryConfig.resolve("RetryConfig"),

        "SharedAsyncSleep" to sleepModule.resolve("SharedAsyncSleep"),

        "Sleep" to sleepModule.resolve("Sleep"),

        "StandardRetryStrategy" to RuntimeType.smithyRuntime(runtimeConfig).resolve("client::retries::strategy::StandardRetryStrategy"),

        "StandardRetryStrategy" to retries.resolve("strategy::StandardRetryStrategy"),

        "SystemTime" to RuntimeType.std.resolve("time::SystemTime"),

        "TimeoutConfig" to timeoutModule.resolve("TimeoutConfig"),

        "RetryMode" to RuntimeType.smithyTypes(runtimeConfig).resolve("retry::RetryMode"),

        "TokenBucket" to retries.resolve("TokenBucket"),

        "ClientRateLimiter" to retries.resolve("ClientRateLimiter"),

        "SharedTimeSource" to RuntimeType.smithyAsync(runtimeConfig).resolve("time::SharedTimeSource"),

        "ClientRateLimiterPartition" to retries.resolve("ClientRateLimiterPartition"),

        "TokenBucketPartition" to retries.resolve("TokenBucketPartition"),

        "RetryPartition" to retries.resolve("RetryPartition"),

        "debug" to RuntimeType.Tracing.resolve("debug"),

    )



    override fun section(section: ServiceConfig) =
@@ -67,6 +82,15 @@ class ResiliencyConfigCustomization(codegenContext: ClientCodegenContext) : Conf 
                            pub fn timeout_config(&self) -> #{Option}<&#{TimeoutConfig}> {

                                self.inner.load::<#{TimeoutConfig}>()

                            }



                            ##[doc(hidden)]

                            /// Returns a reference to the retry partition contained in this config, if any.

                            ///

                            /// WARNING: This method is unstable and may be removed at any time. Do not rely on this

                            /// method for anything!

                            pub fn retry_partition(&self) -> #{Option}<&#{RetryPartition}> {

                                self.inner.load::<#{RetryPartition}>()

                            }

                            """,

                            *codegenScope,

                        )
@@ -311,13 +335,67 @@ class ResiliencyConfigCustomization(codegenContext: ClientCodegenContext) : Conf 
                            *codegenScope,

                        )

                    }



                    if (runtimeMode.defaultToOrchestrator) {

                        Attribute.DocHidden.render(this)

                        rustTemplate(

                            """

                            /// Set the partition for retry-related state. When clients share a retry partition, they will

                            /// also share things like token buckets and client rate limiters. By default, all clients

                            /// for the same service will share a partition.

                            pub fn retry_partition(mut self, retry_partition: #{RetryPartition}) -> Self {

                                self.set_retry_partition(Some(retry_partition));

                                self

                            }

                            """,

                            *codegenScope,

                        )



                        Attribute.DocHidden.render(this)

                        rustTemplate(

                            """

                            /// Set the partition for retry-related state. When clients share a retry partition, they will

                            /// also share things like token buckets and client rate limiters. By default, all clients

                            /// for the same service will share a partition.

                            pub fn set_retry_partition(&mut self, retry_partition: #{Option}<#{RetryPartition}>) -> &mut Self {

                                retry_partition.map(|r| self.inner.store_put(r));

                                self

                            }

                            """,

                            *codegenScope,

                        )

                    }

                }



                ServiceConfig.BuilderBuild -> {

                    if (runtimeMode.defaultToOrchestrator) {

                        rustTemplate(

                            """

                            let retry_partition = layer.load::<#{RetryPartition}>().cloned().unwrap_or_else(|| #{RetryPartition}::new("${codegenContext.serviceShape.id.name}"));

                            let retry_config = layer.load::<#{RetryConfig}>().cloned().unwrap_or_else(#{RetryConfig}::disabled);

                            if retry_config.has_retry() {

                                #{debug}!("creating retry strategy with partition '{}'", retry_partition);

                            }



                            if retry_config.mode() == #{RetryMode}::Adaptive {

                                if let Some(time_source) = layer.load::<#{SharedTimeSource}>().cloned() {

                                    let seconds_since_unix_epoch = time_source

                                        .now()

                                        .duration_since(#{SystemTime}::UNIX_EPOCH)

                                        .expect("the present takes place after the UNIX_EPOCH")

                                        .as_secs_f64();

                                    let client_rate_limiter_partition = #{ClientRateLimiterPartition}::new(retry_partition.clone());

                                    let client_rate_limiter = CLIENT_RATE_LIMITER.get_or_init(client_rate_limiter_partition, || {

                                        #{ClientRateLimiter}::new(seconds_since_unix_epoch)

                                    });

                                    layer.store_put(client_rate_limiter);

                                }

                            }



                            // The token bucket is used for both standard AND adaptive retries.

                            let token_bucket_partition = #{TokenBucketPartition}::new(retry_partition);

                            let token_bucket = TOKEN_BUCKET.get_or_init(token_bucket_partition, #{TokenBucket}::default);

                            layer.store_put(token_bucket);

                            layer.set_retry_strategy(#{DynRetryStrategy}::new(#{StandardRetryStrategy}::new(&retry_config)));

                            """,

                            *codegenScope,
@@ -355,6 +433,10 @@ class ResiliencyReExportCustomization(private val runtimeConfig: RuntimeConfig) 
                "pub use #{types_retry}::{RetryConfig, RetryConfigBuilder, RetryMode};",

                "types_retry" to RuntimeType.smithyTypes(runtimeConfig).resolve("retry"),

            )

            rustTemplate(

                "pub use #{RetryPartition};",

                "RetryPartition" to RuntimeType.smithyRuntime(runtimeConfig).resolve("client::retries::RetryPartition"),

            )

        }

        rustCrate.withModule(ClientRustModule.Config.timeout) {

            rustTemplate(
@@ -364,3 +446,33 @@ class ResiliencyReExportCustomization(private val runtimeConfig: RuntimeConfig) 
        }

    }

}



class ResiliencyServiceRuntimePluginCustomization(codegenContext: ClientCodegenContext) : ServiceRuntimePluginCustomization() {

    private val runtimeConfig = codegenContext.runtimeConfig

    private val smithyRuntimeCrate = RuntimeType.smithyRuntime(runtimeConfig)

    private val retries = smithyRuntimeCrate.resolve("client::retries")

    private val codegenScope = arrayOf(

        "TokenBucket" to retries.resolve("TokenBucket"),

        "TokenBucketPartition" to retries.resolve("TokenBucketPartition"),

        "ClientRateLimiter" to retries.resolve("ClientRateLimiter"),

        "ClientRateLimiterPartition" to retries.resolve("ClientRateLimiterPartition"),

        "StaticPartitionMap" to smithyRuntimeCrate.resolve("static_partition_map::StaticPartitionMap"),

    )



    override fun section(section: ServiceRuntimePluginSection): Writable = writable {

        when (section) {

            is ServiceRuntimePluginSection.DeclareSingletons -> {

                // TODO(enableNewSmithyRuntimeCleanup) We can use the standard library's `OnceCell` once we upgrade the

                //    MSRV to 1.70

                rustTemplate(

                    """

                    static TOKEN_BUCKET: #{StaticPartitionMap}<#{TokenBucketPartition}, #{TokenBucket}> = #{StaticPartitionMap}::new();

                    static CLIENT_RATE_LIMITER: #{StaticPartitionMap}<#{ClientRateLimiterPartition}, #{ClientRateLimiter}> = #{StaticPartitionMap}::new();

                    """,

                    *codegenScope,

                )

            }

            else -> emptySection

        }

    }

}

codegen-client/src/main/kotlin/software/amazon/smithy/rust/codegen/client/smithy/customize/RequiredCustomizations.kt

+11 −0
Original line number Diff line number Diff line
@@ -16,9 +16,11 @@ import software.amazon.smithy.rust.codegen.client.smithy.customizations.Identity 
import software.amazon.smithy.rust.codegen.client.smithy.customizations.InterceptorConfigCustomization

import software.amazon.smithy.rust.codegen.client.smithy.customizations.ResiliencyConfigCustomization

import software.amazon.smithy.rust.codegen.client.smithy.customizations.ResiliencyReExportCustomization

import software.amazon.smithy.rust.codegen.client.smithy.customizations.ResiliencyServiceRuntimePluginCustomization

import software.amazon.smithy.rust.codegen.client.smithy.customizations.TimeSourceCustomization

import software.amazon.smithy.rust.codegen.client.smithy.customizations.TimeSourceOperationCustomization

import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization

import software.amazon.smithy.rust.codegen.client.smithy.generators.ServiceRuntimePluginCustomization

import software.amazon.smithy.rust.codegen.client.smithy.generators.config.ConfigCustomization

import software.amazon.smithy.rust.codegen.core.rustlang.Feature

import software.amazon.smithy.rust.codegen.core.smithy.RustCrate
@@ -95,4 +97,13 @@ class RequiredCustomizations : ClientCodegenDecorator { 
            }

        }

    }



    override fun serviceRuntimePluginCustomizations(

        codegenContext: ClientCodegenContext,

        baseCustomizations: List<ServiceRuntimePluginCustomization>,

    ): List<ServiceRuntimePluginCustomization> = if (codegenContext.smithyRuntimeMode.generateOrchestrator) {

        baseCustomizations + ResiliencyServiceRuntimePluginCustomization(codegenContext)

    } else {

        baseCustomizations

    }

}