Unverified Commit a77c5ca2 authored by Russell Cohen's avatar Russell Cohen Committed by GitHub
Browse files

Add support for Paginators (#1006) 



* Add Support for Paginators

This commit adds pagination support for both clients. It does this via the `FnStream` abstraction which combines a rendezvous-channel with a closure to produce a stream. When the stream is polled, the underlying closure is polled to advance the computation which then pushes data into the channel.

Two paginators are generated:
1. `.paginate()` which produces a stream of `Result<Page, Err>`
2. `.paginate().items()` which produces a stream of `Result<Item, Err>` where items are flattened from the individual pages. This uses `TryFlatMap` to support conveniently generating a flat page iterator.

In addition, docs are generated to point customers towards the paginators.

* Add RFC, more tests

* backout unrelated changes

* Fix paginators for glacier

* Fix clippy error

* Add test for paginators that point to maps & fix apigateway paginator

* Fix docs

* remove extraeneous println

* Fix DynamoDB examples

* Disallow paginating operations with the idempotency token trait

* Apply suggestions from code review

misc typos / cleanups

Co-authored-by: default avatarZelda Hessler <zhessler@amazon.com>

* CR feedback

* Fix typo

* Update changelog

* Apply suggestions from code review

Co-authored-by: default avatarJohn DiSanti <jdisanti@amazon.com>

* CR feedback round 1

* Rename paginate to into_paginator()

* update pr bot comment

Co-authored-by: default avatarZelda Hessler <zhessler@amazon.com>
Co-authored-by: default avatarJohn DiSanti <jdisanti@amazon.com>
parent 76606dd0

.github/workflows/pull-request-bot.yml

+4 −4
Original line number Diff line number Diff line
@@ -104,11 +104,11 @@ jobs: 
        toolchain: ${{ env.rust_version }}

        default: true

    - name: Generate doc preview

      # Only generate two of the smallest services since these get huge. One of these must be

      # STS since aws-config depends on it. STS and Transcribe Streaming were chosen below to stay

      # small while still representing most features. Combined, they are about 11 MB at time of writing.

      # Only generate three of the smallest services since the doc build can be very large. One of these must be

      # STS since aws-config depends on it. STS and Transcribe Streaming and DynamoDB (paginators/waiters) were chosen

      # below to stay small while still representing most features. Combined, they are about ~20MB at time of writing.

      run: |

        ./gradlew -Paws.services=+sts,+transcribestreaming :aws:sdk:assemble

        ./gradlew -Paws.services=+sts,+transcribestreaming,+dynamodb :aws:sdk:assemble



        # Copy the Server runtime crate(s) in

        cp -r rust-runtime/aws-smithy-http-server aws/sdk/build/aws-sdk/sdk

CHANGELOG.next.toml

+12 −0
Original line number Diff line number Diff line
@@ -99,6 +99,18 @@ meta = { "breaking" = false, "tada" = false, "bug" = false } 
references = ["smithy-rs#1002", "aws-sdk-rust#352"]

author = "rcoh"



[[smithy-rs]]

message = "Add support for paginators! Paginated APIs now include `.into_paginator()` and (when supported) `.into_paginator().items()` to enable paginating responses automatically. The paginator API should be considered in preview and is subject to change pending customer feedback."

meta = { "breaking" = false, "tada" = true, "bug" = false }

references = ["aws-sdk-rust#47", "smithy-rs#1006"]

author = "rcoh"



[[aws-sdk-rust]]

message = "Add support for paginators! Paginated APIs now include `.into_paginator()` and (when supported) `.into_paginator().items()` to enable paginating responses automatically. The paginator API should be considered in preview and is subject to change pending customer feedback."

meta = { "breaking" = false, "tada" = true, "bug" = false }

references = ["aws-sdk-rust#47", "smithy-rs#1006"]

author = "rcoh"



[[aws-sdk-rust]]

message = "Example for Config builder region function added"

references = ["smithy-rs#670"]

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsFluentClientDecorator.kt

+13 −22
Original line number Diff line number Diff line
@@ -10,8 +10,6 @@ import software.amazon.smithy.model.traits.TitleTrait 
import software.amazon.smithy.rust.codegen.rustlang.CargoDependency

import software.amazon.smithy.rust.codegen.rustlang.DependencyScope

import software.amazon.smithy.rust.codegen.rustlang.Feature

import software.amazon.smithy.rust.codegen.rustlang.RustMetadata

import software.amazon.smithy.rust.codegen.rustlang.RustModule

import software.amazon.smithy.rust.codegen.rustlang.RustWriter

import software.amazon.smithy.rust.codegen.rustlang.Writable

import software.amazon.smithy.rust.codegen.rustlang.asType
@@ -52,14 +50,6 @@ class AwsFluentClientDecorator : RustCodegenDecorator { 


    override fun extras(codegenContext: CodegenContext, rustCrate: RustCrate) {

        val types = Types(codegenContext.runtimeConfig)

        val module = RustMetadata(public = true)

        rustCrate.withModule(

            RustModule(

                "client",

                module,

                documentation = "Client and fluent builders for calling the service."

            )

        ) { writer ->

        FluentClientGenerator(

            codegenContext,

            generics = ClientGenerics(
@@ -72,7 +62,8 @@ class AwsFluentClientDecorator : RustCodegenDecorator { 
                AwsPresignedFluentBuilderMethod(codegenContext.runtimeConfig),

                AwsFluentClientDocs(codegenContext)

            )

            ).render(writer)

        ).render(rustCrate)

        rustCrate.withModule(FluentClientGenerator.clientModule) { writer ->

            AwsFluentClientExtensions(types).render(writer)

        }

        val awsSmithyClient = "aws-smithy-client"

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/IntegrationTestDependencies.kt

+1 −0
Original line number Diff line number Diff line
@@ -60,6 +60,7 @@ class IntegrationTestDependencies( 
                val smithyClient = CargoDependency.SmithyClient(runtimeConfig)

                    .copy(features = setOf("test-util"), scope = DependencyScope.Dev)

                addDependency(smithyClient)

                addDependency(CargoDependency.SmithyProtocolTestHelpers(runtimeConfig))

                addDependency(SerdeJson)

                addDependency(Tokio)

                addDependency(FuturesUtil)

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/auth/DisabledAuthDecorator.kt

+1 −1
Original line number Diff line number Diff line
@@ -38,7 +38,7 @@ class DisabledAuthDecorator : RustCodegenDecorator { 
        val optionalOperations = optionalAuth[service.id]!!

        return ModelTransformer.create().mapShapes(model) {

            if (optionalOperations.contains(it.id) && it is OperationShape) {

                it.toBuilder().addTrait(AuthTrait(listOf())).build()

                it.toBuilder().addTrait(AuthTrait(setOf())).build()

            } else {

                it

            }