Unverified Commit 21ffa905 authored by Russell Cohen's avatar Russell Cohen Committed by GitHub
Browse files

Fix paginator bug where `None` was returned immediately (#1083) 



* Fix paginator bug where `None` was returned immediately

The escape hatch added by aws-sdk-rust#391 did not properly handle the case where the first response was `None` and _not_ the empty string. This diff:
- Checks for emptiness for both maps and strings
- Fixes the check so that an initial `None` input does not cause an incorrect paginator error

* Update changelog

* Apply suggestions from code review

Co-authored-by: default avatarZelda Hessler <zhessler@amazon.com>

* rustfmt

Co-authored-by: default avatarZelda Hessler <zhessler@amazon.com>
parent 5eb8d4ce

CHANGELOG.next.toml

+12 −0
Original line number Diff line number Diff line
@@ -52,3 +52,15 @@ message = "Silence profile credential warnings in Lambda environment" 
references = ["smithy-rs#1065", "aws-sdk-rust#398"]

meta = { "breaking" = false, "tada" = true, "bug" = true }

author = "nmoutschen"



[[aws-sdk-rust]]

message = "Fixed paginator bug impacting EC2 describe VPCs (and others)"

references = ["aws-sdk-rust#405", "smithy-rs#1083"]

meta = { "breaking" = false, "tada" = false, "bug" = true }

author = "rcoh"



[[smithy-rs]]

message = "Fixed paginator bug impacting EC2 describe VPCs (and others)"

references = ["aws-sdk-rust#405", "smithy-rs#1083"]

meta = { "breaking" = false, "tada" = false, "bug" = true }

author = "rcoh"

aws/sdk/integration-tests/ec2/tests/paginators.rs

+36 −0
Original line number Diff line number Diff line
@@ -50,3 +50,39 @@ async fn paginators_handle_empty_tokens() { 
    assert_eq!(first_item, None);

    conn.assert_requests_match(&[]);

}



/// See https://github.com/awslabs/aws-sdk-rust/issues/405

///

/// EC2 can also reply with the token truly unset which will be interpreted as `None`

#[tokio::test]

async fn paginators_handle_unset_tokens() {

    let request= "Action=DescribeSpotPriceHistory&Version=2016-11-15&AvailabilityZone=eu-north-1a&InstanceType.1=g5.48xlarge&ProductDescription.1=Linux%2FUNIX";

    let response = r#"<?xml version="1.0" encoding="UTF-8"?>

        <DescribeSpotPriceHistoryResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">

            <requestId>edf3e86c-4baf-47c1-9228-9a5ea09542e8</requestId>

            <spotPriceHistorySet/>

        </DescribeSpotPriceHistoryResponse>"#;

    let conn = TestConnection::<&str>::new(vec![(

        http::Request::builder()

            .uri("https://ec2.us-east-1.amazonaws.com/")

            .body(request.into())

            .unwrap(),

        http::Response::builder()

            .status(200)

            .body(response)

            .unwrap(),

    )]);

    let client = Client::from_conf_conn(stub_config(), conn.clone());

    let instance_type = InstanceType::from("g5.48xlarge");

    let mut paginator = client

        .describe_spot_price_history()

        .instance_types(instance_type)

        .product_descriptions("Linux/UNIX")

        .availability_zone("eu-north-1a")

        .into_paginator()

        .items()

        .send();

    let first_item = paginator.try_next().await.expect("success");

    assert_eq!(first_item, None);

    conn.assert_requests_match(&[]);

}

codegen/src/main/kotlin/software/amazon/smithy/rust/codegen/smithy/generators/PaginatorGenerator.kt

+4 −10
Original line number Diff line number Diff line
@@ -9,7 +9,6 @@ import software.amazon.smithy.model.Model 
import software.amazon.smithy.model.knowledge.PaginatedIndex

import software.amazon.smithy.model.shapes.OperationShape

import software.amazon.smithy.model.shapes.ServiceShape

import software.amazon.smithy.model.shapes.StringShape

import software.amazon.smithy.model.traits.IdempotencyTokenTrait

import software.amazon.smithy.model.traits.PaginatedTrait

import software.amazon.smithy.rust.codegen.rustlang.CargoDependency
@@ -175,12 +174,13 @@ class PaginatorGenerator private constructor( 
                            let done = match resp {

                                Ok(ref resp) => {

                                    let new_token = #{output_token}(resp);

                                    if new_token == input.$inputTokenMember.as_ref() {

                                    let is_empty = ${nextTokenEmpty("new_token")};

                                    if !is_empty && new_token == input.$inputTokenMember.as_ref() {

                                        let _ = tx.send(Err(#{SdkError}::ConstructionFailure("next token did not change, aborting paginator. This indicates an SDK or AWS service bug.".into()))).await;

                                        return;

                                    }

                                    input.$inputTokenMember = new_token.cloned();

                                    ${nextTokenEmpty("input.$inputTokenMember")}

                                    is_empty

                                },

                                Err(_) => true,

                            };
@@ -192,7 +192,6 @@ class PaginatorGenerator private constructor( 
                                return

                            }

                        }



                    }))

                }

            }
@@ -276,12 +275,7 @@ class PaginatorGenerator private constructor( 
    }



    private fun nextTokenEmpty(token: String): String {

        val tokenType = model.expectShape(paginationInfo.outputTokenMemberPath.last().target)

        if (tokenType is StringShape) {

            return "$token.as_deref().unwrap_or_default().is_empty()"

        } else {

            return "$token.is_none()"

        }

        return "$token.map(|token|token.is_empty()).unwrap_or(true)"

    }



    private fun pageSizeSetter() = writable {

tools/ci-cdk/canary-lambda/src/paginator_canary.rs

+14 −2
Original line number Diff line number Diff line
@@ -40,9 +40,21 @@ pub async fn paginator_canary(client: ec2::Client, page_size: usize) -> anyhow:: 
        num_pages += 1;

    }

    if dbg!(num_pages) < 2 {

        bail!("should be ~60 of pages of results")

        bail!(

            "expected 3+ pages containing ~60 results but got {} pages",

            num_pages

        )

    }



    // https://github.com/awslabs/aws-sdk-rust/issues/405

    let _ = client

        .describe_vpcs()

        .into_paginator()

        .items()

        .send()

        .collect::<Result<Vec<_>, _>>()

        .await?;



    Ok(())

}
@@ -54,6 +66,6 @@ mod test { 
    async fn test_paginator() {

        let conf = aws_config::load_from_env().await;

        let client = aws_sdk_ec2::Client::new(&conf);

        paginator_canary(client).await.unwrap()

        paginator_canary(client, 20).await.unwrap()

    }

}