Unverified Commit 0286b9fb authored by John DiSanti's avatar John DiSanti Committed by GitHub
Browse files

Fix S3 optional auth (#2907) 

## Motivation and Context
This PR implements a short-term solution for aws-sdk-rust#864 while a
long-term solution is worked out.

## Testing
- Tested manually against S3.
- Added DVR tests.

----

_By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice._
parent d7ffa893

CHANGELOG.next.toml

+6 −0
Original line number Diff line number Diff line
@@ -22,3 +22,9 @@ message = "`RuntimeComponents` and `RuntimeComponentsBuilder` are now re-exporte 
references = ["smithy-rs#2904"]

meta = { "breaking" = false, "tada" = false, "bug" = false, "target" = "client"}

author = "jdisanti"



[[aws-sdk-rust]]

message = "Fix requests to S3 with `no_credentials` set."

references = ["smithy-rs#2907", "aws-sdk-rust#864"]

meta = { "breaking" = false, "tada" = false, "bug" = true }

author = "jdisanti"

aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3Decorator.kt

+24 −0
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ import software.amazon.smithy.model.shapes.ServiceShape 
import software.amazon.smithy.model.shapes.Shape

import software.amazon.smithy.model.shapes.ShapeId

import software.amazon.smithy.model.shapes.StructureShape

import software.amazon.smithy.model.traits.OptionalAuthTrait

import software.amazon.smithy.model.transform.ModelTransformer

import software.amazon.smithy.rulesengine.traits.EndpointTestCase

import software.amazon.smithy.rulesengine.traits.EndpointTestOperationInput
@@ -34,6 +35,7 @@ import software.amazon.smithy.rust.codegen.core.smithy.protocols.ProtocolFunctio 
import software.amazon.smithy.rust.codegen.core.smithy.protocols.ProtocolMap

import software.amazon.smithy.rust.codegen.core.smithy.protocols.RestXml

import software.amazon.smithy.rust.codegen.core.smithy.traits.AllowInvalidXmlRoot

import software.amazon.smithy.rust.codegen.core.util.hasTrait

import software.amazon.smithy.rust.codegen.core.util.letIf

import software.amazon.smithy.rustsdk.getBuiltIn

import software.amazon.smithy.rustsdk.toWritable
@@ -82,6 +84,8 @@ class S3Decorator : ClientCodegenDecorator { 
                    },

                )::transform,

            )

            // enable optional auth for operations commonly used with public buckets

            .let(AddOptionalAuth()::transform)



    override fun endpointCustomizations(codegenContext: ClientCodegenContext): List<EndpointCustomization> {

        return listOf(
@@ -129,6 +133,26 @@ class FilterEndpointTests( 
    }

}



// TODO(P96049742): This model transform may need to change depending on if and how the S3 model is updated.

private class AddOptionalAuth {

    private val s3OptionalAuthOperations = listOf(

        ShapeId.from("com.amazonaws.s3#ListObjects"),

        ShapeId.from("com.amazonaws.s3#ListObjectsV2"),

        ShapeId.from("com.amazonaws.s3#HeadObject"),

        ShapeId.from("com.amazonaws.s3#GetObject"),

    )



    fun transform(model: Model) = ModelTransformer.create().mapShapes(model) { shape ->

        if (shape is OperationShape && s3OptionalAuthOperations.contains(shape.id) && !shape.hasTrait<OptionalAuthTrait>()) {

            shape.toBuilder()

                .addTrait(OptionalAuthTrait())

                .build()

        } else {

            shape

        }

    }

}



class S3ProtocolOverride(codegenContext: CodegenContext) : RestXml(codegenContext) {

    private val runtimeConfig = codegenContext.runtimeConfig

    private val errorScope = arrayOf(

aws/sdk/build.gradle.kts

+1 −1
Original line number Diff line number Diff line
@@ -338,7 +338,7 @@ tasks.register("generateCargoWorkspace") { 
    doFirst {

        outputDir.mkdirs()

        outputDir.resolve("Cargo.toml").writeText(generateCargoWorkspace(awsServices))

        rootProject.rootDir.resolve("clippy-root.toml").copyTo(outputDir.resolve("clippy.toml"))

        rootProject.rootDir.resolve("clippy-root.toml").copyTo(outputDir.resolve("clippy.toml"), overwrite = true)

    }

    inputs.property("servicelist", awsServices.moduleNames.toString())

    if (awsServices.examples.isNotEmpty()) {

aws/sdk/integration-tests/s3/tests/data/no_auth/get-object.json

0 → 100644
+85 −0
Original line number Diff line number Diff line 
{

  "events": [

    {

      "connection_id": 0,

      "action": {

        "Request": {

          "request": {

            "uri": "https://gdc-organoid-pancreatic-phs001611-2-open.s3.us-east-1.amazonaws.com/0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz?x-id=GetObject",

            "headers": {

              "amz-sdk-request": [

                "attempt=1; max=3"

              ],

              "user-agent": [

                "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0"

              ],

              "x-amz-user-agent": [

                "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0"

              ]

            },

            "method": "GET"

          }

        }

      }

    },

    {

      "connection_id": 0,

      "action": {

        "Eof": {

          "ok": true,

          "direction": "Request"

        }

      }

    },

    {

      "connection_id": 0,

      "action": {

        "Response": {

          "response": {

            "Ok": {

              "status": 200,

              "version": "HTTP/1.1",

              "headers": {

                "content-type": [

                  "binary/octet-stream"

                ],

                "x-amz-id-2": [

                  "mO5q2ZSztYdEU923Zi5sHNctHwRRzOyngQEWsZWHwOJEgxrj9dw0KH0IVovTxu2Y8V0ps5z4KMQ="

                ],

                "content-length": [

                  "386910"

                ],

                "accept-ranges": [

                  "bytes"

                ],

                "x-amz-server-side-encryption": [

                  "AES256"

                ],

                "x-amz-request-id": [

                  "EGGB3A7GXR9YWDYM"

                ],

                "last-modified": [

                  "Mon, 27 Jan 2020 20:56:51 GMT"

                ],

                "date": [

                  "Mon, 07 Aug 2023 20:44:42 GMT"

                ],

                "x-amz-meta-description": [

                  "{\"url\": \"s3://cleversafe.service.consul/stage-submission-5/ORGANOID-PANCREATIC/0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz\", \"node_id\": \"0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz\"}"

                ],

                "server": [

                  "AmazonS3"

                ],

                "etag": [

                  "\"446fc665f99183cd0540d7656a79d3ed\""

                ]

              }

            }

          }

        }

      }

    }

  ],

  "docs": "traffic recording of optional auth (no Authorization header is included)",

  "version": "V0"

}

aws/sdk/integration-tests/s3/tests/data/no_auth/head-object.json

0 → 100644
+94 −0
Original line number Diff line number Diff line 
{

  "events": [

    {

      "connection_id": 0,

      "action": {

        "Request": {

          "request": {

            "uri": "https://gdc-organoid-pancreatic-phs001611-2-open.s3.us-east-1.amazonaws.com/0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz",

            "headers": {

              "user-agent": [

                "aws-sdk-rust/0.123.test os/windows/XPSP3 lang/rust/1.50.0"

              ],

              "amz-sdk-request": [

                "attempt=1; max=3"

              ],

              "x-amz-user-agent": [

                "aws-sdk-rust/0.123.test api/test-service/0.123 os/windows/XPSP3 lang/rust/1.50.0"

              ]

            },

            "method": "HEAD"

          }

        }

      }

    },

    {

      "connection_id": 0,

      "action": {

        "Eof": {

          "ok": true,

          "direction": "Request"

        }

      }

    },

    {

      "connection_id": 0,

      "action": {

        "Response": {

          "response": {

            "Ok": {

              "status": 200,

              "version": "HTTP/1.1",

              "headers": {

                "last-modified": [

                  "Mon, 27 Jan 2020 20:56:51 GMT"

                ],

                "content-type": [

                  "binary/octet-stream"

                ],

                "date": [

                  "Mon, 07 Aug 2023 20:44:42 GMT"

                ],

                "server": [

                  "AmazonS3"

                ],

                "content-length": [

                  "386910"

                ],

                "accept-ranges": [

                  "bytes"

                ],

                "x-amz-server-side-encryption": [

                  "AES256"

                ],

                "x-amz-id-2": [

                  "+d6tSM3krTTrvY+y6PFHnkw9OhAtJhQy8RzFrPO6vnUOIuvqViB9gFZvfJCcVMj7gX+dpIvZ3HI="

                ],

                "x-amz-request-id": [

                  "EGGF3G9KFMFHZ3E0"

                ],

                "etag": [

                  "\"446fc665f99183cd0540d7656a79d3ed\""

                ],

                "x-amz-meta-description": [

                  "{\"url\": \"s3://cleversafe.service.consul/stage-submission-5/ORGANOID-PANCREATIC/0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz\", \"node_id\": \"0431cddc-a418-4a79-a34d-6c041394e8e4/a6ddcc84-8e4d-4c68-885c-2d51168eec97.FPKM-UQ.txt.gz\"}"

                ]

              }

            }

          }

        }

      }

    },

    {

      "connection_id": 0,

      "action": {

        "Eof": {

          "ok": true,

          "direction": "Response"

        }

      }

    }

  ],

  "docs": "traffic recording of optional auth (no Authorization header is included)",

  "version": "V0"

}