docs(supplemental-docs): md5 checksum fallback for s3 (#6886)

siddsriv · web-flow · commit a1d8ad3ef4c5 · 2025-02-17T11:54:37.000-05:00
* docs(supplemental-docs): md5 checksum fallback for s3

* test(middleware-flexible-checksums): add e2e test for md5 fallback

* docs(supplemental-docs): version update for JSv3

* test(middleware-flexible-checksums): context command update

* docs(supplemental-docs): line-wrap md

* test(middleware-flexible-checksums): type fix

* chore(middleware-flexible-checksums): supplemental docs link
diff --git a/packages/middleware-flexible-checksums/src/middleware-md5-fallback.e2e.spec.ts b/packages/middleware-flexible-checksums/src/middleware-md5-fallback.e2e.spec.ts
@@ -0,0 +1,140 @@
+// see supplemental-docs/MD5_FALLBACK for more details
+import {
+  CreateBucketCommand,
+  DeleteBucketCommand,
+  DeleteObjectsCommand,
+  PutObjectCommand,
+  S3,
+  ServiceInputTypes,
+  ServiceOutputTypes,
+} from "@aws-sdk/client-s3";
+import type {
+  FinalizeHandler,
+  FinalizeHandlerArguments,
+  FinalizeHandlerOutput,
+  HandlerExecutionContext,
+  HttpRequest,
+} from "@smithy/types";
+import { createHash } from "crypto";
+import { afterAll, beforeAll, describe, expect, test as it } from "vitest";
+
+describe("S3 MD5 Fallback for DeleteObjects", () => {
+  let s3: S3;
+  let Bucket: string;
+  const testFiles = ["md5-test-1.txt", "md5-test-2.txt"];
+
+  beforeAll(async () => {
+    s3 = new S3({ region: "us-west-2" });
+    Bucket = `md5-fallback-test-${Date.now()}`;
+
+    try {
+      await s3.send(new CreateBucketCommand({ Bucket }));
+      await new Promise((resolve) => setTimeout(resolve, 2000));
+
+      for (const Key of testFiles) {
+        await s3.send(
+          new PutObjectCommand({
+            Bucket,
+            Key,
+            Body: "test content",
+          })
+        );
+      }
+    } catch (err) {
+      console.error("Setup failed:", err);
+      throw err;
+    }
+  });
+
+  afterAll(async () => {
+    try {
+      await s3.send(
+        new DeleteObjectsCommand({
+          Bucket,
+          Delete: {
+            Objects: testFiles.map((Key) => ({ Key })),
+          },
+        })
+      );
+      await s3.send(new DeleteBucketCommand({ Bucket }));
+    } catch (error) {
+      console.error("Cleanup failed:", error);
+    }
+  });
+
+  it("should use CRC32 checksum by default for DeleteObjects", async () => {
+    const response = await s3.send(
+      new DeleteObjectsCommand({
+        Bucket,
+        Delete: {
+          Objects: [{ Key: testFiles[0] }],
+        },
+      })
+    );
+
+    // operation successfully deleted exactly one object (CRC32 being used)
+    expect(response.Deleted?.length).toBe(1);
+  });
+
+  it("should use MD5 checksum for DeleteObjects with middleware", async () => {
+    const md5S3Client = new S3({ region: "us-west-2" });
+    let md5Added = false;
+    let crc32Removed = false;
+
+    md5S3Client.middlewareStack.add(
+      (next: FinalizeHandler<ServiceInputTypes, ServiceOutputTypes>, context: HandlerExecutionContext) =>
+        async (
+          args: FinalizeHandlerArguments<ServiceInputTypes>
+        ): Promise<FinalizeHandlerOutput<ServiceOutputTypes>> => {
+          const request = args.request as HttpRequest;
+          const isDeleteObjects = context.commandName === "DeleteObjectsCommand";
+
+          if (!isDeleteObjects) {
+            return next(args);
+          }
+
+          const result = await next(args);
+          const headers = request.headers;
+
+          // Remove checksum headers
+          Object.keys(headers).forEach((header) => {
+            if (
+              header.toLowerCase().startsWith("x-amz-checksum-") ||
+              header.toLowerCase().startsWith("x-amz-sdk-checksum-")
+            ) {
+              delete headers[header];
+              crc32Removed = true;
+            }
+          });
+
+          // Add MD5
+          if (request.body) {
+            const bodyContent = Buffer.from(request.body);
+            const md5Hash = createHash("md5").update(bodyContent).digest("base64");
+            headers["Content-MD5"] = md5Hash;
+            md5Added = true;
+          }
+
+          return result;
+        },
+      {
+        step: "finalizeRequest",
+        name: "addMD5Checksum",
+      }
+    );
+
+    const response = await md5S3Client.send(
+      new DeleteObjectsCommand({
+        Bucket,
+        Delete: {
+          Objects: [{ Key: testFiles[1] }],
+        },
+      })
+    );
+
+    // If MD5 wasn't properly set, this call will fail
+    expect(response.Deleted?.length).toBe(1);
+    expect(md5Added).toBe(true);
+    expect(crc32Removed).toBe(true);
+  });
+});
diff --git a/supplemental-docs/MD5_FALLBACK.md b/supplemental-docs/MD5_FALLBACK.md
@@ -0,0 +1,125 @@
+# MD5 Checksum Fallback for AWS SDK for JavaScript v3
+
+## Background
+
+In AWS SDK for JavaScript [v3.729.0](https://github.com/aws/aws-sdk-js-v3/releases/tag/v3.729.0),
+we shipped a feature that [changed default object integrity in
+S3](https://github.com/aws/aws-sdk-js-v3/issues/6810). The SDKs now default to using more modern
+checksums (like CRC32) to ensure object integrity, whereas previously MD5 checksums were being used.
+Some third-party S3-compatible services currently do not support these checksums. To our knowledge,
+this affects only the S3 `DeleteObjects` operation.
+
+If you wish to fallback to the old behavior of sending MD5 checksums, for operations like
+`DeleteObjectsCommand` this is how you can do it in AWS SDK for JavaScript v3:
+
+## MD5 fallback
+
+The following code provides a custom S3 client that will use MD5 checksums for DeleteObjects
+operations while maintaining the default behavior for all other operations.
+
+```javascript
+// md5ClientS3.mjs
+import { S3Client } from "@aws-sdk/client-s3";
+import { createHash } from "crypto";
+
+/**
+ * Creates an S3 client that uses MD5 checksums for DeleteObjects operations
+ */
+export function createS3ClientWithMD5() {
+  const client = new S3Client({});
+  const md5Hash = createHash("md5");
+
+  client.middlewareStack.add(
+    (next, context) => async (args) => {
+      // Check if this is a DeleteObjects command
+      const isDeleteObjects = context.commandName === "DeleteObjectsCommand";
+
+      if (!isDeleteObjects) {
+        return next(args);
+      }
+
+      const result = await next(args);
+      const headers = args.request.headers;
+
+      // Remove any checksum headers
+      Object.keys(headers).forEach((header) => {
+        if (
+          header.toLowerCase().startsWith("x-amz-checksum-") ||
+          header.toLowerCase().startsWith("x-amz-sdk-checksum-")
+        ) {
+          delete headers[header];
+        }
+      });
+
+      // Add MD5
+      if (args.request.body) {
+        const bodyContent = Buffer.from(args.request.body);
+        headers["Content-MD5"] = md5Hash.update(bodyContent).digest("base64");
+      }
+
+      return result;
+    },
+    {
+      step: "finalizeRequest",
+      name: "addMD5Checksum",
+    }
+  );
+
+  return client;
+}
+```
+
+## Usage
+
+Instead of creating a regular S3 client, use the `createS3ClientWithMD5` function:
+
+```javascript
+import { DeleteObjectsCommand } from "@aws-sdk/client-s3";
+import { createS3ClientWithMD5 } from "./md5ClientS3.mjs";
+
+// Create the client with MD5 support
+const client = createS3ClientWithMD5();
+
+// Use it like a normal S3 client
+const deleteParams = {
+  Bucket: "your-bucket",
+  Delete: {
+    Objects: [{ Key: "file1.txt" }, { Key: "file2.txt" }],
+  },
+};
+
+try {
+  const response = await client.send(new DeleteObjectsCommand(deleteParams));
+  console.log("Successfully deleted objects:", response);
+} catch (err) {
+  console.error("Error:", err);
+}
+```
+
+## How It Works
+
+The solution adds middleware to the S3 client that:
+
+1. Detects DeleteObjects operations using the command name
+2. Lets the SDK add its default headers
+3. Removes any checksum headers in the finalizeRequest step
+4. Calculates an MD5 hash of the request body
+5. Adds the MD5 hash as a Content-MD5 header
+
+This sequence ensures that we properly replace the checksums with MD5 checksum.
+
+## Usage Notes
+
+- The client can be configured with additional options as needed (region, credentials, etc.)
+- If your S3-compatible service supports the SDK's new checksum options or adds support in the
+  future, you should use the standard S3 client instead.
+
+## Debugging
+
+To verify that the MD5 checksum is being correctly applied, you can add console logging to the
+middleware by modifying the code to include logging statements:
+
+```javascript
+// Inside the middleware function, add:
+console.log("Headers:", JSON.stringify(args.request.headers, null, 2));
+```
