fix: Update storage module to work with Minio

Minio provides object storage with an S3-compatible API (mostly). This PR allows overriding the S3 endpoint and enabling path-style requests, both of which are needed to configure the use of a Minio server.
ExpediaGroup · Feb 16, 2022 · 6d0ba6c · 6d0ba6c
1 parent 6fc1dc1
commit 6d0ba6c
Show file tree

Hide file tree

Showing 4 changed files with 13 additions and 3 deletions.
diff --git a/packages/backend/env/.env b/packages/backend/env/.env
@@ -43,6 +43,8 @@ DB_DEBUG=false
 # Object Storage
 S3_BUCKET=
 S3_REGION=
+S3_ENDPOINT=
+S3_FORCE_PATH_STYLE=
 S3_CONCURRENCY_LIMIT=10
 
 # Okta

diff --git a/packages/backend/src/environment.ts b/packages/backend/src/environment.ts
@@ -41,5 +41,5 @@ if (result.error) {
 
 // Configuration validation checks
 if (process.env.GITHUB_USE_WEBHOOK === 'true' && process.env.PUBLIC_URL === '') {
-  logger.error('PUBLIC_URL must be set when GITHUB_USE_WEBHOOK is true');
+  logger.error('[ENV] Configuration Error: PUBLIC_URL must be set when GITHUB_USE_WEBHOOK is true');
 }
diff --git a/packages/backend/src/lib/storage.ts b/packages/backend/src/lib/storage.ts
@@ -22,7 +22,13 @@ import type { ReadStream } from 'fs-extra';
 
 const defaultOptions: S3.Types.ClientConfiguration = {
   region: process.env.S3_REGION,
-  maxRetries: 3
+  maxRetries: 3,
+
+  endpoint: process.env.S3_ENDPOINT !== '' ? process.env.S3_ENDPOINT : undefined,
+
+  // S3 Path-style requests are deprecated
+  // But some S3-compatible APIs may use them (e.g. Minio)
+  s3ForcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true' ? true : undefined
 };
 
 export function createS3Client(options: S3.Types.ClientConfiguration = defaultOptions): S3 {
@@ -64,7 +70,7 @@ export async function streamToS3(stream: ReadStream, fileSize: number, key: stri
   const bucket = process.env.S3_BUCKET!;
 
   const response = await defaultS3Client
-    .putObject({ Body: stream, Bucket: bucket, ContentLength: fileSize, Key: key })
+    .upload({ Body: stream, Bucket: bucket, ContentLength: fileSize, Key: key })
     .promise();
   const uri = `s3://${bucket}/${key}`;
 

diff --git a/packages/backend/src/types/process-env.d.ts b/packages/backend/src/types/process-env.d.ts
@@ -64,6 +64,8 @@ declare namespace NodeJS {
     // Object Storage
     S3_BUCKET: string;
     S3_REGION: string;
+    S3_ENDPOINT?: string;
+    S3_FORCE_PATH_STYLE?: string;
     S3_CONCURRENCY_LIMIT: string;
 
     // Okta