From 2b9b1e69f07494f6b99d1d5f73620ee447f4d156 Mon Sep 17 00:00:00 2001 From: Marko Malenic Date: Wed, 3 Jul 2024 14:59:37 +1000 Subject: [PATCH] docs(filemanager): add docs for API endpoints, tidy routes and bump deps --- .../api-gateway-proxy-integration/index.ts | 2 +- lib/workload/components/api-gateway/index.ts | 11 +- .../stateless/stacks/filemanager/Cargo.lock | 642 ++++++++++++++++-- .../stateless/stacks/filemanager/README.md | 28 +- .../deploy/constructs/functions/api.ts | 26 + .../deploy/constructs/functions/function.ts | 5 + .../deploy/constructs/functions/query.ts | 17 - .../stacks/filemanager/deploy/stack.ts | 10 +- .../filemanager-api-lambda/Cargo.toml | 4 +- .../filemanager-api-lambda/README.md | 14 +- .../filemanager-api-lambda/src/main.rs | 4 +- .../filemanager-api-server/Cargo.toml | 3 +- .../filemanager-api-server/README.md | 8 +- .../filemanager-api-server/src/main.rs | 20 +- .../filemanager/filemanager-build/README.md | 24 +- .../filemanager-ingest-lambda/Cargo.toml | 2 +- .../filemanager-inventory-lambda/Cargo.toml | 2 +- .../filemanager-migrate-lambda/Cargo.toml | 2 +- .../stacks/filemanager/filemanager/Cargo.toml | 10 +- .../stacks/filemanager/filemanager/README.md | 17 +- .../filemanager/filemanager/src/routes/get.rs | 4 +- .../filemanager/src/routes/ingest.rs | 2 +- .../filemanager/src/routes/list.rs | 8 +- .../filemanager/filemanager/src/routes/mod.rs | 8 +- .../filemanager/src/routes/openapi.rs | 8 +- .../deploy/construct/lambda-api/index.ts | 7 +- .../sequence-run-manager/deploy/stack.ts | 4 +- .../stacks/workflow-manager/deploy/stack.ts | 14 +- skel/django-api/deploy/stack.ts | 6 +- 29 files changed, 755 insertions(+), 157 deletions(-) create mode 100644 lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/api.ts delete mode 100644 lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/query.ts diff --git a/lib/workload/components/api-gateway-proxy-integration/index.ts b/lib/workload/components/api-gateway-proxy-integration/index.ts index c6bdc04f6..c0b492c79 100644 --- a/lib/workload/components/api-gateway-proxy-integration/index.ts +++ b/lib/workload/components/api-gateway-proxy-integration/index.ts @@ -30,7 +30,7 @@ export class ApiGatewayProxyIntegration extends Construct { this.apiGateway = new ApiGatewayConstruct(this, 'ApiGateway', props.apiGatewayProps); const integration = new HttpLambdaIntegration('ApiLambdaIntegration', props.handler); - new HttpRoute(scope, id, { + new HttpRoute(this, 'ApiLambdaRoute', { httpApi: this.apiGateway.httpApi, integration, routeKey: HttpRouteKey.with('/{proxy+}', HttpMethod.ANY), diff --git a/lib/workload/components/api-gateway/index.ts b/lib/workload/components/api-gateway/index.ts index 3f585c31b..65887961f 100644 --- a/lib/workload/components/api-gateway/index.ts +++ b/lib/workload/components/api-gateway/index.ts @@ -1,22 +1,13 @@ import { Construct } from 'constructs'; import { aws_ssm, Duration, RemovalPolicy } from 'aws-cdk-lib'; import { HttpJwtAuthorizer } from 'aws-cdk-lib/aws-apigatewayv2-authorizers'; -import { - CorsHttpMethod, - HttpApi, - CfnStage, - DomainName, - HttpRoute, - HttpRouteKey, - HttpMethod, -} from 'aws-cdk-lib/aws-apigatewayv2'; +import { CorsHttpMethod, HttpApi, CfnStage, DomainName } from 'aws-cdk-lib/aws-apigatewayv2'; import { Certificate } from 'aws-cdk-lib/aws-certificatemanager'; import { IStringParameter, StringParameter } from 'aws-cdk-lib/aws-ssm'; import { LogGroup, RetentionDays } from 'aws-cdk-lib/aws-logs'; import { ARecord, HostedZone, RecordTarget } from 'aws-cdk-lib/aws-route53'; import { Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; import { ApiGatewayv2DomainProperties } from 'aws-cdk-lib/aws-route53-targets'; -import { HttpLambdaIntegration } from 'aws-cdk-lib/aws-apigatewayv2-integrations'; export interface ApiGwLogsConfig { /** diff --git a/lib/workload/stateless/stacks/filemanager/Cargo.lock b/lib/workload/stateless/stacks/filemanager/Cargo.lock index 91b67a75c..676148d23 100644 --- a/lib/workload/stateless/stacks/filemanager/Cargo.lock +++ b/lib/workload/stateless/stacks/filemanager/Cargo.lock @@ -157,6 +157,12 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + [[package]] name = "arrayvec" version = "0.7.4" @@ -165,9 +171,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c" +checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" dependencies = [ "arrow-arith", "arrow-array", @@ -186,9 +192,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705" +checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" dependencies = [ "arrow-array", "arrow-buffer", @@ -201,16 +207,16 @@ dependencies = [ [[package]] name = "arrow-array" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" +checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97" dependencies = [ "ahash 0.8.11", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "chrono-tz", + "chrono-tz 0.9.0", "half", "hashbrown 0.14.5", "num", @@ -218,9 +224,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" +checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1" dependencies = [ "bytes", "half", @@ -229,28 +235,30 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" +checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "base64 0.21.7", + "atoi", + "base64 0.22.1", "chrono", "comfy-table", "half", "lexical-core", "num", + "ryu", ] [[package]] name = "arrow-csv" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46af72211f0712612f5b18325530b9ad1bfbdc87290d5fbfd32a7da128983781" +checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" dependencies = [ "arrow-array", "arrow-buffer", @@ -267,9 +275,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" +checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10" dependencies = [ "arrow-buffer", "arrow-schema", @@ -279,9 +287,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" +checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -289,13 +297,14 @@ dependencies = [ "arrow-data", "arrow-schema", "flatbuffers", + "lz4_flex", ] [[package]] name = "arrow-json" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8950719280397a47d37ac01492e3506a8a724b3fb81001900b866637a829ee0f" +checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -313,9 +322,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be" +checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" dependencies = [ "arrow-array", "arrow-buffer", @@ -328,9 +337,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" +checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -343,15 +352,15 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" +checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393" [[package]] name = "arrow-select" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" +checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -363,15 +372,16 @@ dependencies = [ [[package]] name = "arrow-string" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7" +checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "memchr", "num", "regex", "regex-syntax 0.8.3", @@ -388,6 +398,24 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-compression" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +dependencies = [ + "bzip2", + "flate2", + "futures-core", + "futures-io", + "memchr", + "pin-project-lite", + "tokio", + "xz2", + "zstd 0.13.0", + "zstd-safe 7.0.0", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -1051,6 +1079,28 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "blake3" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -1086,9 +1136,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.5.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1097,9 +1147,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.1" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1158,6 +1208,27 @@ dependencies = [ "either", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.98" @@ -1203,7 +1274,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", - "chrono-tz-build", + "chrono-tz-build 0.2.1", + "phf", +] + +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build 0.3.0", "phf", ] @@ -1218,6 +1300,17 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "clap" version = "4.5.7" @@ -1301,6 +1394,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + [[package]] name = "core-foundation" version = "0.9.4" @@ -1468,6 +1567,322 @@ dependencies = [ "syn 2.0.65", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068" +dependencies = [ + "ahash 0.8.11", + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-array", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-sql", + "flate2", + "futures", + "glob", + "half", + "hashbrown 0.14.5", + "indexmap 2.2.6", + "itertools 0.12.1", + "log", + "num_cpus", + "object_store", + "parking_lot", + "parquet", + "paste", + "pin-project-lite", + "rand", + "sqlparser", + "tempfile", + "tokio", + "tokio-util", + "url", + "uuid", + "xz2", + "zstd 0.13.0", +] + +[[package]] +name = "datafusion-common" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" +dependencies = [ + "ahash 0.8.11", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.14.5", + "instant", + "libc", + "num_cpus", + "object_store", + "parquet", + "sqlparser", +] + +[[package]] +name = "datafusion-common-runtime" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0091318129dad1359f08e4c6c71f855163c35bba05d1dbf983196f727857894" +dependencies = [ + "tokio", +] + +[[package]] +name = "datafusion-execution" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1" +dependencies = [ + "arrow", + "chrono", + "dashmap", + "datafusion-common", + "datafusion-expr", + "futures", + "hashbrown 0.14.5", + "log", + "object_store", + "parking_lot", + "rand", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" +dependencies = [ + "ahash 0.8.11", + "arrow", + "arrow-array", + "arrow-buffer", + "chrono", + "datafusion-common", + "paste", + "serde_json", + "sqlparser", + "strum 0.26.2", + "strum_macros", +] + +[[package]] +name = "datafusion-functions" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b" +dependencies = [ + "arrow", + "base64 0.22.1", + "blake2", + "blake3", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "hashbrown 0.14.5", + "hex", + "itertools 0.12.1", + "log", + "md-5", + "rand", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78" +dependencies = [ + "ahash 0.8.11", + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "log", + "paste", + "sqlparser", +] + +[[package]] +name = "datafusion-functions-array" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "itertools 0.12.1", + "log", + "paste", +] + +[[package]] +name = "datafusion-optimizer" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "hashbrown 0.14.5", + "indexmap 2.2.6", + "itertools 0.12.1", + "log", + "regex-syntax 0.8.3", +] + +[[package]] +name = "datafusion-physical-expr" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e" +dependencies = [ + "ahash 0.8.11", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-string", + "base64 0.22.1", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.14.5", + "hex", + "indexmap 2.2.6", + "itertools 0.12.1", + "log", + "paste", + "petgraph", + "regex", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "rand", +] + +[[package]] +name = "datafusion-physical-plan" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff" +dependencies = [ + "ahash 0.8.11", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap 2.2.6", + "itertools 0.12.1", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "rand", + "tokio", +] + +[[package]] +name = "datafusion-sql" +version = "39.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e" +dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", + "datafusion-common", + "datafusion-expr", + "log", + "regex", + "sqlparser", + "strum 0.26.2", +] + [[package]] name = "der" version = "0.6.1" @@ -1721,7 +2136,7 @@ dependencies = [ "flate2", "futures", "hex", - "itertools 0.12.1", + "itertools 0.13.0", "lambda_runtime", "lazy_static", "md5", @@ -1767,6 +2182,7 @@ dependencies = [ "axum", "dotenvy", "filemanager", + "http 1.1.0", "tokio", "tracing", ] @@ -1831,11 +2247,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flatbuffers" -version = "23.5.26" +version = "24.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -2223,6 +2645,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.28" @@ -2393,6 +2821,18 @@ dependencies = [ "syn 2.0.65", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -2435,6 +2875,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -2461,12 +2910,12 @@ dependencies = [ [[package]] name = "lambda_http" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ebde9aedfdf8c2bac4365d9adafae6ca6d631b0ea221734ac802595fcd141cb" +checksum = "56eed1f61ea0efe7e89e1f018d95016959169b4194498a0283e950037bab6bb9" dependencies = [ "aws_lambda_events", - "base64 0.21.7", + "base64 0.22.1", "bytes", "encoding_rs", "futures", @@ -2488,9 +2937,9 @@ dependencies = [ [[package]] name = "lambda_runtime" -version = "0.11.2" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae4606aea513f0e614497c0c4556d0e39f51a8434d9d97e592d32f9e615d4232" +checksum = "a7a3eecde134218c7388f13079ab133552a251c319683a99495c048d30363927" dependencies = [ "async-stream", "base64 0.22.1", @@ -2671,6 +3120,17 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "lzokay-native" version = "0.1.0" @@ -2987,6 +3447,27 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbebfd32c213ba1907fa7a9c9138015a8de2b43e30c5aa45b18f7deb46786ad6" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "humantime", + "itertools 0.12.1", + "parking_lot", + "percent-encoding", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -3001,14 +3482,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "orc-rust" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f78d8dd5a9a215c66771d7fb85b46e30c3812abfe3bff6f94ce6b92e09d11a03" +checksum = "a5654d5043f514575d2fea0e414c64dbf24abefef40b710bfeb868d2a34789fb" dependencies = [ "arrow", + "async-trait", "bytes", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", + "datafusion", + "datafusion-expr", + "datafusion-physical-expr", "fallible-streaming-iterator", "flate2", "futures", @@ -3016,6 +3501,7 @@ dependencies = [ "lz4_flex", "lzokay-native", "num", + "object_store", "prost", "snafu", "snap", @@ -3119,9 +3605,9 @@ dependencies = [ [[package]] name = "parquet" -version = "50.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" +checksum = "29c3b5322cc1bbf67f11c079c42be41a55949099b78732f7dba9e15edde40eab" dependencies = [ "ahash 0.8.11", "arrow-array", @@ -3131,7 +3617,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.21.7", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -3142,13 +3628,15 @@ dependencies = [ "lz4_flex", "num", "num-bigint", + "object_store", "paste", "seq-macro", "snap", "thrift", "tokio", "twox-hash", - "zstd 0.13.1", + "zstd 0.13.0", + "zstd-sys", ] [[package]] @@ -3181,6 +3669,16 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.2.6", +] + [[package]] name = "phf" version = "0.11.2" @@ -4438,6 +4936,27 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "sqlparser" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.65", +] + [[package]] name = "sqlx" version = "0.7.4" @@ -5659,6 +6178,15 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yansi" version = "0.5.1" @@ -5718,11 +6246,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ - "zstd-safe 7.1.0", + "zstd-safe 7.0.0", ] [[package]] @@ -5737,18 +6265,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", "pkg-config", diff --git a/lib/workload/stateless/stacks/filemanager/README.md b/lib/workload/stateless/stacks/filemanager/README.md index 7114cbfce..f9d29a75a 100644 --- a/lib/workload/stateless/stacks/filemanager/README.md +++ b/lib/workload/stateless/stacks/filemanager/README.md @@ -78,12 +78,19 @@ Which runs `cargo test -- --ignored`. To connect to the local postgres database, run: -```bash +```sh make psql ``` Alternatively, just `brew install dbeaver-community` to easily browse the database contents (or any other DB viewer you prefer). +## Local API server + +To use the local API server, run: + +```sh +make api +``` ### Compilation and migrations @@ -105,18 +112,33 @@ docker system prune -a --volumes [deploy]: ./deploy [env-example]: .env.example +## Architecture + +The filemanager ingest functionality operates to ensure eventual consistency in the database records. See the +[ARCHITECTURE.md][architecture] for more details. + ## Project Layout The project is divided into multiple crates that serve different functionality. * [filemanager]: This is the bulk of the filemanager logic, and handles database connections and event processing. -* [filemanager-http-lambda]: This is a Lambda function which calls the SQS queue manually to ingest events. +* [filemanager-api-lambda]: This is a Lambda function which responds to API Gateway requests. +* [filemanager-api-server]: A local server instance of the filemanager API. +* [filemanager-build]: Build related functionality such as database entity generation. * [filemanager-ingest-lambda]: This is a Lambda function which ingests events directly passed from an SQS queue. +* [filemanager-inventory-lambda]: This function ingests events using [S3 Inventory][inventory]. +* [filemanager-migrate-lambda]: A Lambda function to apply database migrations. * [deploy]: CDK deployment code. * [database]: Database migration files and queries. +[architecture]: docs/ARCHITECTURE.md [filemanager]: filemanager -[filemanager-http-lambda]: filemanager-http-lambda +[filemanager-api-lambda]: filemanager-api-lambda +[filemanager-api-server]: filemanager-api-server +[filemanager-build]: filemanager-build [filemanager-ingest-lambda]: filemanager-ingest-lambda +[filemanager-inventory-lambda]: filemanager-inventory-lambda +[filemanager-migrate-lambda]: filemanager-migrate-lambda +[inventory]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/storage-inventory.html [deploy]: deploy [database]: database diff --git a/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/api.ts b/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/api.ts new file mode 100644 index 000000000..04e471a12 --- /dev/null +++ b/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/api.ts @@ -0,0 +1,26 @@ +import { Construct } from 'constructs'; +import * as fn from './function'; +import { DatabaseProps } from './function'; + +/** + * Props for the API function. + */ +export type ApiFunctionProps = fn.FunctionPropsNoPackage & DatabaseProps; + +/** + * A construct for the Lambda API function. + */ +export class ApiFunction extends fn.Function { + constructor(scope: Construct, id: string, props: ApiFunctionProps) { + super(scope, id, { + package: 'filemanager-api-lambda', + environment: { + // This ensures that the API Gateway stage is not included when processing a request. + // See https://github.com/awslabs/aws-lambda-rust-runtime/tree/main/lambda-http#integration-with-api-gateway-stages + // for more info. + AWS_LAMBDA_HTTP_IGNORE_STAGE_IN_PATH: 'true', + }, + ...props, + }); + } +} diff --git a/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/function.ts b/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/function.ts index 230b37c5f..6ca5ed426 100644 --- a/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/function.ts +++ b/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/function.ts @@ -38,6 +38,10 @@ export type FunctionPropsNoPackage = { * Additional build environment variables when building the Lambda function. */ readonly buildEnvironment?: { [key: string]: string }; + /** + * Additional environment variables to set inside the Lambda function + */ + readonly environment?: { [key: string]: string }; /** * RUST_LOG string, defaults to trace on local crates and info everywhere else. */ @@ -121,6 +125,7 @@ export class Function extends Construct { PGUSER: FILEMANAGER_SERVICE_NAME, RUST_LOG: props.rustLog ?? `info,${props.package.replace('-', '_')}=trace,filemanager=trace`, + ...props.environment, }, architecture: Architecture.ARM_64, role: this._role, diff --git a/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/query.ts b/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/query.ts deleted file mode 100644 index 9d3ae056c..000000000 --- a/lib/workload/stateless/stacks/filemanager/deploy/constructs/functions/query.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Construct } from 'constructs'; -import * as fn from './function'; -import { DatabaseProps } from './function'; - -/** - * Props for the (objects) query function. - */ -export type ObjectsQueryFunctionProps = fn.FunctionPropsNoPackage & DatabaseProps; - -/** - * A construct for the Lambda query function. - */ -export class QueryFunction extends fn.Function { - constructor(scope: Construct, id: string, props: ObjectsQueryFunctionProps) { - super(scope, id, { package: 'filemanager-api', ...props }); - } -} diff --git a/lib/workload/stateless/stacks/filemanager/deploy/stack.ts b/lib/workload/stateless/stacks/filemanager/deploy/stack.ts index 26f43cc43..f4a1f481b 100644 --- a/lib/workload/stateless/stacks/filemanager/deploy/stack.ts +++ b/lib/workload/stateless/stacks/filemanager/deploy/stack.ts @@ -1,16 +1,14 @@ import { Construct } from 'constructs'; import { IngestFunction } from './constructs/functions/ingest'; import { MigrateFunction } from './constructs/functions/migrate'; -import { QueryFunction } from './constructs/functions/query'; +import { ApiFunction } from './constructs/functions/api'; import { DatabaseProps } from './constructs/functions/function'; import { Vpc, SecurityGroup, VpcLookupOptions, IVpc, ISecurityGroup } from 'aws-cdk-lib/aws-ec2'; import { Arn, Stack, StackProps } from 'aws-cdk-lib'; import { StringParameter } from 'aws-cdk-lib/aws-ssm'; import { ProviderFunction } from '../../../../components/provider-function'; -import { ApiGatewayConstruct, ApiGwLogsConfig } from '../../../../components/api-gateway'; +import { ApiGwLogsConfig } from '../../../../components/api-gateway'; import { IQueue, Queue } from 'aws-cdk-lib/aws-sqs'; -import { HttpMethod, HttpRoute, HttpRouteKey } from 'aws-cdk-lib/aws-apigatewayv2'; -import { HttpLambdaIntegration } from 'aws-cdk-lib/aws-apigatewayv2-integrations'; import { InventoryFunction } from './constructs/functions/inventory'; import { NamedLambdaRole } from '../../../../components/named-lambda-role'; import { ApiGatewayProxyIntegration } from '../../../../components/api-gateway-proxy-integration'; @@ -133,7 +131,7 @@ export class Filemanager extends Stack { * Query function and API Gateway fronting the function. */ private createApiFunction(props: FilemanagerProps) { - let objectsQueryLambda = new QueryFunction(this, 'ObjectsQueryFunction', { + let apiLambda = new ApiFunction(this, 'ApiFunction', { vpc: this.vpc, host: this.host, securityGroup: this.securityGroup, @@ -141,7 +139,7 @@ export class Filemanager extends Stack { }); new ApiGatewayProxyIntegration(this, 'ProxyIntegration', { - handler: objectsQueryLambda.function, + handler: apiLambda.function, apiGatewayProps: { region: this.region, apiName: 'FileManager', diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/Cargo.toml b/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/Cargo.toml index 3d6fa84cb..889a47bbb 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/Cargo.toml +++ b/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/Cargo.toml @@ -12,7 +12,7 @@ tokio = { version = "1", features = ["macros"] } tracing = { version = "0.1" } axum = "0.7" -lambda_http = "0.11" -lambda_runtime = "0.11" +lambda_http = "0.12" +lambda_runtime = "0.12" filemanager = { path = "../filemanager" } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/README.md b/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/README.md index ae1e80fdd..b45ff2bf6 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/README.md +++ b/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/README.md @@ -1,3 +1,13 @@ -# filemanager-http-lambda +# filemanager-api-lambda -This crate is a Lambda function which ingests events by manually calling the SQS queue receive function. It is intended to be used behind API gateway. \ No newline at end of file +This is a Lambda function integrated with API Gateway to respond to requests in the same way as [`filemanager-api-server`][filemanager-api-server]. +The API Gateway endpoint is authorized via the OrcaBus bearer token. To access the endpoint, authorize the request with +the relevant token: + +```sh +curl -H "Authorization: Bearer $token" https://file..umccr.org/api/v1/s3_objects/count +``` + +The OpenAPI docs are available at `https://file..umccr.org/swagger_ui`. + +[filemanager-api-server]: ../filemanager-api-server \ No newline at end of file diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/src/main.rs b/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/src/main.rs index d46e8d256..4339ffdcc 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/src/main.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-api-lambda/src/main.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use filemanager::database::Client; use filemanager::handlers::aws::{create_database_pool, update_credentials}; use filemanager::handlers::init_tracing; -use filemanager::routes::{api_router, AppState, ErrorResponse, ErrorStatusCode}; +use filemanager::routes::{api_router_prefixed, AppState, ErrorResponse, ErrorStatusCode}; use lambda_http::run; use tracing::debug; @@ -22,7 +22,7 @@ async fn main() -> Result<(), Error> { let client = Client::new(create_database_pool(&config).await?); let state = AppState::new(client, Arc::new(config)); - let app = api_router(state.clone()) + let app = api_router_prefixed(state.clone()) .route_layer(from_fn_with_state(state, update_credentials_middleware)); run(app).await diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-api-server/Cargo.toml b/lib/workload/stateless/stacks/filemanager/filemanager-api-server/Cargo.toml index e181e0226..e5f48c82b 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-api-server/Cargo.toml +++ b/lib/workload/stateless/stacks/filemanager/filemanager-api-server/Cargo.toml @@ -10,5 +10,6 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros"] } tracing = "0.1" axum = "0.7" dotenvy = "0.15" +http = "1" -filemanager = { path = "../filemanager" } \ No newline at end of file +filemanager = { path = "../filemanager" } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-api-server/README.md b/lib/workload/stateless/stacks/filemanager/filemanager-api-server/README.md index 021793b76..a2d3a85b7 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-api-server/README.md +++ b/lib/workload/stateless/stacks/filemanager/filemanager-api-server/README.md @@ -1,4 +1,10 @@ # filemanager-api-server An instance of the filemanager api which can be launched as a webserver. The default address which the webserver uses -is `localhost:8080`. Set the `FILEMANAGER_API_SERVER_ADDR` environment variable to change this. \ No newline at end of file +is `localhost:8080`. Set the `FILEMANAGER_API_SERVER_ADDR` environment variable to change this. To run the local server: + +```sh +make api +``` + +Then, checkout the OpenAPI docs at: `http://localhost:8080/swagger_ui`. \ No newline at end of file diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-api-server/src/main.rs b/lib/workload/stateless/stacks/filemanager/filemanager-api-server/src/main.rs index ac4946a2d..f200599f0 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-api-server/src/main.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager-api-server/src/main.rs @@ -1,13 +1,16 @@ use axum::serve; use filemanager::database::Client; use filemanager::env::Config; +use filemanager::error::Error::IoError; use filemanager::error::Result; use filemanager::handlers::init_tracing_with_format; use filemanager::handlers::Format::Pretty; -use filemanager::routes::{api_router, AppState}; +use filemanager::routes::{api_router_prefixed, AppState}; +use http::Uri; +use std::io; use std::sync::Arc; use tokio::net::TcpListener; -use tracing::debug; +use tracing::{debug, info}; #[tokio::main] async fn main() -> Result<()> { @@ -21,10 +24,19 @@ async fn main() -> Result<()> { let client = Client::from_config(&config).await?; let state = AppState::new(client, config.clone()); - let app = api_router(state); - + let app = api_router_prefixed(state); let listener = TcpListener::bind(config.api_server_addr()).await?; + let local_addr = listener.local_addr()?; debug!("listening on {}", listener.local_addr()?); + let docs = Uri::builder() + .scheme("http") + .authority(local_addr.to_string()) + .path_and_query("/swagger_ui") + .build() + .map_err(|err| IoError(io::Error::other(err)))?; + + info!("OpenAPI docs at {}", docs); + Ok(serve(listener, app).await?) } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-build/README.md b/lib/workload/stateless/stacks/filemanager/filemanager-build/README.md index 17e90b480..75fe143b2 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-build/README.md +++ b/lib/workload/stateless/stacks/filemanager/filemanager-build/README.md @@ -4,4 +4,26 @@ This crate contains code executed by filemanager during the build phase. It is r any SQL queries, such as sea-orm entities, or pre-compiled queries, and migrating databases. Any code that should be executed before compiling the [filemanager] crate should go here. -[filemanager]: ../filemanager \ No newline at end of file +## Code generation + +Parts of this code perform code generation using libraries such as [syn] - a Rust syntax tree parser. The syn library +is typically used when writing procedural macros (such as `#[derive(..)]` macros), however in this case it is useful +to generate helper code such as database entities or OpenAPI attributes that don't typically fit under procedural +macro functionality. + +In general, code generation outputs in build scripts should go inside the `OUT_DIR` path set by Cargo when executing the +`build.rs` script. + +## Project Layout + +This crate is divided into the following modules. + +* [error]: Error functionality and code, primarily written using miette in order to facilitate user-friendly build errors. +* [gen_entities]: Generates database entities using [sea-orm-cli] library. +* [gen_openapi]: Generates OpenAPI helper attributes for [utoipa]. + +[filemanager]: ../filemanager +[sea-orm-cli]: https://www.sea-ql.org/SeaORM/docs/generate-entity/sea-orm-cli/ +[miette]: https://docs.rs/miette/latest/miette/ +[utoipa]: https://github.com/juhaku/utoipa +[syn]: https://docs.rs/syn/latest/syn/ \ No newline at end of file diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-ingest-lambda/Cargo.toml b/lib/workload/stateless/stacks/filemanager/filemanager-ingest-lambda/Cargo.toml index 46f6b125d..22fb46e75 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-ingest-lambda/Cargo.toml +++ b/lib/workload/stateless/stacks/filemanager/filemanager-ingest-lambda/Cargo.toml @@ -12,6 +12,6 @@ tokio = { version = "1", features = ["macros"] } tracing = { version = "0.1" } aws_lambda_events = "0.15" -lambda_runtime = "0.11" +lambda_runtime = "0.12" filemanager = { path = "../filemanager" } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-inventory-lambda/Cargo.toml b/lib/workload/stateless/stacks/filemanager/filemanager-inventory-lambda/Cargo.toml index 6be7b7fe1..58d195ef2 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-inventory-lambda/Cargo.toml +++ b/lib/workload/stateless/stacks/filemanager/filemanager-inventory-lambda/Cargo.toml @@ -11,7 +11,7 @@ serde = { version = "1", features = ["derive"] } tokio = { version = "1", features = ["macros"] } tracing = { version = "0.1" } -lambda_runtime = "0.11" +lambda_runtime = "0.12" filemanager = { path = "../filemanager", features = ["migrate"] } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager-migrate-lambda/Cargo.toml b/lib/workload/stateless/stacks/filemanager/filemanager-migrate-lambda/Cargo.toml index aa48805c0..31b1c9887 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager-migrate-lambda/Cargo.toml +++ b/lib/workload/stateless/stacks/filemanager/filemanager-migrate-lambda/Cargo.toml @@ -12,7 +12,7 @@ tokio = { version = "1", features = ["macros"] } tracing = { version = "0.1" } aws_lambda_events = "0.15" -lambda_runtime = "0.11" +lambda_runtime = "0.12" filemanager = { path = "../filemanager", features = ["migrate"] } diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/Cargo.toml b/lib/workload/stateless/stacks/filemanager/filemanager/Cargo.toml index 46d460786..37e50663b 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/Cargo.toml +++ b/lib/workload/stateless/stacks/filemanager/filemanager/Cargo.toml @@ -42,7 +42,7 @@ thiserror = "1" uuid = { version = "1", features = ["v7"] } mockall = "0.12" mockall_double = "0.3" -itertools = "0.12" +itertools = "0.13" url = "2" bytes = "1.6" envy = "0.4" @@ -52,9 +52,9 @@ csv = "1" flate2 = "1" md5 = "0.7" hex = "0.4" -parquet = { version = "50", features = ["async"] } -arrow = { version = "50", features = ["chrono-tz"] } -arrow-json = "50" +parquet = { version = "52", features = ["async"] } +arrow = { version = "52", features = ["chrono-tz"] } +arrow-json = "52" orc-rust = "0.3" # AWS @@ -64,7 +64,7 @@ aws-sdk-s3 = "1" aws-credential-types = "1" aws-sigv4 = "1" aws-arn = "0.3" -lambda_runtime = "0.11" +lambda_runtime = "0.12" aws_lambda_events = "0.15" [dev-dependencies] diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/README.md b/lib/workload/stateless/stacks/filemanager/filemanager/README.md index 83f1b960f..64c33406d 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/README.md +++ b/lib/workload/stateless/stacks/filemanager/filemanager/README.md @@ -1,9 +1,9 @@ # filemanager This crate contains the library code that has functionality for the filemanager. Crates such as [filemanager-ingest-lambda] -and [filemanager-http-lambda] depend on this code to execute filemanager logic. +and [filemanager-lambda-api] depend on this code to execute filemanager logic. -[filemanager-http-lambda]: ../filemanager-http-lambda +[filemanager-http-lambda]: ../filemanager-lambda-api [filemanager-ingest-lambda]: ../filemanager-ingest-lambda ## Project Layout @@ -17,8 +17,11 @@ contain thin wrappers around clients, and any logic should be elsewhere to ensur * [events]: Converts raw events received from cloud storage into a format accessible by the database. For example, the `Collect` trait is used to transform cloud storage events into data usable by the database `Ingest` trait. * [handlers]: High level code that can be used by other crates to perform filemanager actions. -* [env]: Environment variable handling. +* [queries]: [SeaORM][sea-orm] database queries which can be used by the API and other functionality. +* [routes]: [Axum][axum] routes and logic used by the filemanager API. OpenAPI documentation is generated using [utoipa]. +* [env]: Environment variable handling and config. * [error]: Error code associated with filemanager. +* [uuid]: UUID generation code. Generally code that belongs to a particular cloud service should be put in its own module (e.g. AWS code goes in an `aws` module). @@ -26,6 +29,12 @@ Generally code that belongs to a particular cloud service should be put in its o [database]: src/database [events]: src/events [handlers]: src/handlers +[queries]: src/queries +[routes]: src/routes [error]: src/error.rs +[error]: src/uuid.rs [mockall]: https://github.com/asomers/mockall -[s3-events]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventNotifications.html \ No newline at end of file +[axum]: https://github.com/tokio-rs/axum +[sea-orm]: https://www.sea-ql.org/SeaORM/ +[s3-events]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventNotifications.html +[utoipa]: https://github.com/juhaku/utoipa \ No newline at end of file diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/get.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/get.rs index a43ab5390..f0f374df6 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/get.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/get.rs @@ -24,7 +24,7 @@ pub struct GetObjectById {} (status = OK, description = "Get an object by its object_id", body = Option), ErrorStatusCode, ), - context_path = "/file/v1", + context_path = "/api/v1", )] pub async fn get_object_by_id( state: State, @@ -47,7 +47,7 @@ pub struct GetS3ObjectById {} (status = OK, description = "Get an s3object by its s3_object_id", body = Option), ErrorStatusCode, ), - context_path = "/file/v1", + context_path = "/api/v1", )] pub async fn get_s3_object_by_id( state: State, diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/ingest.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/ingest.rs index a5358fc57..7710bbbe7 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/ingest.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/ingest.rs @@ -33,7 +33,7 @@ pub struct IngestCount { (status = NO_CONTENT, description = "Ingest objects into the database from the SQS queue", body = IngestCount), ErrorStatusCode, ), - context_path = "/file/v1", + context_path = "/api/v1", )] pub async fn ingest_from_sqs(state: State) -> Result> { let n_records = receive_and_ingest( diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/list.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/list.rs index e5f0b7ef3..2afecb141 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/list.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/list.rs @@ -31,7 +31,7 @@ pub struct ListObjectsParams {} (status = OK, description = "List all objects", body = Vec), ErrorStatusCode, ), - context_path = "/file/v1", + context_path = "/api/v1", )] pub async fn list_objects(state: State) -> Result>> { let query = ListQueryBuilder::new(&state.client); @@ -47,7 +47,7 @@ pub async fn list_objects(state: State) -> Result (status = OK, description = "Get the count of all objects", body = ListCount), ErrorStatusCode, ), - context_path = "/file/v1", + context_path = "/api/v1", )] pub async fn count_objects(state: State) -> Result> { let query = ListQueryBuilder::new(&state.client); @@ -69,7 +69,7 @@ pub struct ListS3ObjectsParams {} (status = OK, description = "List all s3 objects", body = Vec), ErrorStatusCode, ), - context_path = "/file/v1", + context_path = "/api/v1", )] pub async fn list_s3_objects(state: State) -> Result>> { let query = ListQueryBuilder::new(&state.client); @@ -85,7 +85,7 @@ pub async fn list_s3_objects(state: State) -> Result) -> Result> { let query = ListQueryBuilder::new(&state.client); diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/mod.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/mod.rs index 49511d226..d8e142f37 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/mod.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/mod.rs @@ -54,6 +54,13 @@ impl AppState { } } +/// Prefixed router with a version number and swagger ui. +pub fn api_router_prefixed(state: AppState) -> Router { + Router::new() + .nest("/api/v1", api_router(state)) + .merge(swagger_ui()) +} + /// The main filemanager router for requests. pub fn api_router(state: AppState) -> Router { Router::new() @@ -67,7 +74,6 @@ pub fn api_router(state: AppState) -> Router { .fallback(fallback) .with_state(state) .layer(TraceLayer::new_for_http()) - .merge(swagger_ui()) } /// The fallback route. diff --git a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/openapi.rs b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/openapi.rs index 1c2b62962..b4259b034 100644 --- a/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/openapi.rs +++ b/lib/workload/stateless/stacks/filemanager/filemanager/src/routes/openapi.rs @@ -16,12 +16,12 @@ use crate::routes::ingest::*; use crate::routes::list::*; use crate::routes::ErrorResponse; -/// A newtype representing a chrono DateTime used to link to the utoipa `DateTime` known value. +/// A newtype equivalent to a `DateTime` with a time zone. #[derive(ToSchema)] #[schema(value_type = DateTime)] pub struct DateTimeWithTimeZone(pub DateTime); -/// A newtype representing a json Value used to link to the utoipa `Value` known value. +/// A newtype equivalent to an arbitrary JSON `Value`. #[derive(ToSchema)] #[schema(value_type = Value)] pub struct Json(pub Value); @@ -85,12 +85,12 @@ mod tests { use tower::ServiceExt; use crate::database::aws::migration::tests::MIGRATOR; - use crate::routes::api_router; + use crate::routes::api_router_prefixed; use crate::routes::AppState; #[sqlx::test(migrator = "MIGRATOR")] async fn get_swagger_ui(pool: PgPool) { - let app = api_router(AppState::from_pool(pool)); + let app = api_router_prefixed(AppState::from_pool(pool)); let response = app .oneshot( Request::builder() diff --git a/lib/workload/stateless/stacks/metadata-manager/deploy/construct/lambda-api/index.ts b/lib/workload/stateless/stacks/metadata-manager/deploy/construct/lambda-api/index.ts index 968471410..3a37fc224 100644 --- a/lib/workload/stateless/stacks/metadata-manager/deploy/construct/lambda-api/index.ts +++ b/lib/workload/stateless/stacks/metadata-manager/deploy/construct/lambda-api/index.ts @@ -1,13 +1,8 @@ import { Construct } from 'constructs'; import { Duration } from 'aws-cdk-lib'; -import { HttpMethod, HttpRoute, HttpRouteKey, IHttpApi } from 'aws-cdk-lib/aws-apigatewayv2'; import { PythonFunction, PythonFunctionProps } from '@aws-cdk/aws-lambda-python-alpha'; import { ISecret } from 'aws-cdk-lib/aws-secretsmanager'; -import { HttpLambdaIntegration } from 'aws-cdk-lib/aws-apigatewayv2-integrations'; -import { - ApiGatewayConstruct, - ApiGatewayConstructProps, -} from '../../../../../../components/api-gateway'; +import { ApiGatewayConstructProps } from '../../../../../../components/api-gateway'; import { ApiGatewayProxyIntegration } from '../../../../../../components/api-gateway-proxy-integration'; type LambdaProps = { diff --git a/lib/workload/stateless/stacks/sequence-run-manager/deploy/stack.ts b/lib/workload/stateless/stacks/sequence-run-manager/deploy/stack.ts index ff20f5e77..7339a1f5b 100644 --- a/lib/workload/stateless/stacks/sequence-run-manager/deploy/stack.ts +++ b/lib/workload/stateless/stacks/sequence-run-manager/deploy/stack.ts @@ -5,10 +5,8 @@ import { Construct } from 'constructs'; import { ISecurityGroup, IVpc, SecurityGroup, Vpc, VpcLookupOptions } from 'aws-cdk-lib/aws-ec2'; import { EventBus, IEventBus } from 'aws-cdk-lib/aws-events'; import { PythonFunction, PythonLayerVersion } from '@aws-cdk/aws-lambda-python-alpha'; -import { HttpLambdaIntegration } from 'aws-cdk-lib/aws-apigatewayv2-integrations'; -import { HttpMethod, HttpRoute, HttpRouteKey } from 'aws-cdk-lib/aws-apigatewayv2'; import { ManagedPolicy, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; -import { ApiGatewayConstruct, ApiGwLogsConfig } from '../../../../components/api-gateway'; +import { ApiGwLogsConfig } from '../../../../components/api-gateway'; import { Architecture } from 'aws-cdk-lib/aws-lambda'; import { PostgresManagerStack } from '../../../../stateful/stacks/postgres-manager/deploy/stack'; import { ApiGatewayProxyIntegration } from '../../../../components/api-gateway-proxy-integration'; diff --git a/lib/workload/stateless/stacks/workflow-manager/deploy/stack.ts b/lib/workload/stateless/stacks/workflow-manager/deploy/stack.ts index ec2d1f63c..d8f716de4 100644 --- a/lib/workload/stateless/stacks/workflow-manager/deploy/stack.ts +++ b/lib/workload/stateless/stacks/workflow-manager/deploy/stack.ts @@ -12,21 +12,9 @@ import { StackProps, } from 'aws-cdk-lib'; import { PythonFunction, PythonLayerVersion } from '@aws-cdk/aws-lambda-python-alpha'; -import { HttpLambdaIntegration } from 'aws-cdk-lib/aws-apigatewayv2-integrations'; -import { - CorsHttpMethod, - HttpApi, - HttpMethod, - HttpRoute, - HttpRouteKey, - HttpStage, -} from 'aws-cdk-lib/aws-apigatewayv2'; import { PostgresManagerStack } from '../../../../stateful/stacks/postgres-manager/deploy/stack'; import { ManagedPolicy, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; -import { - ApiGatewayConstruct, - ApiGwLogsConfig, -} from '../../../../../workload/components/api-gateway'; +import { ApiGwLogsConfig } from '../../../../components/api-gateway'; import { ApiGatewayProxyIntegration } from '../../../../components/api-gateway-proxy-integration'; export interface WorkflowManagerStackProps extends StackProps { diff --git a/skel/django-api/deploy/stack.ts b/skel/django-api/deploy/stack.ts index 58eb576bf..36520041e 100644 --- a/skel/django-api/deploy/stack.ts +++ b/skel/django-api/deploy/stack.ts @@ -2,16 +2,14 @@ import path from 'path'; import { Construct } from 'constructs'; -import { Architecture, ILayerVersion } from 'aws-cdk-lib/aws-lambda'; +import { Architecture } from 'aws-cdk-lib/aws-lambda'; import { ISecurityGroup, IVpc, SecurityGroup, Vpc, VpcLookupOptions } from 'aws-cdk-lib/aws-ec2'; import { EventBus, IEventBus, Rule } from 'aws-cdk-lib/aws-events'; import { aws_events_targets, aws_lambda, aws_secretsmanager, Duration, Stack, StackProps } from 'aws-cdk-lib'; import { PythonFunction, PythonLayerVersion } from '@aws-cdk/aws-lambda-python-alpha'; -import { HttpLambdaIntegration } from 'aws-cdk-lib/aws-apigatewayv2-integrations'; -import { CorsHttpMethod, HttpApi, HttpMethod, HttpRoute, HttpRouteKey, HttpStage } from 'aws-cdk-lib/aws-apigatewayv2'; import { PostgresManagerStack } from '../../../lib/workload/stateful/stacks/postgres-manager/deploy/stack'; import { ManagedPolicy, Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; -import { ApiGatewayConstruct, ApiGwLogsConfig } from '../../../lib/workload/components/api-gateway'; +import { ApiGwLogsConfig } from '../../../lib/workload/components/api-gateway'; import { ApiGatewayProxyIntegration } from '../../../lib/workload/components/api-gateway-proxy-integration'; export interface ProjectNameStackProps { // FIXME change prop interface name