diff --git a/Project.toml b/Project.toml index 005e3f9..6816d45 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AWSClusterManagers" uuid = "69f5933c-162f-5d2f-a6d8-da72a0bfad91" authors = ["Invenia Technical Computing"] -version = "1.1.5" +version = "1.2.0" [deps] AWSBatch = "dcae83d4-2881-5875-9d49-e5534165e9c0" @@ -13,20 +13,20 @@ Mocking = "78c3b35d-d492-501b-9361-3d52fe80e533" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" [compat] -AWSBatch = "1.4" -AWSCore = "0.6" -AWSTools = "1" +AWS = "1" +AWSBatch = "2.0.1" +AWSTools = "2" JSON = "0.20.1, 0.21" Memento = "0.11, 0.12, 0.13, 1" Mocking = "0.7" julia = "1" [extras] -AWSCore = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" +AWS = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" AWSTools = "83bcdc74-1232-581c-948a-f29122bf8259" LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["AWSCore", "AWSTools", "LibGit2", "Printf", "Test"] +test = ["AWS", "AWSTools", "LibGit2", "Printf", "Test"] diff --git a/docs/Manifest.toml b/docs/Manifest.toml index b4cfd67..cc8c5f6 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,52 +1,28 @@ # This file is machine-generated - editing it directly is not advised [[AWS]] -deps = ["Base64", "Compat", "Dates", "GitHub", "HTTP", "IniFile", "JSON", "MbedTLS", "Mocking", "OrderedCollections", "Retry", "Sockets", "UUIDs", "XMLDict"] -git-tree-sha1 = "f336cf3c87420bf8c10416b0b71a1426f550c8ba" +deps = ["Base64", "Compat", "Dates", "Downloads", "GitHub", "HTTP", "IniFile", "JSON", "MbedTLS", "Mocking", "OrderedCollections", "Retry", "Sockets", "URIs", "UUIDs", "XMLDict"] +git-tree-sha1 = "07d944e4d9946c2061f97c1564d1b7ae8ea8f189" uuid = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" -version = "1.25.5" +version = "1.61.0" [[AWSBatch]] -deps = ["AWSCore", "AWSSDK", "AWSTools", "AutoHashEquals", "Dates", "Memento", "Mocking", "OrderedCollections", "Pkg"] -git-tree-sha1 = "2ab3177464be0398b19e55dd5c4f8769b37b717c" +deps = ["AWS", "AutoHashEquals", "Dates", "Memento", "Mocking", "OrderedCollections", "Pkg"] +git-tree-sha1 = "dfe487a88864ca8e58ff2c7c8149d0b6dc2bc94d" uuid = "dcae83d4-2881-5875-9d49-e5534165e9c0" -version = "1.4.2" +version = "2.0.1" [[AWSClusterManagers]] deps = ["AWSBatch", "Dates", "Distributed", "JSON", "Memento", "Mocking", "Sockets"] path = ".." uuid = "69f5933c-162f-5d2f-a6d8-da72a0bfad91" -version = "1.1.4" - -[[AWSCore]] -deps = ["Base64", "Dates", "HTTP", "IniFile", "JSON", "LazyJSON", "MbedTLS", "Mocking", "OrderedCollections", "Retry", "Sockets", "SymDict", "UUIDs", "XMLDict"] -git-tree-sha1 = "1d3c3af424f774d1b5f040f2f62ce51f2f790b70" -uuid = "4f1ea46c-232b-54a6-9b17-cc2d0f3e6598" -version = "0.6.17" - -[[AWSS3]] -deps = ["AWS", "Base64", "Dates", "EzXML", "FilePathsBase", "HTTP", "MbedTLS", "OrderedCollections", "Retry", "SymDict", "UUIDs", "XMLDict"] -git-tree-sha1 = "fc337c0e58d571b4b760849c8f318c08562af015" -uuid = "1c724243-ef5b-51ab-93f4-b0a88ac62a95" -version = "0.8.3" - -[[AWSSDK]] -deps = ["AWSCore", "Test"] -git-tree-sha1 = "1ab1dc61457d0665c851bc78afdeb17befe88b2e" -uuid = "0d499d91-6ae5-5d63-9313-12987b87d5ad" -version = "0.5.0" +version = "1.2.0" -[[AWSTools]] -deps = ["AWSCore", "AWSS3", "Base64", "Dates", "EzXML", "FilePathsBase", "HTTP", "MbedTLS", "Memento", "Mocking", "OrderedCollections", "Random", "XMLDict"] -git-tree-sha1 = "c2380e47f34aa7c005487eae334f4689671bc032" -uuid = "83bcdc74-1232-581c-948a-f29122bf8259" -version = "1.9.0" +[[ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" [[Artifacts]] -deps = ["Pkg"] -git-tree-sha1 = "c30985d8821e0cd73870b17b0ed0ce6dc44cb744" uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" -version = "1.3.0" [[AutoHashEquals]] git-tree-sha1 = "45bb6705d93be619b81451bb2006b7ee5d4e4453" @@ -58,9 +34,9 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[Compat]] deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "919c7f3151e79ff196add81d7f4e45d91bbf420b" +git-tree-sha1 = "4866e381721b30fac8dda4c8cb1d9db45c8d2994" uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "3.25.0" +version = "3.37.0" [[Dates]] deps = ["Printf"] @@ -75,10 +51,10 @@ deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[DocStringExtensions]] -deps = ["LibGit2", "Markdown", "Pkg", "Test"] -git-tree-sha1 = "50ddf44c53698f5e784bbebb3f4b21c5807401b1" +deps = ["LibGit2"] +git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.8.3" +version = "0.8.5" [[Documenter]] deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] @@ -86,10 +62,14 @@ git-tree-sha1 = "3ebb967819b284dc1e3c0422229b58a40a255649" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" version = "0.26.3" +[[Downloads]] +deps = ["ArgTools", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" + [[ExprTools]] -git-tree-sha1 = "10407a39b87f29d47ebaca8edbc75d7c302ff93e" +git-tree-sha1 = "b7e3d17636b348f005f11040025ae8c6f645fe92" uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" -version = "0.1.3" +version = "0.1.6" [[EzXML]] deps = ["Printf", "XML2_jll"] @@ -97,23 +77,21 @@ git-tree-sha1 = "0fa3b52a04a4e210aeb1626def9c90df3ae65268" uuid = "8f5d6c58-4d21-5cfd-889c-e3ad7ee6a615" version = "1.1.0" -[[FilePathsBase]] -deps = ["Dates", "Mmap", "Printf", "Test", "UUIDs"] -git-tree-sha1 = "0f5e8d0cb91a6386ba47bd1527b240bd5725fbae" -uuid = "48062228-2e41-5def-b9a4-89aafe57970f" -version = "0.9.10" +[[Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" [[GitHub]] deps = ["Base64", "Dates", "HTTP", "JSON", "MbedTLS", "Sockets", "SodiumSeal"] -git-tree-sha1 = "a4f61fc1b1724e6eec1d9333eac2d4b01d8fcc8f" +git-tree-sha1 = "c8594dff1ed76e232d8063b2a2555335900af6f3" uuid = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26" -version = "5.4.0" +version = "5.7.0" [[HTTP]] -deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"] -git-tree-sha1 = "c7ec02c4c6a039a98a15f955462cd7aea5df4508" +deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"] +git-tree-sha1 = "60ed5f1643927479f845b0135bb369b031b541fa" uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "0.8.19" +version = "0.9.14" [[IOCapture]] deps = ["Logging"] @@ -137,34 +115,45 @@ uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" version = "1.3.0" [[JLLWrappers]] -git-tree-sha1 = "a431f5f2ca3f4feef3bd7a5e94b8b8d4f2f647a0" +deps = ["Preferences"] +git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.2.0" +version = "1.3.0" [[JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4" +git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.1" +version = "0.21.2" + +[[LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + +[[LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -[[LazyJSON]] -deps = ["JSON", "OrderedCollections", "PropertyDicts"] -git-tree-sha1 = "ce08411caa70e0c9e780f142f59debd89a971738" -uuid = "fc18253b-5e1b-504c-a4a2-9ece4944c004" -version = "0.2.2" +[[LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" [[LibGit2]] -deps = ["Printf"] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +[[LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" + [[Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" [[Libiconv_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "8e924324b2e9275a51407a4e06deb3455b1e359f" +git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778" uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" -version = "1.16.0+7" +version = "1.16.1+1" [[LinearAlgebra]] deps = ["Libdl"] @@ -184,52 +173,57 @@ uuid = "739be429-bea8-5141-9913-cc70e7f3736d" version = "1.0.3" [[MbedTLS_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "0eef589dd1c26a3ac9d753fe1a8bcad63f956fa6" +deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.16.8+1" [[Memento]] deps = ["Dates", "Distributed", "JSON", "Serialization", "Sockets", "Syslogs", "Test", "TimeZones", "UUIDs"] -git-tree-sha1 = "d6dfb54d7e8a9b4a2b1773acf7275a4f607906b2" +git-tree-sha1 = "19650888f97362a2ae6c84f0f5f6cda84c30ac38" uuid = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" -version = "1.1.2" +version = "1.2.0" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[Mocking]] deps = ["ExprTools"] -git-tree-sha1 = "916b850daad0d46b8c71f65f719c49957e9513ed" +git-tree-sha1 = "748f6e1e4de814b101911e64cc12d83a6af66782" uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" -version = "0.7.1" +version = "0.7.2" + +[[MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" + +[[NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" [[OrderedCollections]] -git-tree-sha1 = "4fa2ba51070ec13fcc7517db714445b4ab986bdf" +git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.4.0" +version = "1.4.1" [[Parsers]] deps = ["Dates"] -git-tree-sha1 = "50c9a9ed8c714945e01cd53a21007ed3865ed714" +git-tree-sha1 = "438d35d2d95ae2c5e8780b330592b6de8494e779" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "1.0.15" +version = "2.0.3" [[Pkg]] -deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +[[Preferences]] +deps = ["TOML"] +git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.2.2" + [[Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" -[[PropertyDicts]] -git-tree-sha1 = "429d887daee312e73842cabe6b122e310b72e25d" -uuid = "f8a19df8-e894-5f55-a973-672c1158cbca" -version = "0.1.0" - [[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets"] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[Random]] @@ -237,9 +231,9 @@ deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[RecipesBase]] -git-tree-sha1 = "b3fb709f3c97bfc6e948be68beeecb55a0b340ae" +git-tree-sha1 = "44a75aa7a527910ee3d1751d1f0e4148698add9e" uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" -version = "1.1.1" +version = "1.1.2" [[Retry]] git-tree-sha1 = "41ac127cd281bb33e42aba46a9d3b25cd35fc6d5" @@ -273,27 +267,34 @@ uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -[[SymDict]] -deps = ["Test"] -git-tree-sha1 = "0108ccdaea3ef69d9680eeafc8d5ad198b896ec8" -uuid = "2da68c74-98d7-5633-99d6-8493888d7b1e" -version = "0.3.0" - [[Syslogs]] deps = ["Printf", "Sockets"] git-tree-sha1 = "46badfcc7c6e74535cc7d833a91f4ac4f805f86d" uuid = "cea106d9-e007-5e6c-ad93-58fe2094e9c4" version = "0.3.0" +[[TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" + +[[Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" + [[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [[TimeZones]] -deps = ["Dates", "EzXML", "Mocking", "Pkg", "Printf", "RecipesBase", "Serialization", "Unicode"] -git-tree-sha1 = "4ba8a9579a243400db412b50300cd61d7447e583" +deps = ["Dates", "Future", "LazyArtifacts", "Mocking", "Pkg", "Printf", "RecipesBase", "Serialization", "Unicode"] +git-tree-sha1 = "6c9040665b2da00d30143261aea22c7427aada1c" uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" -version = "1.5.3" +version = "1.5.7" + +[[URIs]] +git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.3.0" [[UUIDs]] deps = ["Random", "SHA"] @@ -304,9 +305,9 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[XML2_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"] -git-tree-sha1 = "be0db24f70aae7e2b89f2f3092e93b8606d659a6" +git-tree-sha1 = "1acf5bdf07aa0907e0a37d3718bb88d4b687b74a" uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" -version = "2.9.10+3" +version = "2.9.12+0" [[XMLDict]] deps = ["EzXML", "IterTools", "OrderedCollections"] @@ -315,13 +316,19 @@ uuid = "228000da-037f-5747-90a9-8195ccbf91a5" version = "0.4.1" [[Zlib_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "320228915c8debb12cb434c59057290f0834dbf6" +deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.11+18" [[libsodium_jll]] -deps = ["Libdl", "Pkg"] -git-tree-sha1 = "7127f5f40332ccfa43ee07dcd0c4d81a27d9bb23" +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "848ab3d00fe39d6fbc2a8641048f8f272af1c51e" uuid = "a9144af2-ca23-56d9-984f-0d03f7b5ccf8" -version = "1.0.18+1" +version = "1.0.20+0" + +[[nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" + +[[p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" diff --git a/test/batch_node_online.jl b/test/batch_node_online.jl index 94546d5..6a9b2c5 100644 --- a/test/batch_node_online.jl +++ b/test/batch_node_online.jl @@ -384,7 +384,7 @@ end @test status(worker_job) == AWSBatch.SUCCEEDED manager_log = log_messages(manager_job) - worker_log = log_messages(worker_job) + worker_log = log_messages(worker_job; retries=0) test_results = [ @test occursin("Only 0 of the 1 workers job have reported in", manager_log) @test isempty(worker_log) @@ -393,7 +393,7 @@ end # Display the logs for all the jobs if any of the log tests fail if any(r -> !(r isa Test.Pass), test_results) @info "Details for manager:\n$(report(manager_job))" - @info "Details for worker:\n$(report(worker_log))" + @info "Details for worker:\n$(report(worker_job))" end end diff --git a/test/batch_online.jl b/test/batch_online.jl index 43b73db..0ba9007 100644 --- a/test/batch_online.jl +++ b/test/batch_online.jl @@ -107,30 +107,35 @@ function run_batch_job( # Note: Do not assume that the "Manager Complete" message will be the last thing written # to the log as busy worker may cause additional warnings messages. # https://github.com/JuliaCloud/AWSClusterManagers.jl/issues/10 + output = "" if status(job) == AWSBatch.SUCCEEDED log_wait_start = time() + while true events = log_events(job) if events !== nothing && !isempty(events) && any(e -> e.message == "Manager Complete", events) + output = join( + [string(event.timestamp, " ", event.message) for event in events], '\n' + ) break elseif time() - log_wait_start > 60 error("CloudWatch logs have not completed ingestion within 1 minute") end + sleep(5) end end - return job + return job, output end @testset "AWSBatchManager (online)" begin # Note: Start with the largest number of workers so the remaining tests don't have # to wait for the cluster to scale up on subsequent tests. @testset "Num workers ($num_workers)" for num_workers in [10, 1, 0] - job = run_batch_job(TEST_IMAGE, num_workers) - output = log_messages(job) + job, output = run_batch_job(TEST_IMAGE, num_workers) m = match(r"(?<=NumProcs: )\d+", output) if m !== nothing @@ -186,17 +191,13 @@ end @testset "exceed worker limit" begin num_workers = typemax(Int64) - job = run_batch_job(TEST_IMAGE, num_workers; should_fail=true) - output = log_messages(job) - - m = match(r"(?<=NumProcs: )\d+", output) - num_procs = m !== nothing ? parse(Int, m.match) : -1 + job, output = run_batch_job(TEST_IMAGE, num_workers; should_fail=true) # Spawned are the AWS Batch job IDs reported upon job submission at launch # while reported is the self-reported job ID of each worker. spawned_jobs = scrape_worker_job_ids(output) - @test num_procs == -1 + @test match(r"(?<=NumProcs: )\d+", output) === nothing @test isempty(spawned_jobs) end end diff --git a/test/runtests.jl b/test/runtests.jl index 6ed93fd..b046570 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,7 @@ +using AWS using AWSBatch using AWSClusterManagers using AWSClusterManagers: desired_workers, launch_timeout -using AWSCore: AWSCore using AWSTools.CloudFormation: stack_output using AWSTools.Docker: Docker using Base: AbstractCmd @@ -16,6 +16,8 @@ using Printf: @sprintf using Sockets using Test +@service Batch + Mocking.activate() const LOGGER = Memento.config!("info"; fmt="[{date} | {level} | {name}]: {msg}") @@ -94,11 +96,14 @@ end include("utils.jl") @testset "AWSClusterManagers" begin - include("container.jl") - include("docker.jl") - include("batch.jl") - include("socket.jl") - include("batch_node.jl") + # Avoid accessing AWSCredentials in offline tests + withenv("AWS_ACCESS_KEY_ID" => "", "AWS_SECRET_ACCESS_KEY" => "") do + include("container.jl") + include("docker.jl") + include("batch.jl") + include("socket.jl") + include("batch_node.jl") + end if "docker" in ONLINE include("docker_online.jl") diff --git a/test/utils.jl b/test/utils.jl index e979685..64c47fa 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -1,6 +1,14 @@ # Gets the logs messages associated with a AWSBatch BatchJob as a single string -function log_messages(job::BatchJob) +function log_messages(job::BatchJob; retries=5, wait_interval=7) + i = 0 events = log_events(job) + # Retry if no logs are found + while i < retries && (events === nothing || isempty(events)) + i += 1 + sleep(wait_interval) + events = log_events(job) + end + events === nothing && return "" return join([string(event.timestamp, " ", event.message) for event in events], '\n') end @@ -46,7 +54,10 @@ time_str(p::Period) = time_str(floor(p, Second)) time_str(::Nothing) = "N/A" # Unable to determine duration function register_job_definition(job_definition::AbstractDict) - output = AWSCore.Services.batch("POST", "/v1/registerjobdefinition", job_definition) + output = Batch.register_job_definition( + job_definition["jobDefinitionName"], job_definition["type"], job_definition + ) + return output["jobDefinitionArn"] end @@ -57,9 +68,7 @@ function submit_job(; node_overrides::Dict=Dict(), retry_strategy::Dict=Dict(), ) - options = Dict{String,Any}( - "jobName" => job_name, "jobDefinition" => job_definition, "jobQueue" => job_queue - ) + options = Dict{String,Any}() if !isempty(node_overrides) options["nodeOverrides"] = node_overrides @@ -71,8 +80,7 @@ function submit_job(; print(JSON.json(options, 4)) - # https://docs.aws.amazon.com/batch/latest/APIReference/API_SubmitJob.html - output = AWSCore.Services.batch("POST", "/v1/submitjob", options) + output = Batch.submit_job(job_definition, job_name, job_queue, options) return BatchJob(output["jobId"]) end @@ -83,10 +91,8 @@ function describe_compute_environment(compute_environment::AbstractString) # --compute-environments $(STACK["ComputeEnvironmentArn"]) # --query computeEnvironments[0] # ``` - output = AWSCore.Services.batch( - "POST", - "/v1/describecomputeenvironments", - Dict("computeEnvironments" => [compute_environment]), + output = Batch.describe_compute_environments( + Dict("computeEnvironments" => [compute_environment]) ) details = if !isempty(output["computeEnvironments"])