Skip to content

Commit

Permalink
fix: remove pending deployments when exceeding 5 minutes #5636 (#5762)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jgomer2001
jgomer2001 authored Aug 2, 2023
1 parent 704d7f3 commit 64ded2c
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 38 deletions.
2 changes: 1 addition & 1 deletion docs/admin/developer/agama/projects-deployment.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ The following tables summarize the available endpoints. All URLs are relative to
|Method|POST|
|Path params|`name` (the project's name)|
|Body|The binary contents of a `.gama` file; example [here](#sample-file). Ensure to use header `Content-Type: application/zip`|
|Output|Textual explanation, e.g. `A deployment task for project XXX has been queued. Use the GET endpoint to poll status`|
|Output|Textual explanation, e.g. `A deployment task for project XXX has been queued. Use the GET endpoint to poll status`|
|Status|202 (the task was created and scheduled for deployment), 409 (there is a task already for this project and it hasn't finished yet), 400 (a param is missing)|


Expand Down
81 changes: 44 additions & 37 deletions jans-auth-server/agama/engine/src/main/java/io/jans/ads/Deployer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,27 +21,14 @@
import jakarta.enterprise.context.ApplicationScoped;

import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.Files;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileVisitor;
import java.nio.file.FileVisitResult;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Base64;
import java.util.*;
import java.util.Base64.Decoder;
import java.util.Base64.Encoder;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import java.util.Date;
import java.util.List;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -77,6 +64,7 @@ public class Deployer {
private static final String METADATA_FILE = "project.json";
private static final boolean ON_CONTAINERS = System.getenv("CN_VERSION") != null;

private static final long DEPLOY_TIMEOUT = TimeUnit.MINUTES.toMillis(5);
private static final Pattern BP_PATT = Pattern.compile("\n[ \\t]+Basepath[ \\t]+\"");

@Inject
Expand Down Expand Up @@ -117,11 +105,18 @@ public void process() throws IOException {

if (deployment == null) {
updateFlowsAndAssets(depls);

//find deployments in course
filter = Filter.createANDFilter(Filter.createEqualityFilter("jansActive", true),
Filter.createNOTFilter(Filter.createPresenceFilter("jansEndDate")));

removeStaleDeployments(entryManager.findEntries(BASE_DN, Deployment.class, filter,
new String[]{ "jansId", "jansStartDate" }), System.currentTimeMillis());
} else {
deployProject(deployment.getDn(), deployment.getId(),
deployment.getDetails().getProjectMetadata().getProjectName());
}

}

private void deployProject(String dn, String prjId, String name) throws IOException {
Expand All @@ -134,11 +129,11 @@ private void deployProject(String dn, String prjId, String name) throws IOExcept
//Here, b64EncodedAssets has the layout of a .gama file
dep.setTaskActive(true);
dep.setAssets(null);

logger.info("Marking deployment task as active");
//This merge helps other nodes/pods not to take charge of this very deployment task
entryManager.merge(dep);

Path p = extractGamaFile(b64EncodedAssets);
String tmpdir = p.toString();
dd.setProjectMetadata(computeMetadata(name, tmpdir));
Expand Down Expand Up @@ -196,8 +191,8 @@ private void deployProject(String dn, String prjId, String name) throws IOExcept
}
logger.info("Finishing deployment task...");

entryManager.merge(dep); //If this fails, deployment will remain pending, see #removeStaleDeployments
projectsFinishTimes.put(prjId, d.getTime());
entryManager.merge(dep);

try {
logger.debug("Cleaning .gama extraction dir");
Expand Down Expand Up @@ -323,10 +318,8 @@ private Set<String> createFlows(Path dir, DeploymentDetails dd, String prjBasepa
}

private ZipFile compileAssetsArchive(Path root, Path webroot, Path lib, String prjBasepath) throws IOException {

String rnd = rndName();

Path agama = Files.createDirectory(Paths.get(root.toString(), rnd));
Path agama = Files.createDirectory(Paths.get(root.toString(), rndName()));
String agamStr = agama.toString();
logger.debug("Created temp directory");

Expand Down Expand Up @@ -427,11 +420,11 @@ private void updateFlowsAndAssets(List<Deployment> deployments) {
//This conditional can only evaluate truthy in a multinode environment (containers) or
//upon application startup in a VM installation
if (finishedAt == null || finishedAt < d.getFinishedAt().getTime()) {
//Retrieve associated assets
String b64EncodedAssets = entryManager.find(d.getDn(), Deployment.class,
new String[]{ Deployment.ASSETS_ATTR }).getAssets();

try {
//Retrieve associated assets
String b64EncodedAssets = entryManager.find(d.getDn(), Deployment.class,
new String[]{ Deployment.ASSETS_ATTR }).getAssets();

if (finishedAt != null) {
purge(projectsBasePaths.get(prjId), projectsLibs.get(prjId));
}
Expand All @@ -441,7 +434,7 @@ private void updateFlowsAndAssets(List<Deployment> deployments) {

logger.info("Assets of project {} were synced", name);
projectsFinishTimes.put(prjId, d.getFinishedAt().getTime());
} catch (IOException e) {
} catch (Exception e) {
logger.error("Error syncing assets of project " + name, e);
}

Expand Down Expand Up @@ -526,6 +519,23 @@ private ProjectMetadata computeMetadata(String name, String path) {
return meta;

}

private void removeStaleDeployments(List<Deployment> deployments, long instant) {

for (Deployment d : deployments) {
if (d.getCreatedAt().getTime() + DEPLOY_TIMEOUT < instant) {

try {
String prjId = d.getId();
logger.info("Removing stale deployment {}", prjId);
entryManager.remove(d.getDn(), Deployment.class);
} catch (Exception e) {
logger.error("Error removing deployment", e);
}
}
}

}

private static String dnFromQname(String qname) {
return String.format("%s=%s,%s", Flow.ATTR_NAMES.QNAME,
Expand Down Expand Up @@ -569,9 +579,8 @@ private void purge(Set<String> dirs, Set<String> filesToRemove) throws IOExcepti
private void extract(String b64EncodedAssets, String destination) throws IOException {

if (b64EncodedAssets == null) return;

String name = rndName();
Path p = Files.createTempFile​(name, null);

Path p = Files.createTempFile​(rndName(), null);
logger.debug("Dumping decoded Base64 representation to {}", p);
Files.write(p, b64Decoder.decode(b64EncodedAssets.getBytes(UTF_8)));

Expand All @@ -586,11 +595,10 @@ private void extract(String b64EncodedAssets, String destination) throws IOExcep
}

private Path extractGamaFile(String b64EncodedContents) throws IOException {

String tmpdir = rndName();
Path p = Files.createTempDirectory(tmpdir);

Path p = Files.createTempDirectory(rndName());
logger.info("Extracting .gama file to {}", p);

extract(b64EncodedContents, p.toString());
return p;

Expand Down Expand Up @@ -650,8 +658,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
try {
Files.copy(dir, targetdir);
} catch (FileAlreadyExistsException e) {
if (!Files.isDirectory(targetdir))
throw e;
if (!Files.isDirectory(targetdir)) throw e;
}
return FileVisitResult.CONTINUE;

Expand Down Expand Up @@ -709,4 +716,4 @@ private void init() {

}

}
}

0 comments on commit 64ded2c

Please # to comment.