Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

fix: remove pending deployments when exceeding 5 minutes #5762

Merged
merged 1 commit into from
Aug 2, 2023
Merged

fix: remove pending deployments when exceeding 5 minutes #5762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/admin/developer/agama/projects-deployment.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ The following tables summarize the available endpoints. All URLs are relative to
|Method|POST|
|Path params|`name` (the project's name)|
|Body|The binary contents of a `.gama` file; example [here](#sample-file). Ensure to use header `Content-Type: application/zip`|
|Output|Textual explanation, e.g. `A deployment task for project XXX has been queued. Use the GET endpoint to poll status`|
|Output|Textual explanation, e.g. `A deployment task for project XXX has been queued. Use the GET endpoint to poll status`|
|Status|202 (the task was created and scheduled for deployment), 409 (there is a task already for this project and it hasn't finished yet), 400 (a param is missing)|


Expand Down
81 changes: 44 additions & 37 deletions jans-auth-server/agama/engine/src/main/java/io/jans/ads/Deployer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,27 +21,14 @@
import jakarta.enterprise.context.ApplicationScoped;

import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.Files;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileVisitor;
import java.nio.file.FileVisitResult;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Base64;
import java.util.*;
import java.util.Base64.Decoder;
import java.util.Base64.Encoder;
import java.util.Collections;
import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
import java.util.Date;
import java.util.List;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -77,6 +64,7 @@ public class Deployer {
private static final String METADATA_FILE = "project.json";
private static final boolean ON_CONTAINERS = System.getenv("CN_VERSION") != null;

private static final long DEPLOY_TIMEOUT = TimeUnit.MINUTES.toMillis(5);
private static final Pattern BP_PATT = Pattern.compile("\n[ \\t]+Basepath[ \\t]+\"");

@Inject
Expand Down Expand Up @@ -117,11 +105,18 @@ public void process() throws IOException {

if (deployment == null) {
updateFlowsAndAssets(depls);

//find deployments in course
filter = Filter.createANDFilter(Filter.createEqualityFilter("jansActive", true),
Filter.createNOTFilter(Filter.createPresenceFilter("jansEndDate")));

removeStaleDeployments(entryManager.findEntries(BASE_DN, Deployment.class, filter,
new String[]{ "jansId", "jansStartDate" }), System.currentTimeMillis());
} else {
deployProject(deployment.getDn(), deployment.getId(),
deployment.getDetails().getProjectMetadata().getProjectName());
}

}

private void deployProject(String dn, String prjId, String name) throws IOException {
Expand All @@ -134,11 +129,11 @@ private void deployProject(String dn, String prjId, String name) throws IOExcept
//Here, b64EncodedAssets has the layout of a .gama file
dep.setTaskActive(true);
dep.setAssets(null);

logger.info("Marking deployment task as active");
//This merge helps other nodes/pods not to take charge of this very deployment task
entryManager.merge(dep);

Path p = extractGamaFile(b64EncodedAssets);
String tmpdir = p.toString();
dd.setProjectMetadata(computeMetadata(name, tmpdir));
Expand Down Expand Up @@ -196,8 +191,8 @@ private void deployProject(String dn, String prjId, String name) throws IOExcept
}
logger.info("Finishing deployment task...");

entryManager.merge(dep); //If this fails, deployment will remain pending, see #removeStaleDeployments
projectsFinishTimes.put(prjId, d.getTime());
entryManager.merge(dep);

try {
logger.debug("Cleaning .gama extraction dir");
Expand Down Expand Up @@ -323,10 +318,8 @@ private Set<String> createFlows(Path dir, DeploymentDetails dd, String prjBasepa
}

private ZipFile compileAssetsArchive(Path root, Path webroot, Path lib, String prjBasepath) throws IOException {

String rnd = rndName();

Path agama = Files.createDirectory(Paths.get(root.toString(), rnd));
Path agama = Files.createDirectory(Paths.get(root.toString(), rndName()));
String agamStr = agama.toString();
logger.debug("Created temp directory");

Expand Down Expand Up @@ -427,11 +420,11 @@ private void updateFlowsAndAssets(List<Deployment> deployments) {
//This conditional can only evaluate truthy in a multinode environment (containers) or
//upon application startup in a VM installation
if (finishedAt == null || finishedAt < d.getFinishedAt().getTime()) {
//Retrieve associated assets
String b64EncodedAssets = entryManager.find(d.getDn(), Deployment.class,
new String[]{ Deployment.ASSETS_ATTR }).getAssets();

try {
//Retrieve associated assets
String b64EncodedAssets = entryManager.find(d.getDn(), Deployment.class,
new String[]{ Deployment.ASSETS_ATTR }).getAssets();

if (finishedAt != null) {
purge(projectsBasePaths.get(prjId), projectsLibs.get(prjId));
}
Expand All @@ -441,7 +434,7 @@ private void updateFlowsAndAssets(List<Deployment> deployments) {

logger.info("Assets of project {} were synced", name);
projectsFinishTimes.put(prjId, d.getFinishedAt().getTime());
} catch (IOException e) {
} catch (Exception e) {
logger.error("Error syncing assets of project " + name, e);
}

Expand Down Expand Up @@ -526,6 +519,23 @@ private ProjectMetadata computeMetadata(String name, String path) {
return meta;

}

private void removeStaleDeployments(List<Deployment> deployments, long instant) {

for (Deployment d : deployments) {
if (d.getCreatedAt().getTime() + DEPLOY_TIMEOUT < instant) {

try {
String prjId = d.getId();
logger.info("Removing stale deployment {}", prjId);
entryManager.remove(d.getDn(), Deployment.class);
} catch (Exception e) {
logger.error("Error removing deployment", e);
}
}
}

}

private static String dnFromQname(String qname) {
return String.format("%s=%s,%s", Flow.ATTR_NAMES.QNAME,
Expand Down Expand Up @@ -569,9 +579,8 @@ private void purge(Set<String> dirs, Set<String> filesToRemove) throws IOExcepti
private void extract(String b64EncodedAssets, String destination) throws IOException {

if (b64EncodedAssets == null) return;

String name = rndName();
Path p = Files.createTempFile​(name, null);

Path p = Files.createTempFile​(rndName(), null);
logger.debug("Dumping decoded Base64 representation to {}", p);
Files.write(p, b64Decoder.decode(b64EncodedAssets.getBytes(UTF_8)));

Expand All @@ -586,11 +595,10 @@ private void extract(String b64EncodedAssets, String destination) throws IOExcep
}

private Path extractGamaFile(String b64EncodedContents) throws IOException {

String tmpdir = rndName();
Path p = Files.createTempDirectory(tmpdir);

Path p = Files.createTempDirectory(rndName());
logger.info("Extracting .gama file to {}", p);

extract(b64EncodedContents, p.toString());
return p;

Expand Down Expand Up @@ -650,8 +658,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
try {
Files.copy(dir, targetdir);
} catch (FileAlreadyExistsException e) {
if (!Files.isDirectory(targetdir))
throw e;
if (!Files.isDirectory(targetdir)) throw e;
}
return FileVisitResult.CONTINUE;

Expand Down Expand Up @@ -709,4 +716,4 @@ private void init() {

}

}
}