Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Bug/6366 optimize first email #117

Merged
merged 7 commits into from
Feb 21, 2025
Merged

Bug/6366 optimize first email #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2d6a6a7
add a timer for optimization
HavierD Feb 19, 2025
06df300
add notify user feature
HavierD Feb 19, 2025
d9ad1b5
implement notification
HavierD Feb 19, 2025
bf6e1d6
add comments
HavierD Feb 20, 2025
5b43f80
Merge branch 'main' into bug/6366-optimize-first-email
HavierD Feb 20, 2025
5350752
change dependency locations
HavierD Feb 21, 2025
6b76531
use property to organise aws versions
HavierD Feb 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
<org.geotools.version>29.2</org.geotools.version>
<maven.deploy.skip>true</maven.deploy.skip>
<versioning.updatePom>true</versioning.updatePom>
<aws.sdk.version>2.29.52</aws.sdk.version>
</properties>
<repositories>
<repository>
Expand Down Expand Up @@ -137,6 +138,21 @@
<artifactId>spatial4j</artifactId>
<version>0.8</version>
</dependency>
<dependency>
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Move duplicate version to a property above will save effort later

<groupId>software.amazon.awssdk</groupId>
<artifactId>batch</artifactId>
<version>${aws.sdk.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>auth</artifactId>
<version>${aws.sdk.version}</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>ses</artifactId>
<version>${aws.sdk.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
<distributionManagement>
Expand Down
6 changes: 4 additions & 2 deletions server/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>batch</artifactId>
<version>2.29.52</version>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>auth</artifactId>
<version>2.29.52</version>
</dependency>
<dependency>
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you put the depends version into the parent pom.xml dependency Management section? This applies to the all with version inside this pom.xml

<groupId>software.amazon.awssdk</groupId>
<artifactId>ses</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
Expand Down
9 changes: 9 additions & 0 deletions server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.util.StopWatch;

import java.io.IOException;
import java.math.BigDecimal;
Expand Down Expand Up @@ -560,6 +561,8 @@ public ElasticSearchBase.SearchResult<StacItemModel> searchFeatureSummary(String
};

try {
var queryTimer = new StopWatch();
queryTimer.start("query timer");
ElasticSearchBase.SearchResult<StacItemModel> result = new ElasticSearchBase.SearchResult<>();
result.setCollections(new ArrayList<>());

Expand All @@ -569,6 +572,10 @@ public ElasticSearchBase.SearchResult<StacItemModel> searchFeatureSummary(String
);
Iterable<CompositeBucket> response = pageableAggregation(builderSupplier, CompositeBucket.class, arguments, null);

queryTimer.stop();
log.info(queryTimer.prettyPrint());
var analyzingTimer = new StopWatch();
analyzingTimer.start("analyzing timer");
for (CompositeBucket node : response) {
if (node != null) {
StacItemModel. StacItemModelBuilder model = StacItemModel.builder();
Expand Down Expand Up @@ -601,6 +608,8 @@ public ElasticSearchBase.SearchResult<StacItemModel> searchFeatureSummary(String
result.getCollections().add(model.build());
}
}
analyzingTimer.stop();
log.info(analyzingTimer.prettyPrint());
return result;
}
catch (Exception e) {
Expand Down
29 changes: 17 additions & 12 deletions server/src/main/java/au/org/aodn/ogcapi/server/processes/RestApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,26 +32,31 @@ public class RestApi implements ProcessesApi {
// because the produces value in the interface declaration includes "/_" which may
// cause exception thrown sometimes. So i re-declared the produces value here
@RequestMapping(value = "/processes/{processID}/execution",
produces = { "application/json", "text/html" },
consumes = { "application/json" },
produces = {"application/json", "text/html"},
consumes = {"application/json"},
method = RequestMethod.POST)
public ResponseEntity<InlineResponse200> execute(
@Parameter(in = ParameterIn.PATH, required=true, schema=@Schema())
@Parameter(in = ParameterIn.PATH, required = true, schema = @Schema())
@PathVariable("processID")
String processID,
@Parameter(in = ParameterIn.DEFAULT, description = "Mandatory execute request JSON", required=true, schema=@Schema())
@Parameter(in = ParameterIn.DEFAULT, description = "Mandatory execute request JSON", required = true, schema = @Schema())
@Valid
@RequestBody Execute body){
@RequestBody Execute body) {

if (processID.equals(ProcessIdEnum.DOWNLOAD_DATASET.getValue())) {

try {
var response = restServices.downloadData(
(String) body.getInputs().get(DatasetDownloadEnums.Condition.UUID.getValue()),
(String) body.getInputs().get(DatasetDownloadEnums.Condition.START_DATE.getValue()),
(String) body.getInputs().get(DatasetDownloadEnums.Condition.END_DATE.getValue()),
body.getInputs().get(DatasetDownloadEnums.Condition.MULTI_POLYGON.getValue()),
(String) body.getInputs().get(DatasetDownloadEnums.Condition.RECIPIENT.getValue())
);

var uuid = (String) body.getInputs().get(DatasetDownloadEnums.Condition.UUID.getValue());
var startDate = (String) body.getInputs().get(DatasetDownloadEnums.Condition.START_DATE.getValue());
var endDate = (String) body.getInputs().get(DatasetDownloadEnums.Condition.END_DATE.getValue());
var multiPolygon = body.getInputs().get(DatasetDownloadEnums.Condition.MULTI_POLYGON.getValue());
var recipient = (String) body.getInputs().get(DatasetDownloadEnums.Condition.RECIPIENT.getValue());

// move the notify user email from data-access-service to here to make the first email faster
restServices.notifyUser(recipient, uuid, startDate, endDate);

var response = restServices.downloadData(uuid, startDate, endDate, multiPolygon, recipient);

var value = new InlineValue(response.getBody());
var status = new InlineValue(Integer.toString(HttpStatus.OK.value()));
Expand Down
63 changes: 48 additions & 15 deletions server/src/main/java/au/org/aodn/ogcapi/server/processes/RestServices.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import software.amazon.awssdk.services.batch.BatchClient;
import software.amazon.awssdk.services.batch.model.SubmitJobRequest;
import software.amazon.awssdk.services.batch.model.SubmitJobResponse;
import software.amazon.awssdk.services.ses.SesClient;
import software.amazon.awssdk.services.ses.model.*;

import java.util.HashMap;
import java.util.Map;
Expand All @@ -23,6 +25,30 @@ public RestServices(BatchClient batchClient, ObjectMapper objectMapper) {
this.objectMapper = objectMapper;
}

public void notifyUser(String recipient, String uuid, String startDate, String endDate) {

try(SesClient ses = SesClient.builder().build()) {
var subject = Content.builder().data("Start processing data file whose uuid is: " + uuid).build();
var content = Content.builder().data(generateStartedEmailContent(startDate, endDate)).build();

var body = Body.builder().text(content).build();
var message = Message.builder()
.subject(subject)
.body(body)
.build();

SendEmailRequest request = SendEmailRequest.builder()
.message(message)
.source(recipient)
.build();

ses.sendEmail(request);

} catch (SesException e) {
log.error("Error sending email: {}", e.getMessage());
}
}

public ResponseEntity<String> downloadData(
String id,
String startDate,
Expand All @@ -31,21 +57,21 @@ public ResponseEntity<String> downloadData(
String recipient
) throws JsonProcessingException {

Map<String, String> parameters = new HashMap<>();
parameters.put(DatasetDownloadEnums.Condition.UUID.getValue(), id);
parameters.put(DatasetDownloadEnums.Condition.START_DATE.getValue(), startDate);
parameters.put(DatasetDownloadEnums.Condition.END_DATE.getValue(), endDate);
parameters.put(DatasetDownloadEnums.Condition.MULTI_POLYGON.getValue(), objectMapper.writeValueAsString(polygons));
parameters.put(DatasetDownloadEnums.Condition.RECIPIENT.getValue(), recipient);


String jobId = submitJob(
"generating-data-file-for-" + recipient.replaceAll("[^a-zA-Z0-9-_]", "-"),
DatasetDownloadEnums.JobQueue.GENERATING_CSV_DATA_FILE.getValue(),
DatasetDownloadEnums.JobDefinition.GENERATE_CSV_DATA_FILE.getValue(),
parameters);
log.info("Job submitted with ID: " + jobId);
return ResponseEntity.ok("Job submitted with ID: " + jobId);
Map<String, String> parameters = new HashMap<>();
parameters.put(DatasetDownloadEnums.Condition.UUID.getValue(), id);
parameters.put(DatasetDownloadEnums.Condition.START_DATE.getValue(), startDate);
parameters.put(DatasetDownloadEnums.Condition.END_DATE.getValue(), endDate);
parameters.put(DatasetDownloadEnums.Condition.MULTI_POLYGON.getValue(), objectMapper.writeValueAsString(polygons));
parameters.put(DatasetDownloadEnums.Condition.RECIPIENT.getValue(), recipient);


String jobId = submitJob(
"generating-data-file-for-" + recipient.replaceAll("[^a-zA-Z0-9-_]", "-"),
DatasetDownloadEnums.JobQueue.GENERATING_CSV_DATA_FILE.getValue(),
DatasetDownloadEnums.JobDefinition.GENERATE_CSV_DATA_FILE.getValue(),
parameters);
log.info("Job submitted with ID: " + jobId);
return ResponseEntity.ok("Job submitted with ID: " + jobId);
}

private String submitJob(String jobName, String jobQueue, String jobDefinition, Map<String, String> parameters) {
Expand All @@ -60,4 +86,11 @@ private String submitJob(String jobName, String jobQueue, String jobDefinition,
SubmitJobResponse submitJobResponse = batchClient.submitJob(submitJobRequest);
return submitJobResponse.jobId();
}

private String generateStartedEmailContent(String startDate, String endDate) {
return "Your request has been received. Date range: Start Date: " +
startDate + ", End Date: " + endDate + ". Please wait for the result. " +
"'After the process is completed, you will receive an email '" +
"with the download link.";
}
}
Loading