diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/Maestro.java b/maestro-app/src/main/java/bio/overture/maestro/app/Maestro.java index 28296661..9dd23f43 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/Maestro.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/Maestro.java @@ -31,7 +31,7 @@ @SpringBootApplication @Import({RootConfiguration.class}) public class Maestro { - public static void main(String[] args) { - SpringApplication.run(Maestro.class, args); - } + public static void main(String[] args) { + SpringApplication.run(Maestro.class, args); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexAnalysisMessage.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexAnalysisMessage.java index 421f9918..a2f28d06 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexAnalysisMessage.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexAnalysisMessage.java @@ -23,13 +23,10 @@ @ToString @AllArgsConstructor class IndexAnalysisMessage { - @NonNull - private String analysisId; - @NonNull - private String studyId; - @NonNull - private String repositoryCode; + @NonNull private String analysisId; + @NonNull private String studyId; + @NonNull private String repositoryCode; - /** if callers set this flag it will do a remove instead of add.*/ - private Boolean removeAnalysis = false; + /** if callers set this flag it will do a remove instead of add. */ + private Boolean removeAnalysis = false; } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessage.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessage.java index f14067cc..df2d0a6e 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessage.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessage.java @@ -26,10 +26,9 @@ @ToString @AllArgsConstructor class IndexMessage { - private String analysisId; - private String studyId; - @NonNull - private String repositoryCode; - /** if callers set this flag it will do a remove instead of add.*/ - private Boolean removeAnalysis = false; + private String analysisId; + private String studyId; + @NonNull private String repositoryCode; + /** if callers set this flag it will do a remove instead of add. */ + private Boolean removeAnalysis = false; } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessagesHelper.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessagesHelper.java index 604fb4a9..a35d84d5 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessagesHelper.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessagesHelper.java @@ -19,45 +19,46 @@ import bio.overture.maestro.domain.api.message.IndexResult; import io.vavr.Tuple2; +import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import lombok.val; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.util.function.Supplier; - @Slf4j public class IndexMessagesHelper { - public static void handleIndexRepository(Supplier>> resultSupplier) { - val result = resultSupplier.get().blockOptional(); - val tuple = result.orElseThrow(() -> new RuntimeException("failed to obtain result")); - if (!tuple._2().isSuccessful()) { - log.error("failed to process message : {} successfully", tuple._1()); - throw new RuntimeException("failed to process the message"); - } + public static void handleIndexRepository( + Supplier>> resultSupplier) { + val result = resultSupplier.get().blockOptional(); + val tuple = result.orElseThrow(() -> new RuntimeException("failed to obtain result")); + if (!tuple._2().isSuccessful()) { + log.error("failed to process message : {} successfully", tuple._1()); + throw new RuntimeException("failed to process the message"); } + } - public static void handleIndexResult(Supplier>> resultSupplier) { - /* - * Why Blocking? - * - * - this is a stream consumer, it's supposed to process one message at a time - * the value of reactive processing diminishes since the queue provides a buffering level, - * without blocking it will async process the messages and if one fails we can - * async add it to a DLQ in the subscriber, However, I opted to use blocking because of the next point. - * - * - spring reactive cloud stream is deprecated in favor of spring cloud functions that support - * stream processing: https://cloud.spring.io/spring-cloud-static/spring-cloud-stream/2.2.0.RELEASE/spring-cloud-stream.html#spring_cloud_function - * so I don't want to use a deprecated library, and if needed we can switch to cloud function in future - * https://stackoverflow.com/questions/53438208/spring-cloud-stream-reactive-how-to-do-the-error-handling-in-case-of-reactive - */ - val result = resultSupplier.get().collectList().blockOptional(); - val tupleList = result.orElseThrow(() -> new RuntimeException("failed to obtain result")); - tupleList.forEach(tuple -> { - if (!tuple._2().isSuccessful()) { - log.error("failed to process message : {} successfully", tuple._1()); - throw new RuntimeException("failed to process the message"); - } + public static void handleIndexResult(Supplier>> resultSupplier) { + /* + * Why Blocking? + * + * - this is a stream consumer, it's supposed to process one message at a time + * the value of reactive processing diminishes since the queue provides a buffering level, + * without blocking it will async process the messages and if one fails we can + * async add it to a DLQ in the subscriber, However, I opted to use blocking because of the next point. + * + * - spring reactive cloud stream is deprecated in favor of spring cloud functions that support + * stream processing: https://cloud.spring.io/spring-cloud-static/spring-cloud-stream/2.2.0.RELEASE/spring-cloud-stream.html#spring_cloud_function + * so I don't want to use a deprecated library, and if needed we can switch to cloud function in future + * https://stackoverflow.com/questions/53438208/spring-cloud-stream-reactive-how-to-do-the-error-handling-in-case-of-reactive + */ + val result = resultSupplier.get().collectList().blockOptional(); + val tupleList = result.orElseThrow(() -> new RuntimeException("failed to obtain result")); + tupleList.forEach( + tuple -> { + if (!tuple._2().isSuccessful()) { + log.error("failed to process message : {} successfully", tuple._1()); + throw new RuntimeException("failed to process the message"); + } }); - } + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexRepositoryMessage.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexRepositoryMessage.java index a626f3d5..dae56011 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexRepositoryMessage.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexRepositoryMessage.java @@ -23,6 +23,5 @@ @ToString @AllArgsConstructor class IndexRepositoryMessage { - @NonNull - private String repositoryCode; + @NonNull private String repositoryCode; } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexStudyMessage.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexStudyMessage.java index ccc44cb3..1548701f 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexStudyMessage.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexStudyMessage.java @@ -23,8 +23,6 @@ @ToString @AllArgsConstructor class IndexStudyMessage { - @NonNull - private String studyId; - @NonNull - private String repositoryCode; + @NonNull private String studyId; + @NonNull private String repositoryCode; } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexingMessagesStreamListener.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexingMessagesStreamListener.java index dbeae9ea..3e48a2b3 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexingMessagesStreamListener.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexingMessagesStreamListener.java @@ -17,6 +17,9 @@ package bio.overture.maestro.app.infra.adapter.inbound.messaging; +import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexRepository; +import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexResult; + import bio.overture.maestro.domain.api.Indexer; import bio.overture.maestro.domain.api.exception.FailureData; import bio.overture.maestro.domain.api.message.*; @@ -32,111 +35,118 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexRepository; -import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexResult; - @Slf4j @EnableBinding(Sink.class) public class IndexingMessagesStreamListener { - private final Indexer indexer; - - public IndexingMessagesStreamListener(@NonNull Indexer indexer) { - this.indexer = indexer; - } - - @StreamListener(Sink.INPUT) - public void handleAnalysisMessage(@Payload IndexMessage indexMessage) { - if (isAnalysisReq(indexMessage)) { - val indexAnalysisMessage = new IndexAnalysisMessage(indexMessage.getAnalysisId(), - indexMessage.getStudyId(), - indexMessage.getRepositoryCode(), - indexMessage.getRemoveAnalysis()); - handleIndexResult(() -> this.indexOrRemoveAnalysis(indexAnalysisMessage)); - } else if (isStudyMsg(indexMessage)) { - val indexStudyMessage = new IndexStudyMessage(indexMessage.getStudyId(), indexMessage.getRepositoryCode()); - handleIndexResult(() -> this.indexStudy(indexStudyMessage)); - } else if (isRepoMsg(indexMessage)) { - val indexRepositoryMessage = new IndexRepositoryMessage(indexMessage.getRepositoryCode()); - handleIndexRepository(() -> this.indexRepository(indexRepositoryMessage)); - } else { - throw new IllegalArgumentException("invalid message format"); - } - } - - private boolean isAnalysisReq(IndexMessage indexMessage) { - return !StringUtils.isEmpty(indexMessage.getAnalysisId()) - && !StringUtils.isEmpty(indexMessage.getStudyId()) - && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); - } - - private boolean isStudyMsg(IndexMessage indexMessage) { - return StringUtils.isEmpty(indexMessage.getAnalysisId()) - && !StringUtils.isEmpty(indexMessage.getStudyId()) - && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); - } - - private boolean isRepoMsg(IndexMessage indexMessage) { - return StringUtils.isEmpty(indexMessage.getAnalysisId()) - && StringUtils.isEmpty(indexMessage.getStudyId()) - && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); + private final Indexer indexer; + + public IndexingMessagesStreamListener(@NonNull Indexer indexer) { + this.indexer = indexer; + } + + @StreamListener(Sink.INPUT) + public void handleAnalysisMessage(@Payload IndexMessage indexMessage) { + if (isAnalysisReq(indexMessage)) { + val indexAnalysisMessage = + new IndexAnalysisMessage( + indexMessage.getAnalysisId(), + indexMessage.getStudyId(), + indexMessage.getRepositoryCode(), + indexMessage.getRemoveAnalysis()); + handleIndexResult(() -> this.indexOrRemoveAnalysis(indexAnalysisMessage)); + } else if (isStudyMsg(indexMessage)) { + val indexStudyMessage = + new IndexStudyMessage(indexMessage.getStudyId(), indexMessage.getRepositoryCode()); + handleIndexResult(() -> this.indexStudy(indexStudyMessage)); + } else if (isRepoMsg(indexMessage)) { + val indexRepositoryMessage = new IndexRepositoryMessage(indexMessage.getRepositoryCode()); + handleIndexRepository(() -> this.indexRepository(indexRepositoryMessage)); + } else { + throw new IllegalArgumentException("invalid message format"); } - - private Flux> indexOrRemoveAnalysis(IndexAnalysisMessage msg) { - if (msg.getRemoveAnalysis()) { - return Flux.from(removeAnalysis(msg)); - } else { - return indexAnalysis(msg); - } + } + + private boolean isAnalysisReq(IndexMessage indexMessage) { + return !StringUtils.isEmpty(indexMessage.getAnalysisId()) + && !StringUtils.isEmpty(indexMessage.getStudyId()) + && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); + } + + private boolean isStudyMsg(IndexMessage indexMessage) { + return StringUtils.isEmpty(indexMessage.getAnalysisId()) + && !StringUtils.isEmpty(indexMessage.getStudyId()) + && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); + } + + private boolean isRepoMsg(IndexMessage indexMessage) { + return StringUtils.isEmpty(indexMessage.getAnalysisId()) + && StringUtils.isEmpty(indexMessage.getStudyId()) + && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); + } + + private Flux> indexOrRemoveAnalysis( + IndexAnalysisMessage msg) { + if (msg.getRemoveAnalysis()) { + return Flux.from(removeAnalysis(msg)); + } else { + return indexAnalysis(msg); } - - private Mono> removeAnalysis(IndexAnalysisMessage msg) { - return indexer.removeAnalysis(RemoveAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .studyId(msg.getStudyId()) - .analysisId(msg.getAnalysisId()) - .repositoryCode(msg.getRepositoryCode()) - .build()) + } + + private Mono> removeAnalysis(IndexAnalysisMessage msg) { + return indexer + .removeAnalysis( + RemoveAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId(msg.getStudyId()) + .analysisId(msg.getAnalysisId()) + .repositoryCode(msg.getRepositoryCode()) + .build()) .build()) - .map(out -> new Tuple2<>(msg, out)) - .onErrorResume((e) -> catchUnhandledErrors(msg, e)); - } + .map(out -> new Tuple2<>(msg, out)) + .onErrorResume((e) -> catchUnhandledErrors(msg, e)); + } + + private Flux> indexAnalysis(IndexAnalysisMessage msg) { + return indexer + .indexAnalysis( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId(msg.getStudyId()) + .analysisId(msg.getAnalysisId()) + .repositoryCode(msg.getRepositoryCode()) + .build()) + .build()) + .map(out -> new Tuple2<>(msg, out)) + .onErrorResume((e) -> catchUnhandledErrors(msg, e)); + } - private Flux> indexAnalysis(IndexAnalysisMessage msg) { - return indexer.indexAnalysis(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() + private Flux> indexStudy(IndexStudyMessage msg) { + return indexer + .indexStudy( + IndexStudyCommand.builder() .studyId(msg.getStudyId()) - .analysisId(msg.getAnalysisId()) .repositoryCode(msg.getRepositoryCode()) - .build() - ).build()) + .build()) + .map(out -> new Tuple2<>(msg, out)); + } + + private Mono> indexRepository( + IndexRepositoryMessage msg) { + return indexer + .indexRepository( + IndexStudyRepositoryCommand.builder().repositoryCode(msg.getRepositoryCode()).build()) .map(out -> new Tuple2<>(msg, out)) .onErrorResume((e) -> catchUnhandledErrors(msg, e)); - } - - private Flux> indexStudy(IndexStudyMessage msg) { - return indexer.indexStudy(IndexStudyCommand.builder() - .studyId(msg.getStudyId()) - .repositoryCode(msg.getRepositoryCode()) - .build()) - .map(out -> new Tuple2<>(msg, out)); - } - - private Mono> indexRepository(IndexRepositoryMessage msg) { - return indexer.indexRepository(IndexStudyRepositoryCommand.builder() - .repositoryCode(msg.getRepositoryCode()) - .build()) - .map(out -> new Tuple2<>(msg, out)) - .onErrorResume((e) -> catchUnhandledErrors(msg, e)); - } - - private Mono> catchUnhandledErrors(T msg, Throwable e) { - log.error("failed processing message: {} ", msg, e); - val indexResult = IndexResult.builder() - .successful(false) - .failureData(FailureData.builder().build()) - .build(); - return Mono.just(new Tuple2<>(msg, indexResult)); - } - + } + + private Mono> catchUnhandledErrors(T msg, Throwable e) { + log.error("failed processing message: {} ", msg, e); + val indexResult = + IndexResult.builder().successful(false).failureData(FailureData.builder().build()).build(); + return Mono.just(new Tuple2<>(msg, indexResult)); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/MessagingConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/MessagingConfig.java index 9ea4d77e..86c2b6fa 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/MessagingConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/MessagingConfig.java @@ -20,9 +20,8 @@ import bio.overture.maestro.app.infra.adapter.inbound.messaging.song.SongAnalysisStreamListener; import org.springframework.context.annotation.Import; - @Import({ - IndexingMessagesStreamListener.class, - SongAnalysisStreamListener.class, + IndexingMessagesStreamListener.class, + SongAnalysisStreamListener.class, }) -public class MessagingConfig { } +public class MessagingConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/AnalysisMessage.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/AnalysisMessage.java index 34e244e3..2afa60b0 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/AnalysisMessage.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/AnalysisMessage.java @@ -26,12 +26,8 @@ @ToString @AllArgsConstructor class AnalysisMessage { - @NonNull - private final String analysisId; - @NonNull - private final String studyId; - @NonNull - private final String state; - @NonNull - private final String songServerId; + @NonNull private final String analysisId; + @NonNull private final String studyId; + @NonNull private final String state; + @NonNull private final String songServerId; } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisSink.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisSink.java index 9be8584d..6ba9769d 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisSink.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisSink.java @@ -18,19 +18,13 @@ package bio.overture.maestro.app.infra.adapter.inbound.messaging.song; import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.messaging.Sink; import org.springframework.messaging.SubscribableChannel; public interface SongAnalysisSink { - /** - * Input channel name. - */ - String NAME = "songInput"; - - /** - * @return input channel. - */ - @Input(SongAnalysisSink.NAME) - SubscribableChannel songInput(); + /** Input channel name. */ + String NAME = "songInput"; + /** @return input channel. */ + @Input(SongAnalysisSink.NAME) + SubscribableChannel songInput(); } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListener.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListener.java index 9784d32d..da81dbf5 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListener.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListener.java @@ -17,6 +17,8 @@ package bio.overture.maestro.app.infra.adapter.inbound.messaging.song; +import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexResult; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.api.Indexer; import bio.overture.maestro.domain.api.message.AnalysisIdentifier; @@ -24,61 +26,63 @@ import bio.overture.maestro.domain.api.message.IndexResult; import bio.overture.maestro.domain.api.message.RemoveAnalysisCommand; import io.vavr.Tuple2; +import java.util.List; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.jetbrains.annotations.NotNull; import org.springframework.cloud.stream.annotation.EnableBinding; import org.springframework.cloud.stream.annotation.StreamListener; import org.springframework.messaging.handler.annotation.Payload; import reactor.core.publisher.Flux; -import java.util.List; - -import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexResult; - @Slf4j @EnableBinding(SongAnalysisSink.class) public class SongAnalysisStreamListener { - private Indexer indexer; - private final List indexableStudyStatuses; + private Indexer indexer; + private final List indexableStudyStatuses; - public SongAnalysisStreamListener(Indexer indexer, ApplicationProperties properties) { - this.indexer = indexer; - this.indexableStudyStatuses = List.of(properties.indexableStudyStatuses().split(",")); - } + public SongAnalysisStreamListener(Indexer indexer, ApplicationProperties properties) { + this.indexer = indexer; + this.indexableStudyStatuses = List.of(properties.indexableStudyStatuses().split(",")); + } - @StreamListener(SongAnalysisSink.NAME) - public void handleMessage(@Payload AnalysisMessage analysisMessage) { - log.info("received message : {}", analysisMessage); - handleIndexResult(() -> this.doHandle(analysisMessage)); - } + @StreamListener(SongAnalysisSink.NAME) + public void handleMessage(@Payload AnalysisMessage analysisMessage) { + log.info("received message : {}", analysisMessage); + handleIndexResult(() -> this.doHandle(analysisMessage)); + } - private Flux> doHandle(AnalysisMessage msg) { - Flux result; - try { - if (this.indexableStudyStatuses.contains(msg.getState())) { - result = indexer.indexAnalysis(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .repositoryCode(msg.getSongServerId()) - .studyId(msg.getStudyId()) - .analysisId(msg.getAnalysisId()) - .build()) + private Flux> doHandle(AnalysisMessage msg) { + Flux result; + try { + if (this.indexableStudyStatuses.contains(msg.getState())) { + result = + indexer.indexAnalysis( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .repositoryCode(msg.getSongServerId()) + .studyId(msg.getStudyId()) + .analysisId(msg.getAnalysisId()) + .build()) + .build()); + } else { + val mono = + indexer.removeAnalysis( + RemoveAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .analysisId(msg.getAnalysisId()) + .studyId(msg.getStudyId()) + .repositoryCode(msg.getSongServerId()) + .build()) .build()); - } else { - val mono = indexer.removeAnalysis(RemoveAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .analysisId(msg.getAnalysisId()) - .studyId(msg.getStudyId()) - .repositoryCode(msg.getSongServerId()) - .build() - ).build()); - result = Flux.from(mono); - } - return result.map(indexResult -> new Tuple2<>(msg, indexResult)); - } catch (Exception e) { - log.error("failed reading message: {} ", msg, e); - return Flux.just(new Tuple2<>(msg, IndexResult.builder().successful(false).build())); - } + result = Flux.from(mono); + } + return result.map(indexResult -> new Tuple2<>(msg, indexResult)); + } catch (Exception e) { + log.error("failed reading message: {} ", msg, e); + return Flux.just(new Tuple2<>(msg, IndexResult.builder().successful(false).build())); } + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ErrorDetails.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ErrorDetails.java index 6b491a93..b15562e3 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ErrorDetails.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ErrorDetails.java @@ -17,18 +17,17 @@ package bio.overture.maestro.app.infra.adapter.inbound.webapi; +import java.util.Date; import lombok.AllArgsConstructor; import lombok.Getter; -import java.util.Date; - /** * This class is a generic representation for errors to be returned by the GlobalWebExceptionHandler */ @AllArgsConstructor @Getter class ErrorDetails { - private Date timestamp; - private String message; - private String details; -} \ No newline at end of file + private Date timestamp; + private String message; + private String details; +} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/GlobalWebExceptionHandler.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/GlobalWebExceptionHandler.java index 5fdb3682..44e1b446 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/GlobalWebExceptionHandler.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/GlobalWebExceptionHandler.java @@ -17,8 +17,8 @@ package bio.overture.maestro.app.infra.adapter.inbound.webapi; - import bio.overture.maestro.domain.api.exception.NotFoundException; +import java.util.Date; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; import org.springframework.http.server.reactive.ServerHttpRequest; @@ -28,32 +28,28 @@ import org.springframework.web.bind.annotation.ResponseStatus; import reactor.core.publisher.Mono; -import java.util.Date; - @Slf4j @ControllerAdvice public class GlobalWebExceptionHandler { - @ExceptionHandler(NotFoundException.class) - @ResponseStatus(HttpStatus.NOT_FOUND) - @ResponseBody - public Mono resourceNotFoundException(NotFoundException ex, ServerHttpRequest request) { - log.error("Resource not found exception", ex); - return getErrorDetails(ex, request); - } - - @ExceptionHandler(Exception.class) - @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) - @ResponseBody - public Mono globalExceptionHandler(Exception ex, ServerHttpRequest request) { - log.error("Unhandled exception", ex); - return getErrorDetails(ex, request); - } - - private Mono getErrorDetails(Exception ex, ServerHttpRequest request) { - return Mono.just(new ErrorDetails(new Date(), - ex.getMessage(), request.getPath().toString()) - ); - } - -} \ No newline at end of file + @ExceptionHandler(NotFoundException.class) + @ResponseStatus(HttpStatus.NOT_FOUND) + @ResponseBody + public Mono resourceNotFoundException( + NotFoundException ex, ServerHttpRequest request) { + log.error("Resource not found exception", ex); + return getErrorDetails(ex, request); + } + + @ExceptionHandler(Exception.class) + @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) + @ResponseBody + public Mono globalExceptionHandler(Exception ex, ServerHttpRequest request) { + log.error("Unhandled exception", ex); + return getErrorDetails(ex, request); + } + + private Mono getErrorDetails(Exception ex, ServerHttpRequest request) { + return Mono.just(new ErrorDetails(new Date(), ex.getMessage(), request.getPath().toString())); + } +} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ManagementController.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ManagementController.java index e5020648..2fee3340 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ManagementController.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ManagementController.java @@ -71,8 +71,7 @@ public Mono removeAnalysis( .build()); } - @PostMapping( - "/index/repository/{repositoryCode}/study/{studyId}/analysis/{analysisId}") + @PostMapping("/index/repository/{repositoryCode}/study/{studyId}/analysis/{analysisId}") @ResponseStatus(HttpStatus.CREATED) public Flux indexAnalysis( @PathVariable String analysisId, diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/AnalysisCentricElasticSearchAdapter.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/AnalysisCentricElasticSearchAdapter.java index bcc0be42..fc1d22ae 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/AnalysisCentricElasticSearchAdapter.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/AnalysisCentricElasticSearchAdapter.java @@ -14,7 +14,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.PropertyNamingStrategy; - import java.util.*; import javax.inject.Inject; import lombok.NonNull; @@ -92,8 +91,7 @@ public Mono batchUpsertAnalysisRepositories( this.indexName, this.elasticsearchRestClient, AnalysisCentricDocument::getAnalysisId, - this::mapAnalysisToUpsertRepositoryQuery - ); + this::mapAnalysisToUpsertRepositoryQuery); } @SneakyThrows diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/ElasticSearchConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/ElasticSearchConfig.java index cf00ccd1..c3077289 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/ElasticSearchConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/ElasticSearchConfig.java @@ -17,9 +17,16 @@ package bio.overture.maestro.app.infra.adapter.outbound.indexing.elasticsearch; +import static bio.overture.maestro.app.infra.config.RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.stream.Collectors; import lombok.val; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; @@ -36,90 +43,85 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.stream.Collectors; - -import static bio.overture.maestro.app.infra.config.RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER; - /** * Elasticsearch related configuration this allows us to keep the beans package private to avoid - * other packages using them instead of the interface, and be more explicit about configuration scope. + * other packages using them instead of the interface, and be more explicit about configuration + * scope. */ @Configuration @Import({ - FileCentricElasticSearchAdapter.class, - AnalysisCentricElasticSearchAdapter.class, - SnakeCaseJacksonSearchResultMapper.class + FileCentricElasticSearchAdapter.class, + AnalysisCentricElasticSearchAdapter.class, + SnakeCaseJacksonSearchResultMapper.class }) public class ElasticSearchConfig { - /** - * this bean executes when the application starts it's used to initialize the - * indexes in elastic search server, can be extended as needed. - */ - @Bean - CommandLineRunner elasticsearchBootstrapper(FileCentricElasticSearchAdapter adapter) { - return (args) -> adapter.initialize(); - } + /** + * this bean executes when the application starts it's used to initialize the indexes in elastic + * search server, can be extended as needed. + */ + @Bean + CommandLineRunner elasticsearchBootstrapper(FileCentricElasticSearchAdapter adapter) { + return (args) -> adapter.initialize(); + } - @Bean - CommandLineRunner analysisElasticsearchBootstrapper(AnalysisCentricElasticSearchAdapter adapter) { - return (args) -> adapter.initialize(); - } + @Bean + CommandLineRunner analysisElasticsearchBootstrapper(AnalysisCentricElasticSearchAdapter adapter) { + return (args) -> adapter.initialize(); + } - @Bean - RestHighLevelClient client(ApplicationProperties properties) { - val httpHostArrayList = new ArrayList(properties.elasticSearchClusterNodes() - .stream() - .map(HttpHost::create) - .collect(Collectors.toUnmodifiableList())); - val builder = RestClient.builder(httpHostArrayList.toArray(new HttpHost[]{})); + @Bean + RestHighLevelClient client(ApplicationProperties properties) { + val httpHostArrayList = + new ArrayList( + properties.elasticSearchClusterNodes().stream() + .map(HttpHost::create) + .collect(Collectors.toUnmodifiableList())); + val builder = RestClient.builder(httpHostArrayList.toArray(new HttpHost[] {})); - builder.setHttpClientConfigCallback((httpAsyncClientBuilder) -> { - val connectTimeout = properties.elasticSearchClientConnectionTimeoutMillis(); - val timeout = properties.elasticSearchClientSocketTimeoutMillis(); - val requestConfigBuilder = RequestConfig.custom(); + builder.setHttpClientConfigCallback( + (httpAsyncClientBuilder) -> { + val connectTimeout = properties.elasticSearchClientConnectionTimeoutMillis(); + val timeout = properties.elasticSearchClientSocketTimeoutMillis(); + val requestConfigBuilder = RequestConfig.custom(); - if (connectTimeout > 0) { - requestConfigBuilder.setConnectTimeout(connectTimeout); - requestConfigBuilder.setConnectionRequestTimeout(connectTimeout); - } - if (timeout > 0) { - requestConfigBuilder.setSocketTimeout(timeout); - } - httpAsyncClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); + if (connectTimeout > 0) { + requestConfigBuilder.setConnectTimeout(connectTimeout); + requestConfigBuilder.setConnectionRequestTimeout(connectTimeout); + } + if (timeout > 0) { + requestConfigBuilder.setSocketTimeout(timeout); + } + httpAsyncClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); - if (properties.elasticSearchTlsTrustSelfSigned()) { - SSLContextBuilder sslCtxBuilder = new SSLContextBuilder(); - try { - sslCtxBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy()); - httpAsyncClientBuilder.setSSLContext(sslCtxBuilder.build()); - } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { - throw new RuntimeException("failed to build Elastic rest client"); - } + if (properties.elasticSearchTlsTrustSelfSigned()) { + SSLContextBuilder sslCtxBuilder = new SSLContextBuilder(); + try { + sslCtxBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy()); + httpAsyncClientBuilder.setSSLContext(sslCtxBuilder.build()); + } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { + throw new RuntimeException("failed to build Elastic rest client"); } + } - // set the credentials provider for auth - if (properties.elasticSearchBasicAuthEnabled()) { - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, - new UsernamePasswordCredentials(properties.elasticSearchAuthUser(), - properties.elasticSearchAuthPassword())); - httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); - } - return httpAsyncClientBuilder; + // set the credentials provider for auth + if (properties.elasticSearchBasicAuthEnabled()) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials( + properties.elasticSearchAuthUser(), properties.elasticSearchAuthPassword())); + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + } + return httpAsyncClientBuilder; }); - return new RestHighLevelClient(builder); - } - - @Bean(name = ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) - ObjectMapper documentObjectMapper() { - val mapper = new ObjectMapper(); - mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); - return mapper; - } + return new RestHighLevelClient(builder); + } + @Bean(name = ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) + ObjectMapper documentObjectMapper() { + val mapper = new ObjectMapper(); + mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); + return mapper; + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapter.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapter.java index f27764ea..597b442c 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapter.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapter.java @@ -116,8 +116,7 @@ public Mono batchUpsertFileRepositories( this.indexName, this.elasticsearchRestClient, this::getAnalysisId, - this::mapFileToUpsertRepositoryQuery - ); + this::mapFileToUpsertRepositoryQuery); } private String getAnalysisId(FileCentricDocument d) { @@ -197,8 +196,7 @@ private void deleteByAnalysisIdRunnable(String analysisId) { QueryBuilders.boolQuery() .must( QueryBuilders.termQuery( - FileCentricDocument.Fields.analysis + "." + "analysis_id", - analysisId))); + FileCentricDocument.Fields.analysis + "." + "analysis_id", analysisId))); this.elasticsearchRestClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SearchAdapterHelper.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SearchAdapterHelper.java index 30342ef7..fcecd970 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SearchAdapterHelper.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SearchAdapterHelper.java @@ -6,6 +6,11 @@ import io.github.resilience4j.retry.Retry; import io.github.resilience4j.retry.RetryConfig; import io.vavr.control.Try; +import java.io.IOException; +import java.time.Duration; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.SneakyThrows; @@ -24,179 +29,182 @@ import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; -import java.io.IOException; -import java.time.Duration; -import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; - - @Slf4j @NoArgsConstructor public class SearchAdapterHelper { - private static final String ANALYSIS_ID = "analysisId"; - - public static Mono batchUpsertDocuments( - @NonNull List documents, - int documentsPerBulkRequest, - int maxRetriesAttempts, - long retriesWaitDuration, - String indexName, - RestHighLevelClient client, - Function documentAnalysisIdExtractor, - Function mapper - ) { - log.debug( - "in batchUpsertAnalysisRepositories, analyses count: {} ", - documents.size()); - return Mono.fromSupplier( - () -> bulkUpsertAnalysisRepositories(documents, + private static final String ANALYSIS_ID = "analysisId"; + + public static Mono batchUpsertDocuments( + @NonNull List documents, + int documentsPerBulkRequest, + int maxRetriesAttempts, + long retriesWaitDuration, + String indexName, + RestHighLevelClient client, + Function documentAnalysisIdExtractor, + Function mapper) { + log.debug("in batchUpsertAnalysisRepositories, analyses count: {} ", documents.size()); + return Mono.fromSupplier( + () -> + bulkUpsertAnalysisRepositories( + documents, + documentsPerBulkRequest, + maxRetriesAttempts, + retriesWaitDuration, + indexName, + client, + documentAnalysisIdExtractor, + mapper)) + .subscribeOn(Schedulers.elastic()); + } + + @SneakyThrows + private static IndexResult bulkUpsertAnalysisRepositories( + List analyses, + int documentsPerBulkRequest, + int maxRetriesAttempts, + long retriesWaitDuration, + String indexName, + RestHighLevelClient client, + Function documentAnalysisIdExtractor, + Function mapper) { + log.trace( + "in SearchAdapterHelper - bulkUpsertAnalysisRepositories, analyses count : {} ", + analyses.size()); + val failures = + Parallel.blockingScatterGather( + analyses, documentsPerBulkRequest, - maxRetriesAttempts, - retriesWaitDuration, - indexName, - client, - documentAnalysisIdExtractor, - mapper) - ).subscribeOn(Schedulers.elastic()); - } - - @SneakyThrows - private static IndexResult bulkUpsertAnalysisRepositories(List analyses, - int documentsPerBulkRequest, - int maxRetriesAttempts, - long retriesWaitDuration, - String indexName, - RestHighLevelClient client, - Function documentAnalysisIdExtractor, - Function mapper) { - log.trace("in SearchAdapterHelper - bulkUpsertAnalysisRepositories, analyses count : {} ", analyses.size()); - val failures = - Parallel.blockingScatterGather(analyses, - documentsPerBulkRequest, - (list) -> tryBulkUpsertRequestForPart(list, maxRetriesAttempts, retriesWaitDuration, mapper, documentAnalysisIdExtractor, client)) - .stream() - .flatMap(Set::stream) - .collect(Collectors.toUnmodifiableSet()); - return buildIndexResult(failures, indexName); - } - - @NotNull - private static Set tryBulkUpsertRequestForPart( - Map.Entry> entry, - int maxRetriesAttempts, - long retriesWaitDuration, - Function mapper, - Function documentAnalysisIdExtractor, - RestHighLevelClient client - ) { - val partNum = entry.getKey(); - val listPart = entry.getValue(); - val listPartHash = Objects.hashCode(listPart); - val retry = buildRetry(maxRetriesAttempts, retriesWaitDuration); - val decorated = - Retry.>decorateCheckedSupplier( - retry, - () -> { - log.trace( - "SearchAdapterHelper - tryBulkUpsertRequestForPart, sending part#: {}, hash: {} ", - partNum, - listPartHash); - doRequestForPart(listPart, mapper, client); - log.trace("SearchAdapterHelper - tryBulkUpsertRequestForPart: done bulk upsert all docs"); - return Set.of(); + (list) -> + tryBulkUpsertRequestForPart( + list, + maxRetriesAttempts, + retriesWaitDuration, + mapper, + documentAnalysisIdExtractor, + client)) + .stream() + .flatMap(Set::stream) + .collect(Collectors.toUnmodifiableSet()); + return buildIndexResult(failures, indexName); + } + + @NotNull + private static Set tryBulkUpsertRequestForPart( + Map.Entry> entry, + int maxRetriesAttempts, + long retriesWaitDuration, + Function mapper, + Function documentAnalysisIdExtractor, + RestHighLevelClient client) { + val partNum = entry.getKey(); + val listPart = entry.getValue(); + val listPartHash = Objects.hashCode(listPart); + val retry = buildRetry(maxRetriesAttempts, retriesWaitDuration); + val decorated = + Retry.>decorateCheckedSupplier( + retry, + () -> { + log.trace( + "SearchAdapterHelper - tryBulkUpsertRequestForPart, sending part#: {}, hash: {} ", + partNum, + listPartHash); + doRequestForPart(listPart, mapper, client); + log.trace( + "SearchAdapterHelper - tryBulkUpsertRequestForPart: done bulk upsert all docs"); + return Set.of(); + }); + val result = + Try.of(decorated) + .recover( + (t) -> { + log.error( + "failed sending request for: part#: {}, hash: {} to elastic search," + + " gathering failed Ids.", + partNum, + listPartHash, + t); + return listPart.stream() + .map(documentAnalysisIdExtractor) + .collect(Collectors.toUnmodifiableSet()); }); - val result = - Try.of(decorated) - .recover( - (t) -> { - log.error( - "failed sending request for: part#: {}, hash: {} to elastic search," - + " gathering failed Ids.", - partNum, - listPartHash, - t); - return listPart.stream() - .map(documentAnalysisIdExtractor) - .collect(Collectors.toUnmodifiableSet()); - }); - return result.get(); + return result.get(); + } + + private static void doRequestForPart( + List listPart, Function mapper, RestHighLevelClient client) + throws IOException { + bulkUpdateRequest(listPart.stream().map(mapper).collect(Collectors.toList()), client); + } + + private static void bulkUpdateRequest(List requests, RestHighLevelClient client) + throws IOException { + val bulkRequest = buildBulkUpdateRequest(requests); + checkForBulkUpdateFailure(client.bulk(bulkRequest, RequestOptions.DEFAULT)); + } + + public static IndexResult buildIndexResult( + @NonNull Set failures, @NonNull String indexName) { + val fails = + failures.isEmpty() + ? FailureData.builder().build() + : FailureData.builder().failingIds(Map.of(ANALYSIS_ID, failures)).build(); + return IndexResult.builder() + .indexName(indexName) + .failureData(fails) + .successful(failures.isEmpty()) + .build(); + } + + public static Retry buildRetry(int maxRetriesAttempts, long retriesWaitDuration) { + val retryConfig = + RetryConfig.custom() + .maxAttempts(maxRetriesAttempts) + .retryExceptions(IOException.class) + .waitDuration(Duration.ofMillis(retriesWaitDuration)) + .build(); + val retry = Retry.of("tryBulkUpsertRequestForPart", retryConfig); + return retry; + } + + public static BulkRequest buildBulkUpdateRequest(List requests) + throws IOException { + val bulkRequest = new BulkRequest(); + for (UpdateRequest query : requests) { + bulkRequest.add(prepareUpdate(query)); } - - private static void doRequestForPart(List listPart, - Function mapper, - RestHighLevelClient client) throws IOException { - bulkUpdateRequest( - listPart.stream() - .map(mapper) - .collect(Collectors.toList()), - client); + return bulkRequest; + } + + public static void checkForBulkUpdateFailure(BulkResponse bulkResponse) { + if (bulkResponse.hasFailures()) { + val failedDocuments = new HashMap(); + for (BulkItemResponse item : bulkResponse.getItems()) { + if (item.isFailed()) failedDocuments.put(item.getId(), item.getFailureMessage()); + } + throw new RuntimeException( + "Bulk indexing has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages [" + + failedDocuments + + "]"); } - - private static void bulkUpdateRequest(List requests, RestHighLevelClient client) throws IOException { - val bulkRequest = buildBulkUpdateRequest(requests); - checkForBulkUpdateFailure(client.bulk(bulkRequest, RequestOptions.DEFAULT)); - } - - public static IndexResult buildIndexResult(@NonNull Set failures, @NonNull String indexName){ - val fails = - failures.isEmpty() - ? FailureData.builder().build() - : FailureData.builder().failingIds(Map.of(ANALYSIS_ID, failures)).build(); - return IndexResult.builder() - .indexName(indexName) - .failureData(fails).successful(failures.isEmpty()).build(); - } - - public static Retry buildRetry(int maxRetriesAttempts, long retriesWaitDuration){ - val retryConfig = - RetryConfig.custom() - .maxAttempts(maxRetriesAttempts) - .retryExceptions(IOException.class) - .waitDuration(Duration.ofMillis(retriesWaitDuration)) - .build(); - val retry = Retry.of("tryBulkUpsertRequestForPart", retryConfig); - return retry; - } - - public static BulkRequest buildBulkUpdateRequest(List requests) throws IOException { - val bulkRequest = new BulkRequest(); - for (UpdateRequest query : requests) { - bulkRequest.add(prepareUpdate(query)); - } - return bulkRequest; - } - - public static void checkForBulkUpdateFailure(BulkResponse bulkResponse) { - if (bulkResponse.hasFailures()) { - val failedDocuments = new HashMap(); - for (BulkItemResponse item : bulkResponse.getItems()) { - if (item.isFailed()) failedDocuments.put(item.getId(), item.getFailureMessage()); - } - throw new RuntimeException( - "Bulk indexing has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages [" - + failedDocuments - + "]"); - } - } - - public static Script getInline(Map parameters) { - val inline = - new Script( - ScriptType.INLINE, - "painless", - "if (!ctx._source.repositories.contains(params.repository)) { ctx._source.repositories.add(params.repository) }", - parameters); - return inline; - } - - private static UpdateRequest prepareUpdate(UpdateRequest req) { - Assert.notNull(req, "No IndexRequest define for Query"); - String indexName = req.index(); - Assert.notNull(indexName, "No index defined for Query"); - Assert.notNull(req.id(), "No Id define for Query"); - return req; - } - -} \ No newline at end of file + } + + public static Script getInline(Map parameters) { + val inline = + new Script( + ScriptType.INLINE, + "painless", + "if (!ctx._source.repositories.contains(params.repository)) { ctx._source.repositories.add(params.repository) }", + parameters); + return inline; + } + + private static UpdateRequest prepareUpdate(UpdateRequest req) { + Assert.notNull(req, "No IndexRequest define for Query"); + String indexName = req.index(); + Assert.notNull(indexName, "No index defined for Query"); + Assert.notNull(req.id(), "No Id define for Query"); + return req; + } +} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SnakeCaseJacksonSearchResultMapper.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SnakeCaseJacksonSearchResultMapper.java index a51f6bc8..2a950aa0 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SnakeCaseJacksonSearchResultMapper.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SnakeCaseJacksonSearchResultMapper.java @@ -19,38 +19,37 @@ import bio.overture.maestro.app.infra.config.RootConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Arrays; +import java.util.LinkedList; import lombok.SneakyThrows; import lombok.val; import org.elasticsearch.action.get.MultiGetResponse; import org.springframework.beans.factory.annotation.Qualifier; -import java.util.Arrays; -import java.util.LinkedList; - - class SnakeCaseJacksonSearchResultMapper { - private ObjectMapper objectMapper; - - public SnakeCaseJacksonSearchResultMapper(@Qualifier(RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) - ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - } - - @SneakyThrows - LinkedList mapResults(MultiGetResponse responses, Class clazz) { - val list = new LinkedList(); - Arrays.stream(responses.getResponses()) - .filter((response) -> !response.isFailed() && response.getResponse().isExists()) - .forEach((response) -> { - T result = convertSourceToObject(clazz, response.getResponse().getSourceAsString()); - list.add(result); + private ObjectMapper objectMapper; + + public SnakeCaseJacksonSearchResultMapper( + @Qualifier(RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @SneakyThrows + LinkedList mapResults(MultiGetResponse responses, Class clazz) { + val list = new LinkedList(); + Arrays.stream(responses.getResponses()) + .filter((response) -> !response.isFailed() && response.getResponse().isExists()) + .forEach( + (response) -> { + T result = convertSourceToObject(clazz, response.getResponse().getSourceAsString()); + list.add(result); }); - return list; - } - - @SneakyThrows - private T convertSourceToObject(Class clazz, String source) { - return objectMapper.readValue(source, clazz); - } -} \ No newline at end of file + return list; + } + + @SneakyThrows + private T convertSourceToObject(Class clazz, String source) { + return objectMapper.readValue(source, clazz); + } +} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ApplicationPropertiesExclusionRulesDAO.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ApplicationPropertiesExclusionRulesDAO.java index 0811f51f..6f9f3cc5 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ApplicationPropertiesExclusionRulesDAO.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ApplicationPropertiesExclusionRulesDAO.java @@ -17,89 +17,88 @@ package bio.overture.maestro.app.infra.adapter.outbound.indexing.rules; +import static java.text.MessageFormat.format; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionRule; import bio.overture.maestro.domain.entities.indexing.rules.IDExclusionRule; import bio.overture.maestro.domain.entities.metadata.study.*; import bio.overture.maestro.domain.port.outbound.indexing.rules.ExclusionRulesDAO; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import lombok.*; -import lombok.extern.slf4j.Slf4j; -import org.springframework.core.io.Resource; -import reactor.core.publisher.Mono; - -import javax.annotation.PostConstruct; -import javax.inject.Inject; -import java.io.FileNotFoundException; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; - -import static java.text.MessageFormat.format; +import javax.inject.Inject; +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; /** - * Rules dao backed by application properties as source, it assumes a yaml - * structure matching the class: {@link RuleConfig} + * Rules dao backed by application properties as source, it assumes a yaml structure matching the + * class: {@link RuleConfig} */ @Slf4j public class ApplicationPropertiesExclusionRulesDAO implements ExclusionRulesDAO { - private Map, List> exclusionRules; + private Map, List> exclusionRules; - @Inject - public ApplicationPropertiesExclusionRulesDAO(ApplicationProperties properties) { - this.init(properties.idExclusionRules()); - } + @Inject + public ApplicationPropertiesExclusionRulesDAO(ApplicationProperties properties) { + this.init(properties.idExclusionRules()); + } - @SneakyThrows - public Mono, List>> getExclusionRules() { - return Mono.just(this.exclusionRules); - } + @SneakyThrows + public Mono, List>> getExclusionRules() { + return Mono.just(this.exclusionRules); + } - private void init(Map> idExclusionRules) { - try { - if (idExclusionRules == null || idExclusionRules.isEmpty()) { - this.exclusionRules = Map.of(); - return; - } + private void init(Map> idExclusionRules) { + try { + if (idExclusionRules == null || idExclusionRules.isEmpty()) { + this.exclusionRules = Map.of(); + return; + } - val rulesByEntity = new LinkedHashMap, List>(); - idExclusionRules.forEach((entity, ids) -> { - if (ids.isEmpty()) return; - val entityClass = getClassFor(entity); - rulesByEntity.put(entityClass, List.of(IDExclusionRule.builder() - .clazz(entityClass) - .ids(ids) - .build() - ) - ); - }); + val rulesByEntity = new LinkedHashMap, List>(); + idExclusionRules.forEach( + (entity, ids) -> { + if (ids.isEmpty()) return; + val entityClass = getClassFor(entity); + rulesByEntity.put( + entityClass, + List.of(IDExclusionRule.builder().clazz(entityClass).ids(ids).build())); + }); - this.exclusionRules = Map.copyOf(rulesByEntity); - log.info("loaded exclusionRules :{}", this.exclusionRules.size()); - } catch (Exception e) { - this.exclusionRules = Map.of(); - log.error("failed to read exclusion rules", e); - } + this.exclusionRules = Map.copyOf(rulesByEntity); + log.info("loaded exclusionRules :{}", this.exclusionRules.size()); + } catch (Exception e) { + this.exclusionRules = Map.of(); + log.error("failed to read exclusion rules", e); } + } - private Class getClassFor(String entity) { - switch (entity) { - case "studyId": return Study.class; - case "analysis": return Analysis.class; - case "files": return File.class; - case "donor": return Donor.class; - case "samples": return Sample.class; - case "specimen" : return Specimen.class; - } - throw new IllegalArgumentException(format("entity : {0} is not recognized for exclusion rules", entity)); + private Class getClassFor(String entity) { + switch (entity) { + case "studyId": + return Study.class; + case "analysis": + return Analysis.class; + case "files": + return File.class; + case "donor": + return Donor.class; + case "samples": + return Sample.class; + case "specimen": + return Specimen.class; } + throw new IllegalArgumentException( + format("entity : {0} is not recognized for exclusion rules", entity)); + } - @Getter - @NoArgsConstructor - @AllArgsConstructor - private static class RuleConfig { - private Map> byId; - } + @Getter + @NoArgsConstructor + @AllArgsConstructor + private static class RuleConfig { + private Map> byId; + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ExclusionRulesConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ExclusionRulesConfig.java index 147022d0..fa677306 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ExclusionRulesConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ExclusionRulesConfig.java @@ -22,10 +22,9 @@ /** * Elasticsearch related configuration this allows us to keep the beans package private to avoid - * other packages using them instead of the interface, and be more explicit about configuration scope. + * other packages using them instead of the interface, and be more explicit about configuration + * scope. */ @Configuration -@Import({ - ApplicationPropertiesExclusionRulesDAO.class -}) -public class ExclusionRulesConfig { } +@Import({ApplicationPropertiesExclusionRulesDAO.class}) +public class ExclusionRulesConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/PropertyFileStudyRepositoryDAO.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/PropertyFileStudyRepositoryDAO.java index 81827385..2989ae3f 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/PropertyFileStudyRepositoryDAO.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/PropertyFileStudyRepositoryDAO.java @@ -17,10 +17,14 @@ package bio.overture.maestro.app.infra.adapter.outbound.metadata.repostiory; +import static bio.overture.maestro.domain.utility.Exceptions.notFound; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.app.infra.config.properties.PropertiesFileRepository; import bio.overture.maestro.domain.entities.metadata.repository.StudyRepository; import bio.overture.maestro.domain.port.outbound.metadata.repository.StudyRepositoryDAO; +import java.util.List; +import javax.inject.Inject; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.NonNull; @@ -28,57 +32,55 @@ import lombok.val; import reactor.core.publisher.Mono; -import javax.inject.Inject; -import java.util.List; - -import static bio.overture.maestro.domain.utility.Exceptions.notFound; - /** - * Properties files backed repository store, reads the information by binding - * to the application.config files property: maestro.repositories - * the configurable attributes can be found here: {@link PropertiesFileRepository} + * Properties files backed repository store, reads the information by binding to the + * application.config files property: maestro.repositories the configurable attributes can be found + * here: {@link PropertiesFileRepository} * - * this serves as a default in memory store, more sophisticated cases may create a DB backed store. + *

this serves as a default in memory store, more sophisticated cases may create a DB backed + * store. */ @Slf4j @Getter @NoArgsConstructor class PropertyFileStudyRepositoryDAO implements StudyRepositoryDAO { - private static final String MSG_REPO_NOT_FOUND = "Repository {0} not found"; - private List repositories; + private static final String MSG_REPO_NOT_FOUND = "Repository {0} not found"; + private List repositories; - @Inject - public PropertyFileStudyRepositoryDAO(ApplicationProperties properties) { - this.repositories = properties.repositories(); - } + @Inject + public PropertyFileStudyRepositoryDAO(ApplicationProperties properties) { + this.repositories = properties.repositories(); + } - @Override - @NonNull - public Mono getFilesRepository(@NonNull String code) { - val repository = repositories.stream() - .filter(propertiesFileRepository -> propertiesFileRepository.getCode().equalsIgnoreCase(code)) + @Override + @NonNull + public Mono getFilesRepository(@NonNull String code) { + val repository = + repositories.stream() + .filter( + propertiesFileRepository -> + propertiesFileRepository.getCode().equalsIgnoreCase(code)) .distinct() - .map(this :: toFilesRepository) + .map(this::toFilesRepository) .findFirst() .orElse(null); - if (repository == null) { - return Mono.error(notFound(MSG_REPO_NOT_FOUND, code)); - } - log.debug("loaded repository : {}", repository); - return Mono.just(repository); - } - - private StudyRepository toFilesRepository(PropertiesFileRepository propertiesFileRepository) { - log.trace("Converting : {} to StudyRepository ", propertiesFileRepository); - return StudyRepository.builder() - .code(propertiesFileRepository.getCode()) - .url(propertiesFileRepository.getUrl()) - .name(propertiesFileRepository.getName()) - .organization(propertiesFileRepository.getOrganization()) - .country(propertiesFileRepository.getCountry()) - .storageType(propertiesFileRepository.getStorageType()) - .build(); + if (repository == null) { + return Mono.error(notFound(MSG_REPO_NOT_FOUND, code)); } + log.debug("loaded repository : {}", repository); + return Mono.just(repository); + } + private StudyRepository toFilesRepository(PropertiesFileRepository propertiesFileRepository) { + log.trace("Converting : {} to StudyRepository ", propertiesFileRepository); + return StudyRepository.builder() + .code(propertiesFileRepository.getCode()) + .url(propertiesFileRepository.getUrl()) + .name(propertiesFileRepository.getName()) + .organization(propertiesFileRepository.getOrganization()) + .country(propertiesFileRepository.getCountry()) + .storageType(propertiesFileRepository.getStorageType()) + .build(); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/RepositoryConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/RepositoryConfig.java index 7a95fd66..9e26ce63 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/RepositoryConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/RepositoryConfig.java @@ -21,8 +21,5 @@ import org.springframework.context.annotation.Import; @Configuration -@Import({ - PropertyFileStudyRepositoryDAO.class -}) -public class RepositoryConfig { -} +@Import({PropertyFileStudyRepositoryDAO.class}) +public class RepositoryConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongConfig.java index ce10142d..f58fa085 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongConfig.java @@ -21,8 +21,5 @@ import org.springframework.context.annotation.Import; @Configuration -@Import({ - SongStudyDAO.class -}) -public class SongConfig { -} +@Import({SongStudyDAO.class}) +public class SongConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAO.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAO.java index 430e3517..7b3fe821 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAO.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAO.java @@ -17,6 +17,10 @@ package bio.overture.maestro.app.infra.adapter.outbound.metadata.study.song; +import static bio.overture.maestro.domain.utility.Exceptions.notFound; +import static java.text.MessageFormat.format; +import static reactor.core.publisher.Mono.error; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.api.exception.NotFoundException; import bio.overture.maestro.domain.entities.metadata.study.Analysis; @@ -25,137 +29,167 @@ import bio.overture.maestro.domain.port.outbound.metadata.study.GetAnalysisCommand; import bio.overture.maestro.domain.port.outbound.metadata.study.GetStudyAnalysesCommand; import bio.overture.maestro.domain.port.outbound.metadata.study.StudyDAO; +import java.time.Duration; +import java.util.List; +import java.util.function.Function; +import javax.inject.Inject; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; -import org.springframework.web.reactive.function.client.ExchangeStrategies; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.retry.Retry; -import javax.inject.Inject; -import java.time.Duration; -import java.util.List; -import java.util.function.Function; - -import static bio.overture.maestro.domain.utility.Exceptions.notFound; -import static java.text.MessageFormat.format; -import static reactor.core.publisher.Mono.error; @Slf4j class SongStudyDAO implements StudyDAO { - private static final String STUDY_ANALYSES_URL_TEMPLATE = "{0}/studies/{1}/analysis?analysisStates={2}"; - private static final String STUDY_ANALYSIS_URL_TEMPLATE = "{0}/studies/{1}/analysis/{2}"; - private static final String STUDIES_URL_TEMPLATE = "{0}/studies/all"; - private static final String MSG_STUDY_DOES_NOT_EXIST = "studyId {0} doesn't exist in the specified repository"; - private static final String MSG_ANALYSIS_DOES_NOT_EXIST = - "analysis {0} doesn't exist for studyId {1}, repository {2} (or not in a matching state)"; - private static final int FALLBACK_SONG_TIMEOUT = 60; - private static final int FALLBACK_SONG_ANALYSIS_TIMEOUT = 5; - private static final int FALLBACK_SONG_MAX_RETRY = 0; - private final WebClient webClient; - private final int songMaxRetries; - private final int minBackoffSec = 1; - private final int maxBackoffSec = 5; - private final String indexableStudyStatuses; - private final List indexableStudyStatusesList; - /** - * must be bigger than 0 or all calls will fail - */ - private final int studyCallTimeoutSeconds; - private final int analysisCallTimeoutSeconds; + private static final String STUDY_ANALYSES_URL_TEMPLATE = + "{0}/studies/{1}/analysis?analysisStates={2}"; + private static final String STUDY_ANALYSIS_URL_TEMPLATE = "{0}/studies/{1}/analysis/{2}"; + private static final String STUDIES_URL_TEMPLATE = "{0}/studies/all"; + private static final String MSG_STUDY_DOES_NOT_EXIST = + "studyId {0} doesn't exist in the specified repository"; + private static final String MSG_ANALYSIS_DOES_NOT_EXIST = + "analysis {0} doesn't exist for studyId {1}, repository {2} (or not in a matching state)"; + private static final int FALLBACK_SONG_TIMEOUT = 60; + private static final int FALLBACK_SONG_ANALYSIS_TIMEOUT = 5; + private static final int FALLBACK_SONG_MAX_RETRY = 0; + private final WebClient webClient; + private final int songMaxRetries; + private final int minBackoffSec = 1; + private final int maxBackoffSec = 5; + private final String indexableStudyStatuses; + private final List indexableStudyStatusesList; + /** must be bigger than 0 or all calls will fail */ + private final int studyCallTimeoutSeconds; + + private final int analysisCallTimeoutSeconds; - @Inject - public SongStudyDAO(@NonNull WebClient webClient, @NonNull ApplicationProperties applicationProperties) { - this.webClient = webClient; - this.indexableStudyStatuses = applicationProperties.indexableStudyStatuses(); - this.indexableStudyStatusesList = List.of(indexableStudyStatuses.split(",")); - this.songMaxRetries = applicationProperties.songMaxRetries() >= 0 ? applicationProperties.songMaxRetries() + @Inject + public SongStudyDAO( + @NonNull WebClient webClient, @NonNull ApplicationProperties applicationProperties) { + this.webClient = webClient; + this.indexableStudyStatuses = applicationProperties.indexableStudyStatuses(); + this.indexableStudyStatusesList = List.of(indexableStudyStatuses.split(",")); + this.songMaxRetries = + applicationProperties.songMaxRetries() >= 0 + ? applicationProperties.songMaxRetries() : FALLBACK_SONG_MAX_RETRY; - this.studyCallTimeoutSeconds = applicationProperties.songStudyCallTimeoutSeconds() > 0 ? applicationProperties.songStudyCallTimeoutSeconds() + this.studyCallTimeoutSeconds = + applicationProperties.songStudyCallTimeoutSeconds() > 0 + ? applicationProperties.songStudyCallTimeoutSeconds() : FALLBACK_SONG_TIMEOUT; - this.analysisCallTimeoutSeconds = applicationProperties.songAnalysisCallTimeoutSeconds() > 0 ? - applicationProperties.songAnalysisCallTimeoutSeconds(): FALLBACK_SONG_ANALYSIS_TIMEOUT; - } + this.analysisCallTimeoutSeconds = + applicationProperties.songAnalysisCallTimeoutSeconds() > 0 + ? applicationProperties.songAnalysisCallTimeoutSeconds() + : FALLBACK_SONG_ANALYSIS_TIMEOUT; + } - @Override - public Mono> getStudyAnalyses(GetStudyAnalysesCommand getStudyAnalysesCommand) { - log.trace("in getStudyAnalyses, args: {} ", getStudyAnalysesCommand); - val repoBaseUrl = getStudyAnalysesCommand.getFilesRepositoryBaseUrl(); - val studyId = getStudyAnalysesCommand.getStudyId(); - val analysisListType = new ParameterizedTypeReference>(){}; - val retryConfig = Retry.allBut(NotFoundException.class) + @Override + public Mono> getStudyAnalyses(GetStudyAnalysesCommand getStudyAnalysesCommand) { + log.trace("in getStudyAnalyses, args: {} ", getStudyAnalysesCommand); + val repoBaseUrl = getStudyAnalysesCommand.getFilesRepositoryBaseUrl(); + val studyId = getStudyAnalysesCommand.getStudyId(); + val analysisListType = new ParameterizedTypeReference>() {}; + val retryConfig = + Retry.allBut(NotFoundException.class) .retryMax(this.songMaxRetries) - .doOnRetry(retryCtx -> log.error("exception happened, retrying {}", getStudyAnalysesCommand, retryCtx.exception())) - .exponentialBackoff(Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); + .doOnRetry( + retryCtx -> + log.error( + "exception happened, retrying {}", + getStudyAnalysesCommand, + retryCtx.exception())) + .exponentialBackoff( + Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); - return this.webClient.get() - .uri(format(STUDY_ANALYSES_URL_TEMPLATE, repoBaseUrl, studyId, this.indexableStudyStatuses)) - .retrieve() - .onStatus(HttpStatus.NOT_FOUND :: equals, - clientResponse -> error(notFound(MSG_STUDY_DOES_NOT_EXIST, studyId))) - .bodyToMono(analysisListType) - .transform(retryAndTimeout(retryConfig, Duration.ofSeconds(this.studyCallTimeoutSeconds))) - .doOnSuccess((list) -> log.trace("getStudyAnalyses out, analyses count {} args: {}", - list.size(), getStudyAnalysesCommand)); - } + return this.webClient + .get() + .uri(format(STUDY_ANALYSES_URL_TEMPLATE, repoBaseUrl, studyId, this.indexableStudyStatuses)) + .retrieve() + .onStatus( + HttpStatus.NOT_FOUND::equals, + clientResponse -> error(notFound(MSG_STUDY_DOES_NOT_EXIST, studyId))) + .bodyToMono(analysisListType) + .transform(retryAndTimeout(retryConfig, Duration.ofSeconds(this.studyCallTimeoutSeconds))) + .doOnSuccess( + (list) -> + log.trace( + "getStudyAnalyses out, analyses count {} args: {}", + list.size(), + getStudyAnalysesCommand)); + } - @Override - public Flux getStudies(@NonNull GetAllStudiesCommand getAllStudiesCommand) { - log.trace("in getStudyAnalyses, args: {} ", getAllStudiesCommand); - val repoBaseUrl = getAllStudiesCommand.getFilesRepositoryBaseUrl(); - val StringListType = new ParameterizedTypeReference>(){}; - val retryConfig = Retry.allBut(NotFoundException.class) + @Override + public Flux getStudies(@NonNull GetAllStudiesCommand getAllStudiesCommand) { + log.trace("in getStudyAnalyses, args: {} ", getAllStudiesCommand); + val repoBaseUrl = getAllStudiesCommand.getFilesRepositoryBaseUrl(); + val StringListType = new ParameterizedTypeReference>() {}; + val retryConfig = + Retry.allBut(NotFoundException.class) .retryMax(this.songMaxRetries) - .doOnRetry(retryCtx -> log.error("retrying {}", getAllStudiesCommand, retryCtx.exception())) - .exponentialBackoff(Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); + .doOnRetry( + retryCtx -> log.error("retrying {}", getAllStudiesCommand, retryCtx.exception())) + .exponentialBackoff( + Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); - return this.webClient.get() - .uri(format(STUDIES_URL_TEMPLATE, repoBaseUrl)) - .accept(MediaType.APPLICATION_JSON) - .retrieve() - // we need to first parse as mono then stream it as flux because of this : - // https://github.com/spring-projects/spring-framework/issues/22662 - .bodyToMono(StringListType) - .transform(retryAndTimeout(retryConfig, Duration.ofSeconds(this.studyCallTimeoutSeconds))) - .flatMapMany(Flux::fromIterable) - .map(id -> Study.builder().studyId(id).build()); - } + return this.webClient + .get() + .uri(format(STUDIES_URL_TEMPLATE, repoBaseUrl)) + .accept(MediaType.APPLICATION_JSON) + .retrieve() + // we need to first parse as mono then stream it as flux because of this : + // https://github.com/spring-projects/spring-framework/issues/22662 + .bodyToMono(StringListType) + .transform(retryAndTimeout(retryConfig, Duration.ofSeconds(this.studyCallTimeoutSeconds))) + .flatMapMany(Flux::fromIterable) + .map(id -> Study.builder().studyId(id).build()); + } - @Override - public Mono getAnalysis(@NonNull GetAnalysisCommand command) { - log.trace("in getAnalysis, args: {} ", command); - val repoBaseUrl = command.getFilesRepositoryBaseUrl(); - val analysisId = command.getAnalysisId(); - val studyId = command.getStudyId(); - val retryConfig = Retry.allBut(NotFoundException.class) + @Override + public Mono getAnalysis(@NonNull GetAnalysisCommand command) { + log.trace("in getAnalysis, args: {} ", command); + val repoBaseUrl = command.getFilesRepositoryBaseUrl(); + val analysisId = command.getAnalysisId(); + val studyId = command.getStudyId(); + val retryConfig = + Retry.allBut(NotFoundException.class) .retryMax(this.songMaxRetries) - .doOnRetry(retryCtx -> log.error("exception happened, retrying {}", command, retryCtx.exception())) - .exponentialBackoff(Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); + .doOnRetry( + retryCtx -> + log.error("exception happened, retrying {}", command, retryCtx.exception())) + .exponentialBackoff( + Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); - return this.webClient.get() - .uri(format(STUDY_ANALYSIS_URL_TEMPLATE, repoBaseUrl, studyId, analysisId)) - .retrieve() - .onStatus(HttpStatus.NOT_FOUND::equals, - clientResponse -> error(notFound(MSG_ANALYSIS_DOES_NOT_EXIST, analysisId, studyId, repoBaseUrl))) - .bodyToMono(Analysis.class) - .transform(retryAndTimeout(retryConfig, Duration.ofSeconds(this.analysisCallTimeoutSeconds))) - .doOnSuccess((analysis) -> log.trace("getAnalysis out, analysis {} args: {}", - analysis, command)) - .flatMap(analysis -> { - if (!indexableStudyStatusesList.contains(analysis.getAnalysisState())) { - return error(notFound(MSG_ANALYSIS_DOES_NOT_EXIST, analysisId, studyId, repoBaseUrl)); - } - return Mono.just(analysis); + return this.webClient + .get() + .uri(format(STUDY_ANALYSIS_URL_TEMPLATE, repoBaseUrl, studyId, analysisId)) + .retrieve() + .onStatus( + HttpStatus.NOT_FOUND::equals, + clientResponse -> + error(notFound(MSG_ANALYSIS_DOES_NOT_EXIST, analysisId, studyId, repoBaseUrl))) + .bodyToMono(Analysis.class) + .transform( + retryAndTimeout(retryConfig, Duration.ofSeconds(this.analysisCallTimeoutSeconds))) + .doOnSuccess( + (analysis) -> log.trace("getAnalysis out, analysis {} args: {}", analysis, command)) + .flatMap( + analysis -> { + if (!indexableStudyStatusesList.contains(analysis.getAnalysisState())) { + return error( + notFound(MSG_ANALYSIS_DOES_NOT_EXIST, analysisId, studyId, repoBaseUrl)); + } + return Mono.just(analysis); }); - } + } - private Function, Mono> retryAndTimeout(Retry retry, Duration timeout) { - // order is important here timeouts should be retried. - return (in) -> in.timeout(timeout).retryWhen(retry); - } + private Function, Mono> retryAndTimeout(Retry retry, Duration timeout) { + // order is important here timeouts should be retried. + return (in) -> in.timeout(timeout).retryWhen(retry); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/FileBasedFailuresLogger.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/FileBasedFailuresLogger.java index 709a83ea..49971ebe 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/FileBasedFailuresLogger.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/FileBasedFailuresLogger.java @@ -17,38 +17,34 @@ package bio.overture.maestro.app.infra.adapter.outbound.notification; +import static bio.overture.maestro.app.infra.config.properties.ApplicationProperties.FAILURE_LOG_PROP_KEY; + import bio.overture.maestro.domain.api.NotificationChannel; import bio.overture.maestro.domain.api.NotificationName; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import reactor.core.publisher.Mono; -import java.util.Set; - -import static bio.overture.maestro.app.infra.config.properties.ApplicationProperties.FAILURE_LOG_PROP_KEY; - /** - * This channel will store any failure in an append only log files - * it uses logback loggers to do the write operation instead of manually - * writing to files. + * This channel will store any failure in an append only log files it uses logback loggers to do the + * write operation instead of manually writing to files. * - * the logs go to separate log files. see logback-spring.xml for the configs. + *

the logs go to separate log files. see logback-spring.xml for the configs. */ @Slf4j @ConditionalOnProperty(value = FAILURE_LOG_PROP_KEY, havingValue = "true") public class FileBasedFailuresLogger implements NotificationChannel { - @Override - public Mono send(IndexerNotification notification) { - log.error("{}", notification); - return Mono.just(true); - } + @Override + public Mono send(IndexerNotification notification) { + log.error("{}", notification); + return Mono.just(true); + } - @Override - public Set subscriptions() { - return Set.of( - NotificationName.ALL - ); - } + @Override + public Set subscriptions() { + return Set.of(NotificationName.ALL); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/LoggingNotificationChannel.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/LoggingNotificationChannel.java index d008ff64..86d53207 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/LoggingNotificationChannel.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/LoggingNotificationChannel.java @@ -20,50 +20,45 @@ import bio.overture.maestro.domain.api.NotificationChannel; import bio.overture.maestro.domain.api.NotificationName; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.springframework.core.annotation.Order; import reactor.core.publisher.Mono; -import java.util.Set; - - /** - * This channel is to log the failures to the standard logger (console). - * it's different than the FileBasedFailuresLogger in the fact that this is - * not persistent or for failures review. + * This channel is to log the failures to the standard logger (console). it's different than the + * FileBasedFailuresLogger in the fact that this is not persistent or for failures review. */ @Slf4j @Order(1) public class LoggingNotificationChannel implements NotificationChannel { - private static final int MAX_NOTIFICATION_STRING_LENGTH = 1024; - - @Override - public Mono send(IndexerNotification notification) { - val notificationString = notification.toString(); - val notificationStringTruncated = notificationString.substring(0, - Math.min(notificationString.length(), MAX_NOTIFICATION_STRING_LENGTH)); + private static final int MAX_NOTIFICATION_STRING_LENGTH = 1024; - switch (notification.getNotificationName().getCategory()) { - case ERROR: - log.error("{}", notificationStringTruncated); - break; - case WARN: - log.warn("{}", notificationStringTruncated); - break; - default: - log.info("{}", notificationStringTruncated); - } + @Override + public Mono send(IndexerNotification notification) { + val notificationString = notification.toString(); + val notificationStringTruncated = + notificationString.substring( + 0, Math.min(notificationString.length(), MAX_NOTIFICATION_STRING_LENGTH)); - return Mono.just(true); + switch (notification.getNotificationName().getCategory()) { + case ERROR: + log.error("{}", notificationStringTruncated); + break; + case WARN: + log.warn("{}", notificationStringTruncated); + break; + default: + log.info("{}", notificationStringTruncated); } - @Override - public Set subscriptions() { - return Set.of( - NotificationName.ALL - ); - } + return Mono.just(true); + } + @Override + public Set subscriptions() { + return Set.of(NotificationName.ALL); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/NotificationConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/NotificationConfig.java index 8ee98e81..8cda3b67 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/NotificationConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/NotificationConfig.java @@ -22,8 +22,8 @@ @Configuration @Import({ - LoggingNotificationChannel.class, - FileBasedFailuresLogger.class, - Slack.class, + LoggingNotificationChannel.class, + FileBasedFailuresLogger.class, + Slack.class, }) public class NotificationConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/Slack.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/Slack.java index f58ef45e..7f69f95c 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/Slack.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/Slack.java @@ -17,10 +17,16 @@ package bio.overture.maestro.app.infra.adapter.outbound.notification; +import static bio.overture.maestro.app.infra.config.properties.ApplicationProperties.MAESTRO_NOTIFICATIONS_SLACK_ENABLED_PROP_KEY; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.api.NotificationChannel; import bio.overture.maestro.domain.api.NotificationName; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; +import java.time.Duration; +import java.util.Map; +import java.util.Set; +import javax.inject.Inject; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -30,79 +36,85 @@ import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; -import javax.inject.Inject; -import java.time.Duration; -import java.util.Map; -import java.util.Set; - -import static bio.overture.maestro.app.infra.config.properties.ApplicationProperties.MAESTRO_NOTIFICATIONS_SLACK_ENABLED_PROP_KEY; - @Slf4j @ConditionalOnProperty(value = MAESTRO_NOTIFICATIONS_SLACK_ENABLED_PROP_KEY, havingValue = "true") public class Slack implements NotificationChannel { - private static final String TYPE = "##TYPE##"; - private static final String DATA = "##DATA##"; - private final WebClient webClient; - private final SlackChannelInfo slackChannelInfo; + private static final String TYPE = "##TYPE##"; + private static final String DATA = "##DATA##"; + private final WebClient webClient; + private final SlackChannelInfo slackChannelInfo; - @Inject - public Slack(ApplicationProperties properties, WebClient webClient) { - this.webClient = webClient; - this.slackChannelInfo = properties.getSlackChannelInfo(); + @Inject + public Slack(ApplicationProperties properties, WebClient webClient) { + this.webClient = webClient; + this.slackChannelInfo = properties.getSlackChannelInfo(); + } + + @Override + public Mono send(@NonNull IndexerNotification notification) { + var template = this.slackChannelInfo.infoTemplate(); + switch (notification.getNotificationName().getCategory()) { + case ERROR: + template = this.slackChannelInfo.errorTemplate(); + break; + case WARN: + template = this.slackChannelInfo.warningTemplate(); + break; } - @Override - public Mono send(@NonNull IndexerNotification notification) { - var template = this.slackChannelInfo.infoTemplate(); - switch (notification.getNotificationName().getCategory()) { - case ERROR: - template = this.slackChannelInfo.errorTemplate(); break; - case WARN: - template = this.slackChannelInfo.warningTemplate(); break; - } - - val dataString = notification.getAttributes().toString(); - val dataStringTruncated = dataString.substring(0, - Math.min(dataString.length(), this.slackChannelInfo.maxDataLength())); - - val text = template. - replace(TYPE, notification.getNotificationName().name()) + val dataString = notification.getAttributes().toString(); + val dataStringTruncated = + dataString.substring( + 0, Math.min(dataString.length(), this.slackChannelInfo.maxDataLength())); + + val text = + template + .replace(TYPE, notification.getNotificationName().name()) .replace(DATA, dataStringTruncated); - val payload = Map.of( + val payload = + Map.of( "text", text, "channel", this.slackChannelInfo.channel(), - "username", this.slackChannelInfo.username() - ); - - return this.webClient.post() - .uri(this.slackChannelInfo.url()) - .contentType(MediaType.APPLICATION_JSON) - .body(BodyInserters.fromObject(payload)) - .retrieve() - .bodyToMono(String.class) - .timeout(Duration.ofSeconds(3L)) - .map((ignored) -> true) - .onErrorResume(e -> { - log.error("failed to send message to slack", e); - return Mono.just(false); + "username", this.slackChannelInfo.username()); + + return this.webClient + .post() + .uri(this.slackChannelInfo.url()) + .contentType(MediaType.APPLICATION_JSON) + .body(BodyInserters.fromObject(payload)) + .retrieve() + .bodyToMono(String.class) + .timeout(Duration.ofSeconds(3L)) + .map((ignored) -> true) + .onErrorResume( + e -> { + log.error("failed to send message to slack", e); + return Mono.just(false); }); - } + } - @Override - public Set subscriptions() { - return Set.copyOf(slackChannelInfo.subscriptions()); - } + @Override + public Set subscriptions() { + return Set.copyOf(slackChannelInfo.subscriptions()); + } - public interface SlackChannelInfo { - String url(); - String channel(); - String username(); - String errorTemplate(); - String warningTemplate(); - String infoTemplate(); - int maxDataLength(); - Set subscriptions(); - } + public interface SlackChannelInfo { + String url(); + + String channel(); + + String username(); + + String errorTemplate(); + + String warningTemplate(); + + String infoTemplate(); + + int maxDataLength(); + + Set subscriptions(); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/SlackInfo.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/SlackInfo.java index 9cc5b66f..beaf83f9 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/SlackInfo.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/SlackInfo.java @@ -16,5 +16,3 @@ */ package bio.overture.maestro.app.infra.adapter.outbound.notification; - - diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/RootConfiguration.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/RootConfiguration.java index 52983476..7b36b823 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/RootConfiguration.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/RootConfiguration.java @@ -35,70 +35,66 @@ import org.springframework.context.annotation.Primary; import org.springframework.web.reactive.function.client.WebClient; -/** - * Aggregates all configuration in one place - */ +/** Aggregates all configuration in one place */ @Configuration @Import({ - DomainApiConfig.class, - PortsConfig.class, - InfraConfig.class, + DomainApiConfig.class, + PortsConfig.class, + InfraConfig.class, }) public class RootConfiguration { - public final static String ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER = "documentObjectMapper"; + public static final String ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER = "documentObjectMapper"; } /** - * Configuration about domain related beans (ports implementations), delegated here to keep the domain module agnostic of injection framework . + * Configuration about domain related beans (ports implementations), delegated here to keep the + * domain module agnostic of injection framework . */ @Configuration @Import({ - ElasticSearchConfig.class, - ExclusionRulesConfig.class, - MessagingConfig.class, - WebConfig.class, - SongConfig.class, - RepositoryConfig.class, - NotificationConfig.class, + ElasticSearchConfig.class, + ExclusionRulesConfig.class, + MessagingConfig.class, + WebConfig.class, + SongConfig.class, + RepositoryConfig.class, + NotificationConfig.class, }) class PortsConfig {} /** - * Aggregator for all configurations related to - * the infrastructure beans (I/O & networking, properties, datasources, etc) + * Aggregator for all configurations related to the infrastructure beans (I/O & networking, + * properties, datasources, etc) */ @Configuration @Import({ - PropertiesConfig.class, + PropertiesConfig.class, }) class InfraConfig { - @Bean - WebClient webClient(ApplicationProperties properties) { - return WebClient.builder().codecs( - c -> c.defaultCodecs().maxInMemorySize(properties.webClientMaxInMemorySize())).build(); - } + @Bean + WebClient webClient(ApplicationProperties properties) { + return WebClient.builder() + .codecs(c -> c.defaultCodecs().maxInMemorySize(properties.webClientMaxInMemorySize())) + .build(); + } } -/** - * Configuration related to the indexer web api - */ +/** Configuration related to the indexer web api */ @Configuration @Import({ - GlobalWebExceptionHandler.class, - ManagementController.class, + GlobalWebExceptionHandler.class, + ManagementController.class, }) class WebConfig { - private static final String DEFAULT_DOCUMENT_JSON_MAPPER = "DEFAULT_DOCUMENT_JSON_MAPPER" ; + private static final String DEFAULT_DOCUMENT_JSON_MAPPER = "DEFAULT_DOCUMENT_JSON_MAPPER"; - /** - * This bean is needed for spring webflux to not use the ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER - * marked as primary so by default callers who don't specify which bean they need, will get this. - */ - @Primary - @Bean(name = DEFAULT_DOCUMENT_JSON_MAPPER) - public ObjectMapper objectMapper() { - return new ObjectMapper(); - } + /** + * This bean is needed for spring webflux to not use the ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER + * marked as primary so by default callers who don't specify which bean they need, will get this. + */ + @Primary + @Bean(name = DEFAULT_DOCUMENT_JSON_MAPPER) + public ObjectMapper objectMapper() { + return new ObjectMapper(); + } } - - diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/ApplicationProperties.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/ApplicationProperties.java index 0ae609ff..0e9e397d 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/ApplicationProperties.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/ApplicationProperties.java @@ -27,58 +27,58 @@ * the original properties */ public interface ApplicationProperties { - String FAILURE_LOG_PROP_KEY = "maestro.failureLog.enabled"; - String MAESTRO_NOTIFICATIONS_SLACK_ENABLED_PROP_KEY = "maestro.notifications.slack.enabled"; + String FAILURE_LOG_PROP_KEY = "maestro.failureLog.enabled"; + String MAESTRO_NOTIFICATIONS_SLACK_ENABLED_PROP_KEY = "maestro.notifications.slack.enabled"; - int webClientMaxInMemorySize(); + int webClientMaxInMemorySize(); - List elasticSearchClusterNodes(); + List elasticSearchClusterNodes(); - String fileCentricAlias(); + String fileCentricAlias(); - String fileCentricIndexName(); + String fileCentricIndexName(); - boolean isFileCentricIndexEnabled(); + boolean isFileCentricIndexEnabled(); - String analysisCentricAlias(); + String analysisCentricAlias(); - String analysisCentricIndexName(); + String analysisCentricIndexName(); - boolean isAnalysisCentricIndexEnabled(); + boolean isAnalysisCentricIndexEnabled(); - int maxDocsPerBulkRequest(); + int maxDocsPerBulkRequest(); - int elasticSearchClientConnectionTimeoutMillis(); + int elasticSearchClientConnectionTimeoutMillis(); - int elasticSearchClientSocketTimeoutMillis(); + int elasticSearchClientSocketTimeoutMillis(); - boolean elasticSearchTlsTrustSelfSigned(); + boolean elasticSearchTlsTrustSelfSigned(); - boolean elasticSearchBasicAuthEnabled(); + boolean elasticSearchBasicAuthEnabled(); - String elasticSearchAuthUser(); + String elasticSearchAuthUser(); - String elasticSearchAuthPassword(); + String elasticSearchAuthPassword(); - List repositories(); + List repositories(); - Resource fileCentricIndex(); + Resource fileCentricIndex(); - Resource analysisCentricIndex(); + Resource analysisCentricIndex(); - Map> idExclusionRules(); + Map> idExclusionRules(); - int songMaxRetries(); + int songMaxRetries(); - int songStudyCallTimeoutSeconds(); + int songStudyCallTimeoutSeconds(); - long elasticSearchRetryWaitDurationMillis(); + long elasticSearchRetryWaitDurationMillis(); - int elasticSearchRetryMaxAttempts(); + int elasticSearchRetryMaxAttempts(); - String indexableStudyStatuses(); + String indexableStudyStatuses(); - int songAnalysisCallTimeoutSeconds(); + int songAnalysisCallTimeoutSeconds(); - Slack.SlackChannelInfo getSlackChannelInfo(); + Slack.SlackChannelInfo getSlackChannelInfo(); } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesConfig.java index 4d34e43e..278f621f 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesConfig.java @@ -22,7 +22,6 @@ @Configuration @Import({ - DefaultApplicationProperties.class, + DefaultApplicationProperties.class, }) -public class PropertiesConfig { -} +public class PropertiesConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesFileRepository.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesFileRepository.java index b444ce1e..927b49ba 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesFileRepository.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesFileRepository.java @@ -18,19 +18,19 @@ package bio.overture.maestro.app.infra.config.properties; public interface PropertiesFileRepository { - String getName(); + String getName(); - String getCode(); + String getCode(); - String getUrl(); + String getUrl(); - String getDataPath(); + String getDataPath(); - String getMetadataPath(); + String getMetadataPath(); - String getOrganization(); + String getOrganization(); - String getCountry(); + String getCountry(); - bio.overture.maestro.domain.entities.indexing.StorageType getStorageType(); + bio.overture.maestro.domain.entities.indexing.StorageType getStorageType(); } diff --git a/maestro-app/src/main/java/bio/overture/maestro/domain/api/DomainApiConfig.java b/maestro-app/src/main/java/bio/overture/maestro/domain/api/DomainApiConfig.java index b5ff6a09..f17501d9 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/domain/api/DomainApiConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/domain/api/DomainApiConfig.java @@ -25,17 +25,16 @@ @Configuration @Import({ - DefaultIndexer.class, - Notifier.class, + DefaultIndexer.class, + Notifier.class, }) public class DomainApiConfig { @Bean IndexEnabled indexEnabled(@Autowired ApplicationProperties applicationProperties) { - return new IndexEnabled - .IndexEnabledBuilder() - .isAnalysisCentricEnabled(applicationProperties.isAnalysisCentricIndexEnabled()) - .isFileCentricEnabled(applicationProperties.isFileCentricIndexEnabled()) - .build(); + return new IndexEnabled.IndexEnabledBuilder() + .isAnalysisCentricEnabled(applicationProperties.isAnalysisCentricIndexEnabled()) + .isFileCentricEnabled(applicationProperties.isFileCentricIndexEnabled()) + .build(); } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/domain/api/package-info.java b/maestro-app/src/main/java/bio/overture/maestro/domain/api/package-info.java index 225ad47b..b26de2c7 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/domain/api/package-info.java +++ b/maestro-app/src/main/java/bio/overture/maestro/domain/api/package-info.java @@ -19,4 +19,4 @@ /* * This package is here to define the package hidden beans from maestro domain module * as spring beans without making the classes public to avoid misusing them. - */ \ No newline at end of file + */ diff --git a/maestro-app/src/test/java/bio/overture/maestro/MaestroIntegrationTest.java b/maestro-app/src/test/java/bio/overture/maestro/MaestroIntegrationTest.java index 046f62ea..9a9449ff 100644 --- a/maestro-app/src/test/java/bio/overture/maestro/MaestroIntegrationTest.java +++ b/maestro-app/src/test/java/bio/overture/maestro/MaestroIntegrationTest.java @@ -34,41 +34,39 @@ import org.springframework.cloud.contract.wiremock.AutoConfigureWireMock; import org.springframework.test.context.ContextConfiguration; - /** - * Base class for full integration tests. - * it will create an elasticsearch container (using test containers). + * Base class for full integration tests. it will create an elasticsearch container (using test + * containers). * - * if you need a more light weight faster tests, avoid using this class, this is meant for full end to end. + *

if you need a more light weight faster tests, avoid using this class, this is meant for full + * end to end. */ @Slf4j @Tag(TestCategory.INT_TEST) @ContextConfiguration(classes = {Maestro.class}) @AutoConfigureWireMock(port = 0) // we bind the port in the application.yml ${wiremock.server.port} -@SpringBootTest(classes = {Maestro.class}, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = { - "embedded.elasticsearch.enabled=true" -}) +@SpringBootTest( + classes = {Maestro.class}, + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = {"embedded.elasticsearch.enabled=true"}) public abstract class MaestroIntegrationTest { - /** wait time for elastic search to update */ - @Value("${maestro.test.elasticsearch.sleep_millis:2500}") - protected int sleepMillis; - - @Autowired - private ApplicationProperties properties; + /** wait time for elastic search to update */ + @Value("${maestro.test.elasticsearch.sleep_millis:2500}") + protected int sleepMillis; - @Autowired - private RestHighLevelClient client; + @Autowired private ApplicationProperties properties; - @AfterEach - @SneakyThrows - void tearDown() { - // clean indexes after each test to keep tests isolated - DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(); - deleteByQueryRequest.indices(properties.fileCentricAlias()); - deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); - client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); - Thread.sleep(sleepMillis); - } + @Autowired private RestHighLevelClient client; -} \ No newline at end of file + @AfterEach + @SneakyThrows + void tearDown() { + // clean indexes after each test to keep tests isolated + DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(); + deleteByQueryRequest.indices(properties.fileCentricAlias()); + deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); + client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); + Thread.sleep(sleepMillis); + } +} diff --git a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListenerTest.java b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListenerTest.java index da241eca..c6c54996 100644 --- a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListenerTest.java +++ b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListenerTest.java @@ -17,6 +17,11 @@ package bio.overture.maestro.app.infra.adapter.inbound.messaging.song; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.*; + import bio.overture.maestro.app.infra.adapter.inbound.messaging.MessagingConfig; import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.api.Indexer; @@ -38,11 +43,6 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.BDDMockito.then; -import static org.mockito.Mockito.*; - /* * This test is based on : https://docs.spring.io/spring-cloud-stream/docs/current/reference/htmlsingle/#_testing * Using a real kafka test container didn't turn out as I expected because spring didn't register diff --git a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapterUnavaibilityTest.java b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapterUnavaibilityTest.java index 29120e2f..8ec2568c 100644 --- a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapterUnavaibilityTest.java +++ b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapterUnavaibilityTest.java @@ -17,6 +17,11 @@ package bio.overture.maestro.app.infra.adapter.outbound.indexing.elasticsearch; +import static bio.overture.maestro.app.infra.config.RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import bio.overture.maestro.app.infra.config.properties.PropertiesConfig; import bio.overture.maestro.domain.api.exception.FailureData; import bio.overture.maestro.domain.api.message.IndexResult; @@ -26,6 +31,10 @@ import bio.overture.masestro.test.TestCategory; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.http.HttpHost; @@ -45,86 +54,71 @@ import org.springframework.web.reactive.function.client.WebClient; import reactor.test.StepVerifier; -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.Set; - -import static bio.overture.maestro.app.infra.config.RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - @Slf4j @Tag(TestCategory.INT_TEST) -@SpringBootTest(properties = { - "embedded.elasticsearch.enabled=false" -}) +@SpringBootTest(properties = {"embedded.elasticsearch.enabled=false"}) @ContextConfiguration(classes = {FileCentricElasticSearchAdapterUnavaibilityTest.Config.class}) class FileCentricElasticSearchAdapterUnavaibilityTest { - @SpyBean - private RestHighLevelClient client; + @SpyBean private RestHighLevelClient client; - @Autowired - private FileCentricElasticSearchAdapter adapter; + @Autowired private FileCentricElasticSearchAdapter adapter; - @Test - void shouldRetryUpsertOnIOException() throws IOException { - // given - val files = Arrays.asList(Fixture.loadJsonFixtureSnakeCase( - this.getClass(), "PEME-CA.files.json", FileCentricDocument[].class)); + @Test + void shouldRetryUpsertOnIOException() throws IOException { + // given + val files = + Arrays.asList( + Fixture.loadJsonFixtureSnakeCase( + this.getClass(), "PEME-CA.files.json", FileCentricDocument[].class)); - val expectedResult = IndexResult.builder() - .indexName("file_centric") - .failureData( - FailureData.builder() + val expectedResult = + IndexResult.builder() + .indexName("file_centric") + .failureData( + FailureData.builder() .failingIds(Map.of("analysisId", Set.of("EGAZ00001254368"))) - .build() - ).successful(false) - .build(); - - // when - val result = adapter.batchUpsertFileRepositories(BatchIndexFilesCommand.builder() - .files(files) - .build()); - - //then - StepVerifier.create(result) - .expectNext(expectedResult) - .verifyComplete(); - - // since this is a final method I had to add the mockito-extensions directory to test resources - // see why.md there for more info. - verify(client, times(3)).bulk(any(BulkRequest.class), any(RequestOptions.class)); + .build()) + .successful(false) + .build(); + + // when + val result = + adapter.batchUpsertFileRepositories(BatchIndexFilesCommand.builder().files(files).build()); + + // then + StepVerifier.create(result).expectNext(expectedResult).verifyComplete(); + + // since this is a final method I had to add the mockito-extensions directory to test resources + // see why.md there for more info. + verify(client, times(3)).bulk(any(BulkRequest.class), any(RequestOptions.class)); + } + + @Import({ + FileCentricElasticSearchAdapter.class, + SnakeCaseJacksonSearchResultMapper.class, + PropertiesConfig.class + }) + @Configuration + static class Config { + + @Bean + WebClient webClient() { + return WebClient.builder().build(); } - @Import({ - FileCentricElasticSearchAdapter.class, - SnakeCaseJacksonSearchResultMapper.class, - PropertiesConfig.class - }) - @Configuration - static class Config { - - @Bean - WebClient webClient() { - return WebClient.builder().build(); - } - - @Bean - RestHighLevelClient mockClient() { - //this will trigger an IO exception - val restClient = RestClient.builder(new HttpHost("nonexisting", 9201, "http")); - return new RestHighLevelClient(restClient); - } - - @Bean(name = ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) - ObjectMapper documentObjectMapper() { - val mapper = new ObjectMapper(); - mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); - return mapper; - } + @Bean + RestHighLevelClient mockClient() { + // this will trigger an IO exception + val restClient = RestClient.builder(new HttpHost("nonexisting", 9201, "http")); + return new RestHighLevelClient(restClient); + } + @Bean(name = ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) + ObjectMapper documentObjectMapper() { + val mapper = new ObjectMapper(); + mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); + return mapper; } -} \ No newline at end of file + } +} diff --git a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAOTest.java b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAOTest.java index c53f2c79..38f854f6 100644 --- a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAOTest.java +++ b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAOTest.java @@ -17,6 +17,13 @@ package bio.overture.maestro.app.infra.adapter.outbound.metadata.study.song; +import static bio.overture.masestro.test.Fixture.loadJsonFixture; +import static bio.overture.masestro.test.Fixture.loadJsonString; +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.entities.metadata.study.Analysis; import bio.overture.maestro.domain.port.outbound.metadata.study.GetAnalysisCommand; @@ -25,6 +32,7 @@ import bio.overture.masestro.test.TestCategory; import com.fasterxml.jackson.core.type.TypeReference; import com.github.tomakehurst.wiremock.stubbing.Scenario; +import java.util.List; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -42,189 +50,164 @@ import reactor.retry.RetryExhaustedException; import reactor.test.StepVerifier; -import java.util.List; -import static bio.overture.masestro.test.Fixture.loadJsonFixture; -import static bio.overture.masestro.test.Fixture.loadJsonString; -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - @Slf4j @Tag(TestCategory.INT_TEST) -@SpringBootTest(properties = { - "embedded.elasticsearch.enabled=false" -}) +@SpringBootTest(properties = {"embedded.elasticsearch.enabled=false"}) @ContextConfiguration(classes = {SongStudyDAOTest.Config.class}) @AutoConfigureWireMock(port = 0) class SongStudyDAOTest { - @Value("${wiremock.server.port}") - private int wiremockPort; + @Value("${wiremock.server.port}") + private int wiremockPort; - @Autowired - private StudyDAO songStudyDAO; + @Autowired private StudyDAO songStudyDAO; - @Test - void fetchingAnalysisShouldReturnMonoErrorIfRetriesExhausted() { + @Test + void fetchingAnalysisShouldReturnMonoErrorIfRetriesExhausted() { - //given - val analysisId = "EGAZ00001254247"; - val command = GetAnalysisCommand.builder() - .filesRepositoryBaseUrl("http://localhost:"+ wiremockPort) + // given + val analysisId = "EGAZ00001254247"; + val command = + GetAnalysisCommand.builder() + .filesRepositoryBaseUrl("http://localhost:" + wiremockPort) .studyId("PEME-CA") .analysisId(analysisId) .build(); - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) - .willReturn(aResponse() + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) + .willReturn( + aResponse() .withStatus(400) .withBody("

Some wierd unexpected text

") - .withHeader("content-type", "text/html") - ) - ); - - //when - val analysesMono = songStudyDAO.getAnalysis(command); - - //then - StepVerifier.create(analysesMono) - .expectErrorSatisfies((e) -> { - assertTrue(e instanceof RetryExhaustedException); - }).verify(); - } - - @Test - @SneakyThrows - void shouldRetryFetchingAnalysisOnFailure() { - val analysis = loadJsonString(this.getClass(), - "PEME-CA.analysis.json"); - val analysisObj = loadJsonFixture(this.getClass(), - "PEME-CA.analysis.json", Analysis.class); - val analysisId = "EGAZ00001254247"; - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) - .inScenario("RANDOM_FAILURE") - .whenScenarioStateIs(Scenario.STARTED) - .willReturn(aResponse() + .withHeader("content-type", "text/html"))); + + // when + val analysesMono = songStudyDAO.getAnalysis(command); + + // then + StepVerifier.create(analysesMono) + .expectErrorSatisfies( + (e) -> { + assertTrue(e instanceof RetryExhaustedException); + }) + .verify(); + } + + @Test + @SneakyThrows + void shouldRetryFetchingAnalysisOnFailure() { + val analysis = loadJsonString(this.getClass(), "PEME-CA.analysis.json"); + val analysisObj = loadJsonFixture(this.getClass(), "PEME-CA.analysis.json", Analysis.class); + val analysisId = "EGAZ00001254247"; + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) + .inScenario("RANDOM_FAILURE") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + aResponse() .withStatus(400) .withBody("

some wierd unexpected text

") - .withHeader("content-type", "text/html") - ) - .willSetStateTo("WORKING") - ); - - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) - .inScenario("RANDOM_FAILURE") - .whenScenarioStateIs("WORKING") - .willReturn(aResponse() + .withHeader("content-type", "text/html")) + .willSetStateTo("WORKING")); + + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) + .inScenario("RANDOM_FAILURE") + .whenScenarioStateIs("WORKING") + .willReturn( + aResponse() .withBody(analysis) .withStatus(200) - .withHeader("content-type", "application/json") - ) - ); - - val analysesMono = songStudyDAO.getAnalysis(GetAnalysisCommand.builder() - .filesRepositoryBaseUrl("http://localhost:"+ wiremockPort) - .studyId("PEME-CA") - .analysisId(analysisId) - .build() - ); - - StepVerifier.create(analysesMono) - .expectNext(analysisObj) - .verifyComplete(); - - } - - @Test - @SneakyThrows - void shouldRetryFetchingStudyAnalysesOnFailure() { - val analyses = loadJsonString(this.getClass(), - "PEME-CA.study.json"); - val analysesList = loadJsonFixture(this.getClass(), - "PEME-CA.study.json", new TypeReference>() {}); - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .inScenario("RANDOM_FAILURE") - .whenScenarioStateIs(Scenario.STARTED) - .willReturn(aResponse() + .withHeader("content-type", "application/json"))); + + val analysesMono = + songStudyDAO.getAnalysis( + GetAnalysisCommand.builder() + .filesRepositoryBaseUrl("http://localhost:" + wiremockPort) + .studyId("PEME-CA") + .analysisId(analysisId) + .build()); + + StepVerifier.create(analysesMono).expectNext(analysisObj).verifyComplete(); + } + + @Test + @SneakyThrows + void shouldRetryFetchingStudyAnalysesOnFailure() { + val analyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); + val analysesList = + loadJsonFixture( + this.getClass(), "PEME-CA.study.json", new TypeReference>() {}); + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .inScenario("RANDOM_FAILURE") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + aResponse() .withStatus(400) .withBody("

some wierd unexpected text

") - .withHeader("content-type", "text/html") - ) - .willSetStateTo("WORKING") - ); - - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .inScenario("RANDOM_FAILURE") - .whenScenarioStateIs("WORKING") - .willReturn(aResponse() + .withHeader("content-type", "text/html")) + .willSetStateTo("WORKING")); + + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .inScenario("RANDOM_FAILURE") + .whenScenarioStateIs("WORKING") + .willReturn( + aResponse() .withBody(analyses) .withStatus(200) - .withHeader("content-type", "application/json") - ) - ); - - val analysesMono = songStudyDAO.getStudyAnalyses(GetStudyAnalysesCommand.builder() - .filesRepositoryBaseUrl("http://localhost:"+ wiremockPort) - .studyId("PEME-CA") - .build() - ); - - StepVerifier.create(analysesMono) - .expectNext(analysesList) - .verifyComplete(); - - } - - @Test - void fetchingStudyAnalysesShouldReturnRetryExhaustedException() { - //given - val command = GetStudyAnalysesCommand.builder() - .filesRepositoryBaseUrl("http://localhost:"+ wiremockPort) + .withHeader("content-type", "application/json"))); + + val analysesMono = + songStudyDAO.getStudyAnalyses( + GetStudyAnalysesCommand.builder() + .filesRepositoryBaseUrl("http://localhost:" + wiremockPort) + .studyId("PEME-CA") + .build()); + + StepVerifier.create(analysesMono).expectNext(analysesList).verifyComplete(); + } + + @Test + void fetchingStudyAnalysesShouldReturnRetryExhaustedException() { + // given + val command = + GetStudyAnalysesCommand.builder() + .filesRepositoryBaseUrl("http://localhost:" + wiremockPort) .studyId("PEME-CA") .build(); - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() .withStatus(400) .withBody("

Some wierd unexpected text

") - .withHeader("content-type", "text/html") - ) - ); + .withHeader("content-type", "text/html"))); - //when - val analysesMono = songStudyDAO.getStudyAnalyses(command); + // when + val analysesMono = songStudyDAO.getStudyAnalyses(command); - //then - StepVerifier.create(analysesMono) - .expectError(RetryExhaustedException.class) - .verify(); - } + // then + StepVerifier.create(analysesMono).expectError(RetryExhaustedException.class).verify(); + } - @Import({ - SongConfig.class - }) - @Configuration - static class Config { - @Bean - WebClient webClient() { - return WebClient.builder().build(); - } - - @Bean - ApplicationProperties properties() { - ApplicationProperties properties = mock(ApplicationProperties.class); - when(properties.songMaxRetries()).thenReturn(3); - when(properties.songStudyCallTimeoutSeconds()).thenReturn(20); - when(properties.indexableStudyStatuses()).thenReturn("PUBLISHED"); - return properties; - } + @Import({SongConfig.class}) + @Configuration + static class Config { + @Bean + WebClient webClient() { + return WebClient.builder().build(); } + @Bean + ApplicationProperties properties() { + ApplicationProperties properties = mock(ApplicationProperties.class); + when(properties.songMaxRetries()).thenReturn(3); + when(properties.songStudyCallTimeoutSeconds()).thenReturn(20); + when(properties.indexableStudyStatuses()).thenReturn("PUBLISHED"); + return properties; + } + } } - diff --git a/maestro-app/src/test/java/bio/overture/maestro/domain/api/IndexerIntegrationTest.java b/maestro-app/src/test/java/bio/overture/maestro/domain/api/IndexerIntegrationTest.java index d67cca36..adee206d 100644 --- a/maestro-app/src/test/java/bio/overture/maestro/domain/api/IndexerIntegrationTest.java +++ b/maestro-app/src/test/java/bio/overture/maestro/domain/api/IndexerIntegrationTest.java @@ -17,6 +17,17 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.DefaultIndexer.*; +import static bio.overture.masestro.test.Fixture.loadJsonFixture; +import static bio.overture.masestro.test.Fixture.loadJsonString; +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static java.text.MessageFormat.format; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.times; + import bio.overture.maestro.MaestroIntegrationTest; import bio.overture.maestro.app.infra.adapter.outbound.notification.Slack; import bio.overture.maestro.app.infra.config.RootConfiguration; @@ -32,6 +43,9 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; import lombok.SneakyThrows; import lombok.val; import org.elasticsearch.action.search.SearchRequest; @@ -51,547 +65,582 @@ import org.springframework.boot.test.mock.mockito.SpyBean; import reactor.test.StepVerifier; -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; - -import static bio.overture.maestro.domain.api.DefaultIndexer.*; -import static bio.overture.masestro.test.Fixture.loadJsonFixture; -import static bio.overture.masestro.test.Fixture.loadJsonString; -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static java.text.MessageFormat.format; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.BDDMockito.then; -import static org.mockito.Mockito.times; - @Tag(TestCategory.INT_TEST) class IndexerIntegrationTest extends MaestroIntegrationTest { - @Autowired - private RestHighLevelClient restHighLevelClient; + @Autowired private RestHighLevelClient restHighLevelClient; - @Autowired - private ApplicationProperties applicationProperties; + @Autowired private ApplicationProperties applicationProperties; - @Autowired - @Qualifier(RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) - private ObjectMapper elasticSearchJsonMapper; + @Autowired + @Qualifier(RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) + private ObjectMapper elasticSearchJsonMapper; - @SpyBean - private Notifier notifier; + @SpyBean private Notifier notifier; - @SpyBean - private Slack slackChannel; + @SpyBean private Slack slackChannel; - private String alias; + private String alias; - @Autowired - private DefaultIndexer indexer; + @Autowired private DefaultIndexer indexer; - @BeforeEach - void setUp() { - alias = applicationProperties.fileCentricAlias(); - } + @BeforeEach + void setUp() { + alias = applicationProperties.fileCentricAlias(); + } - @Test - void shouldHandleErrorsIfStudyDaoThrowException() { + @Test + void shouldHandleErrorsIfStudyDaoThrowException() { - val analysisId = "EGAZ00001254368"; - val repoId = "collab"; - val studyId = "PEME-CA"; + val analysisId = "EGAZ00001254368"; + val repoId = "collab"; + val studyId = "PEME-CA"; - // Given - stubFor( - request("GET", urlEqualTo(format("/{0}/studies/{1}/analysis/{2}", repoId, studyId, analysisId))) - .willReturn(aResponse() + // Given + stubFor( + request( + "GET", + urlEqualTo(format("/{0}/studies/{1}/analysis/{2}", repoId, studyId, analysisId))) + .willReturn( + aResponse() .withStatus(500) .withBody("

Some weird unexpected text

") - .withHeader("content-type", "text/html") - ) - ); - - // checks the notification request - stubFor(request("POST", urlEqualTo("/slack")) - .withRequestBody(equalToJson("{\"username\":\"maestro\"," + - "\"text\":\":bangbang: Error : " + NotificationName.FAILED_TO_FETCH_ANALYSIS.name() - + ", Error Info: ```{analysisId=" + analysisId + ", " + - "repoCode=" + repoId + ", studyId=" + studyId + ", " + - "err=org.springframework.web.reactive.function.UnsupportedMediaTypeException: Content type 'text/html' " + - "not supported for bodyType=bio.overture.maestro.domain.entities.metadata.studyId.Analysis}```\"," + - "\"channel\":\"maestro-test\"}") - ) - .willReturn(aResponse() - .withStatus(200) - .withBody("ok") - ) - ); - - // test - val result = indexer.indexAnalysisToFileCentric(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .analysisId(analysisId) - .studyId(studyId) - .repositoryCode(repoId) - .build() - ).build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder() + .withHeader("content-type", "text/html"))); + + // checks the notification request + stubFor( + request("POST", urlEqualTo("/slack")) + .withRequestBody( + equalToJson( + "{\"username\":\"maestro\"," + + "\"text\":\":bangbang: Error : " + + NotificationName.FAILED_TO_FETCH_ANALYSIS.name() + + ", Error Info: ```{analysisId=" + + analysisId + + ", " + + "repoCode=" + + repoId + + ", studyId=" + + studyId + + ", " + + "err=org.springframework.web.reactive.function.UnsupportedMediaTypeException: Content type 'text/html' " + + "not supported for bodyType=bio.overture.maestro.domain.entities.metadata.studyId.Analysis}```\"," + + "\"channel\":\"maestro-test\"}")) + .willReturn(aResponse().withStatus(200).withBody("ok"))); + + // test + val result = + indexer.indexAnalysisToFileCentric( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .analysisId(analysisId) + .studyId(studyId) + .repositoryCode(repoId) + .build()) + .build()); + + StepVerifier.create(result) + .expectNext( + IndexResult.builder() .failureData( FailureData.builder() - .failingIds(Map.of(ANALYSIS_ID, Set.of(analysisId))).build() - ).successful(false).build() - ).verifyComplete(); - - ArgumentMatcher matcher = (arg) -> - arg.getNotificationName().equals(NotificationName.FAILED_TO_FETCH_ANALYSIS) + .failingIds(Map.of(ANALYSIS_ID, Set.of(analysisId))) + .build()) + .successful(false) + .build()) + .verifyComplete(); + + ArgumentMatcher matcher = + (arg) -> + arg.getNotificationName().equals(NotificationName.FAILED_TO_FETCH_ANALYSIS) && arg.getAttributes().get(ANALYSIS_ID).equals(analysisId) && arg.getAttributes().get(REPO_CODE).equals(repoId) && arg.getAttributes().get(STUDY_ID).equals(studyId) && arg.getAttributes().containsKey(ERR); - then(notifier).should(times(1)).notify(Mockito.argThat(matcher)); - then(slackChannel).should(times(1)).send(Mockito.argThat(matcher)); - - } - - @Test - void shouldIndexAnalysis() throws InterruptedException, IOException { - // Given - val analyses = loadJsonString(this.getClass(), "PEME-CA.analysis.json"); - val expectedDoc0 = loadJsonFixture(this.getClass(), + then(notifier).should(times(1)).notify(Mockito.argThat(matcher)); + then(slackChannel).should(times(1)).send(Mockito.argThat(matcher)); + } + + @Test + void shouldIndexAnalysis() throws InterruptedException, IOException { + // Given + val analyses = loadJsonString(this.getClass(), "PEME-CA.analysis.json"); + val expectedDoc0 = + loadJsonFixture( + this.getClass(), "doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - stubFor(request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis/EGAZ00001254368")) - .willReturn(aResponse() - .withStatus(200) - .withBody(analyses) - .withHeader("Content-Type", "application/json") - ) - ); - - // test - val result = indexer.indexAnalysisToFileCentric(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .analysisId("EGAZ00001254368") - .studyId("PEME-CA") - .repositoryCode("collab") - .build() - ).build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - - assertNotNull(docs); - assertEquals(1L, docs.size()); - assertEquals(expectedDoc0, docs.get(0)); - } - - @Test - void shouldIndexCRAMAnalysis() throws InterruptedException, IOException { - // Given - val analyses = loadJsonString(this.getClass(), "CRAM-TEST.analysis.json"); - val expectedDoc = loadJsonFixture(this.getClass(), + stubFor( + request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis/EGAZ00001254368")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(analyses) + .withHeader("Content-Type", "application/json"))); + + // test + val result = + indexer.indexAnalysisToFileCentric( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .analysisId("EGAZ00001254368") + .studyId("PEME-CA") + .repositoryCode("collab") + .build()) + .build()); + + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + + assertNotNull(docs); + assertEquals(1L, docs.size()); + assertEquals(expectedDoc0, docs.get(0)); + } + + @Test + void shouldIndexCRAMAnalysis() throws InterruptedException, IOException { + // Given + val analyses = loadJsonString(this.getClass(), "CRAM-TEST.analysis.json"); + val expectedDoc = + loadJsonFixture( + this.getClass(), "cram.doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - stubFor(request("GET", urlEqualTo("/collab/studies/CRAM-TEST/analysis/EGAZ00001254369")) - .willReturn(aResponse() - .withStatus(200) - .withBody(analyses) - .withHeader("Content-Type", "application/json") - ) - ); - - // test - val result = indexer.indexAnalysisToFileCentric(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .analysisId("EGAZ00001254369") - .studyId("CRAM-TEST") - .repositoryCode("collab") - .build() - ).build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - - assertNotNull(docs); - assertEquals(1L, docs.size()); - assertEquals(expectedDoc, docs.get(0)); - } - - @Test - void shouldIndexStudyRepositoryWithExclusionsApplied() throws InterruptedException, IOException { - // Given - val studiesArray = loadJsonFixture(this.getClass(), "studies.json", String[].class); - val studies = Arrays.stream(studiesArray) - .map(s-> Study.builder().studyId(s).build()) + stubFor( + request("GET", urlEqualTo("/collab/studies/CRAM-TEST/analysis/EGAZ00001254369")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(analyses) + .withHeader("Content-Type", "application/json"))); + + // test + val result = + indexer.indexAnalysisToFileCentric( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .analysisId("EGAZ00001254369") + .studyId("CRAM-TEST") + .repositoryCode("collab") + .build()) + .build()); + + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + + assertNotNull(docs); + assertEquals(1L, docs.size()); + assertEquals(expectedDoc, docs.get(0)); + } + + @Test + void shouldIndexStudyRepositoryWithExclusionsApplied() throws InterruptedException, IOException { + // Given + val studiesArray = loadJsonFixture(this.getClass(), "studies.json", String[].class); + val studies = + Arrays.stream(studiesArray) + .map(s -> Study.builder().studyId(s).build()) .collect(Collectors.toList()); - val expectedDocs = Arrays.asList( - loadJsonFixture(this.getClass(), + val expectedDocs = + Arrays.asList( + loadJsonFixture( + this.getClass(), "docs.json", - FileCentricDocument[].class, elasticSearchJsonMapper, - Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl()) - ) - ); - - for (Study study: studies) { - val studyId = study.getStudyId(); - val studyAnalyses = getStudyAnalysesAsString(studyId); - stubFor(request("GET", urlEqualTo("/collab/studies/" + studyId + "/analysis?analysisStates=PUBLISHED")) - .willReturn( - aResponse() - .withStatus(200) - .withBody(studyAnalyses) - .withHeader("Content-Type", "application/json") - ) - ); - } - - stubFor(request("GET", urlEqualTo("/collab/studies/all")) + FileCentricDocument[].class, + elasticSearchJsonMapper, + Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl()))); + + for (Study study : studies) { + val studyId = study.getStudyId(); + val studyAnalyses = getStudyAnalysesAsString(studyId); + stubFor( + request( + "GET", + urlEqualTo("/collab/studies/" + studyId + "/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(studyAnalyses) + .withHeader("Content-Type", "application/json"))); + } + + stubFor( + request("GET", urlEqualTo("/collab/studies/all")) .willReturn(ResponseDefinitionBuilder.okForJson(studiesArray))); - // test - val result = indexer.indexRepository(IndexStudyRepositoryCommand.builder() - .repositoryCode("collab") - .build()); + // test + val result = + indexer.indexRepository( + IndexStudyRepositoryCommand.builder().repositoryCode("collab").build()); - StepVerifier.create(result) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); - Thread.sleep(sleepMillis); + Thread.sleep(sleepMillis); - // assertions - val docs = getFileCentricDocuments(); - assertNotNull(docs); - assertEquals(32L, docs.size()); - val sortedExpected = expectedDocs.stream() + // assertions + val docs = getFileCentricDocuments(); + assertNotNull(docs); + assertEquals(32L, docs.size()); + val sortedExpected = + expectedDocs.stream() .sorted(Comparator.comparing(FileCentricDocument::getObjectId)) .collect(Collectors.toList()); - val sortedDocs = docs.stream() + val sortedDocs = + docs.stream() .sorted(Comparator.comparing(FileCentricDocument::getObjectId)) .collect(Collectors.toList()); - assertEquals(sortedExpected, sortedDocs); - - } - - @Test - void shouldIndexStudyWithExclusionsApplied() throws InterruptedException, IOException { - // Given - @SuppressWarnings("all") - val analyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); - val expectedDoc0 = loadJsonFixture(this.getClass(), + assertEquals(sortedExpected, sortedDocs); + } + + @Test + void shouldIndexStudyWithExclusionsApplied() throws InterruptedException, IOException { + // Given + @SuppressWarnings("all") + val analyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); + val expectedDoc0 = + loadJsonFixture( + this.getClass(), "doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val expectedDoc1 = loadJsonFixture(this.getClass(), + val expectedDoc1 = + loadJsonFixture( + this.getClass(), "doc1.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - stubFor(request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withStatus(200) - .withBody(analyses) - .withHeader("Content-Type", "application/json") - ) - ); - - // test - val result = indexer.indexStudy(IndexStudyCommand.builder() - .repositoryCode("collab") - .studyId("PEME-CA") - .build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - - assertNotNull(docs); - assertEquals(2L, docs.size()); - assertEquals(expectedDoc1, docs.get(1)); - assertEquals(expectedDoc0, docs.get(0)); - } - - @Test - void shouldDeleteSingleAnalysis() throws InterruptedException, IOException { - // Given - @SuppressWarnings("all") - val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); - @SuppressWarnings("all") - val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.study.json"); - val expectedDoc0 = loadJsonFixture(this.getClass(), + stubFor( + request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(analyses) + .withHeader("Content-Type", "application/json"))); + + // test + val result = + indexer.indexStudy( + IndexStudyCommand.builder().repositoryCode("collab").studyId("PEME-CA").build()); + + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + + assertNotNull(docs); + assertEquals(2L, docs.size()); + assertEquals(expectedDoc1, docs.get(1)); + assertEquals(expectedDoc0, docs.get(0)); + } + + @Test + void shouldDeleteSingleAnalysis() throws InterruptedException, IOException { + // Given + @SuppressWarnings("all") + val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); + @SuppressWarnings("all") + val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.study.json"); + val expectedDoc0 = + loadJsonFixture( + this.getClass(), "doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val expectedDoc1 = loadJsonFixture(this.getClass(), "doc1.json", + val expectedDoc1 = + loadJsonFixture( + this.getClass(), + "doc1.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - stubFor(request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withBody(collabAnalyses) - .withHeader("Content-Type", "application/json") - .withStatus(200) - ) - ); - stubFor(request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withBody(awsStudyAnalyses) - .withHeader("Content-Type", "application/json") - .withStatus(200) - ) - ); - - populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); - - // test - val result = indexer.removeAnalysis(RemoveAnalysisCommand.builder() - .analysisIdentifier( - AnalysisIdentifier.builder() - .analysisId("EGAZ00001254247") - .studyId("PEME-CA") - .repositoryCode("aws") - .build() - ).build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder().successful(true).build()) - .verifyComplete(); - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - assertNotNull(docs); - assertEquals(1L, docs.size()); - assertEquals(expectedDoc0, docs.get(0)); - } - - @Test - void shouldUpdateExistingFileDocRepository() throws InterruptedException, IOException { - // Given - @SuppressWarnings("all") - val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); - @SuppressWarnings("all") - val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.study.json"); - - val expectedDoc0 = loadJsonFixture(this.getClass(), + stubFor( + request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withBody(collabAnalyses) + .withHeader("Content-Type", "application/json") + .withStatus(200))); + stubFor( + request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withBody(awsStudyAnalyses) + .withHeader("Content-Type", "application/json") + .withStatus(200))); + + populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); + + // test + val result = + indexer.removeAnalysis( + RemoveAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .analysisId("EGAZ00001254247") + .studyId("PEME-CA") + .repositoryCode("aws") + .build()) + .build()); + + StepVerifier.create(result) + .expectNext(IndexResult.builder().successful(true).build()) + .verifyComplete(); + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + assertNotNull(docs); + assertEquals(1L, docs.size()); + assertEquals(expectedDoc0, docs.get(0)); + } + + @Test + void shouldUpdateExistingFileDocRepository() throws InterruptedException, IOException { + // Given + @SuppressWarnings("all") + val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); + @SuppressWarnings("all") + val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.study.json"); + + val expectedDoc0 = + loadJsonFixture( + this.getClass(), "doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val expectedDoc1 = loadJsonFixture(this.getClass(), "doc1.json", + val expectedDoc1 = + loadJsonFixture( + this.getClass(), + "doc1.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val multiRepoDoc = loadJsonFixture(this.getClass(), + val multiRepoDoc = + loadJsonFixture( + this.getClass(), "doc2.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of( "COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl(), - "AWS_REPO_URL", applicationProperties.repositories().get(1).getUrl() - ) - ); - stubFor(request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withStatus(200) - .withBody(collabAnalyses) - .withHeader("Content-Type", "application/json") - ) - ); - - stubFor(request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withStatus(200) - .withBody(awsStudyAnalyses) - .withHeader("Content-Type", "application/json") - ) - ); - - // test - // step 1 - populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); - - // step 2 index the same files from another repository: - val secondResult = indexer.indexStudy(IndexStudyCommand.builder() - .repositoryCode("aws") - .studyId("PEME-CA") - .build()); - - StepVerifier.create(secondResult) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - assertNotNull(docs); - assertEquals(2L, docs.size()); - assertEquals(multiRepoDoc, docs.get(1)); - assertEquals(expectedDoc0, docs.get(0)); - } - - @Test - void shouldDetectAndNotifyConflictingDocuments() throws InterruptedException, IOException { - // Given - @SuppressWarnings("all") - val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); - - // this has a different analysis id than the one in previous files - @SuppressWarnings("all") - val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.conflicting.study.json"); - val awsStudyAnalysesList = loadJsonFixture(this.getClass(), "PEME-CA.aws.conflicting.study.json", new TypeReference>() {}); - val expectedDoc0 = loadJsonFixture(this.getClass(), + "AWS_REPO_URL", applicationProperties.repositories().get(1).getUrl())); + stubFor( + request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(collabAnalyses) + .withHeader("Content-Type", "application/json"))); + + stubFor( + request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(awsStudyAnalyses) + .withHeader("Content-Type", "application/json"))); + + // test + // step 1 + populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); + + // step 2 index the same files from another repository: + val secondResult = + indexer.indexStudy( + IndexStudyCommand.builder().repositoryCode("aws").studyId("PEME-CA").build()); + + StepVerifier.create(secondResult) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + assertNotNull(docs); + assertEquals(2L, docs.size()); + assertEquals(multiRepoDoc, docs.get(1)); + assertEquals(expectedDoc0, docs.get(0)); + } + + @Test + void shouldDetectAndNotifyConflictingDocuments() throws InterruptedException, IOException { + // Given + @SuppressWarnings("all") + val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); + + // this has a different analysis id than the one in previous files + @SuppressWarnings("all") + val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.conflicting.study.json"); + val awsStudyAnalysesList = + loadJsonFixture( + this.getClass(), + "PEME-CA.aws.conflicting.study.json", + new TypeReference>() {}); + val expectedDoc0 = + loadJsonFixture( + this.getClass(), "doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val expectedDoc1 = loadJsonFixture(this.getClass(), "doc1.json", + val expectedDoc1 = + loadJsonFixture( + this.getClass(), + "doc1.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val expectedNotification = new IndexerNotification(NotificationName.INDEX_FILE_CONFLICT, + val expectedNotification = + new IndexerNotification( + NotificationName.INDEX_FILE_CONFLICT, getConflicts(expectedDoc1, awsStudyAnalysesList.get(0).getAnalysisId())); - stubFor(request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withStatus(200) - .withBody(collabAnalyses) - .withHeader("Content-Type", "application/json") - ) - ); - stubFor(request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withStatus(200) - .withBody(awsStudyAnalyses) - .withHeader("Content-Type", "application/json") - ) - ); - - // test - populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); - - // index the same files from another repository: - // test - val secondResult = indexer.indexStudy(IndexStudyCommand.builder() - .repositoryCode("aws") - .studyId("PEME-CA") - .build()); - - StepVerifier.create(secondResult) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - assertNotNull(docs); - then(notifier).should(times(1)).notify(eq(expectedNotification)); - assertEquals(2L, docs.size()); - assertEquals(expectedDoc0, docs.get(0)); - assertEquals(expectedDoc1, docs.get(1)); - } - - @NotNull - private Map getConflicts(FileCentricDocument document, String differentAnalysisId) { - return Map.of("conflicts", List.of(DefaultIndexer.FileConflict.builder() - .indexedFile( - DefaultIndexer.ConflictingFile.builder() - .studyId(document.getStudyId()) - .analysisId(document.getAnalysis().getAnalysisId()) - .objectId(document.getObjectId()) - .repoCode(document - .getRepositories() - .stream().map(Repository::getCode) - .collect(Collectors.toUnmodifiableSet())) - .build() - ).newFile( - DefaultIndexer.ConflictingFile.builder() - .studyId(document.getStudyId()) - .analysisId(differentAnalysisId) - .objectId(document.getObjectId()) - .repoCode(Set.of("aws")) - .build() - ).build())); - } - - @SneakyThrows - private void populateIndexWithCollabStudy(FileCentricDocument expectedDoc0, FileCentricDocument expectedDoc1) throws InterruptedException { - val result = indexer.indexStudy(IndexStudyCommand.builder() - .repositoryCode("COLLAB") - .studyId("PEME-CA") - .build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - - Thread.sleep(sleepMillis); - - // assertions - List docs = getFileCentricDocuments(); - - assertNotNull(docs); - assertEquals(2L, docs.size()); - assertEquals(expectedDoc1, docs.get(1)); - assertEquals(expectedDoc0, docs.get(0)); - } - - @NotNull - private List getFileCentricDocuments() throws IOException { - val searchRequest = new SearchRequest(alias); - val searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.matchAllQuery()); - searchSourceBuilder.from(0); - searchSourceBuilder.size(100); - searchRequest.source(searchSourceBuilder); - val response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); - val docs = new ArrayList(); - - for(SearchHit hit : response.getHits().getHits()) { - String source = hit.getSourceAsString(); - val doc = elasticSearchJsonMapper.readValue(source, FileCentricDocument.class); - docs.add(doc); - } - return docs; - } - - @SneakyThrows - private String getStudyAnalysesAsString(String studyId) { - return loadJsonString(getClass(), studyId +".analysis.json"); + stubFor( + request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(collabAnalyses) + .withHeader("Content-Type", "application/json"))); + stubFor( + request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(awsStudyAnalyses) + .withHeader("Content-Type", "application/json"))); + + // test + populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); + + // index the same files from another repository: + // test + val secondResult = + indexer.indexStudy( + IndexStudyCommand.builder().repositoryCode("aws").studyId("PEME-CA").build()); + + StepVerifier.create(secondResult) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + assertNotNull(docs); + then(notifier).should(times(1)).notify(eq(expectedNotification)); + assertEquals(2L, docs.size()); + assertEquals(expectedDoc0, docs.get(0)); + assertEquals(expectedDoc1, docs.get(1)); + } + + @NotNull + private Map getConflicts( + FileCentricDocument document, String differentAnalysisId) { + return Map.of( + "conflicts", + List.of( + DefaultIndexer.FileConflict.builder() + .indexedFile( + DefaultIndexer.ConflictingFile.builder() + .studyId(document.getStudyId()) + .analysisId(document.getAnalysis().getAnalysisId()) + .objectId(document.getObjectId()) + .repoCode( + document.getRepositories().stream() + .map(Repository::getCode) + .collect(Collectors.toUnmodifiableSet())) + .build()) + .newFile( + DefaultIndexer.ConflictingFile.builder() + .studyId(document.getStudyId()) + .analysisId(differentAnalysisId) + .objectId(document.getObjectId()) + .repoCode(Set.of("aws")) + .build()) + .build())); + } + + @SneakyThrows + private void populateIndexWithCollabStudy( + FileCentricDocument expectedDoc0, FileCentricDocument expectedDoc1) + throws InterruptedException { + val result = + indexer.indexStudy( + IndexStudyCommand.builder().repositoryCode("COLLAB").studyId("PEME-CA").build()); + + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + + Thread.sleep(sleepMillis); + + // assertions + List docs = getFileCentricDocuments(); + + assertNotNull(docs); + assertEquals(2L, docs.size()); + assertEquals(expectedDoc1, docs.get(1)); + assertEquals(expectedDoc0, docs.get(0)); + } + + @NotNull + private List getFileCentricDocuments() throws IOException { + val searchRequest = new SearchRequest(alias); + val searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchSourceBuilder.from(0); + searchSourceBuilder.size(100); + searchRequest.source(searchSourceBuilder); + val response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + val docs = new ArrayList(); + + for (SearchHit hit : response.getHits().getHits()) { + String source = hit.getSourceAsString(); + val doc = elasticSearchJsonMapper.readValue(source, FileCentricDocument.class); + docs.add(doc); } - -} \ No newline at end of file + return docs; + } + + @SneakyThrows + private String getStudyAnalysesAsString(String studyId) { + return loadJsonString(getClass(), studyId + ".analysis.json"); + } +} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverter.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverter.java index 0883fb23..16f2f141 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverter.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverter.java @@ -1,5 +1,7 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.exception.NotFoundException.checkNotFound; + import bio.overture.maestro.domain.entities.indexing.Repository; import bio.overture.maestro.domain.entities.indexing.analysis.AnalysisCentricDocument; import bio.overture.maestro.domain.entities.indexing.analysis.AnalysisCentricDonor; @@ -9,15 +11,13 @@ import bio.overture.maestro.domain.entities.metadata.study.File; import bio.overture.maestro.domain.entities.metadata.study.Sample; import bio.overture.maestro.domain.entities.metadata.study.Specimen; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; import lombok.NonNull; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import lombok.val; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import static bio.overture.maestro.domain.api.exception.NotFoundException.checkNotFound; @Slf4j @UtilityClass @@ -27,22 +27,25 @@ static List fromAnalysis(Analysis analysis, StudyReposi return List.of(convertAnalysis(analysis, repository)); } - static AnalysisCentricDocument convertAnalysis(Analysis analysis, StudyRepository repository){ - val doc = AnalysisCentricDocument.builder() + static AnalysisCentricDocument convertAnalysis(Analysis analysis, StudyRepository repository) { + val doc = + AnalysisCentricDocument.builder() .analysisId(analysis.getAnalysisId()) .analysisState(analysis.getAnalysisState()) .analysisType(analysis.getAnalysisType().getName()) .analysisVersion(analysis.getAnalysisType().getVersion()) .studyId(analysis.getStudyId()) .donors(getDonors(analysis)) - .repositories(List.of(Repository.builder() - .type(repository.getStorageType().name().toUpperCase()) - .organization(repository.getOrganization()) - .name(repository.getName()) - .code(repository.getCode()) - .country(repository.getCountry()) - .url(repository.getUrl()) - .build())) + .repositories( + List.of( + Repository.builder() + .type(repository.getStorageType().name().toUpperCase()) + .organization(repository.getOrganization()) + .name(repository.getName()) + .code(repository.getCode()) + .country(repository.getCountry()) + .url(repository.getUrl()) + .build())) .experiment(analysis.getExperiment()) .files(buildAnalysisCentricFiles(analysis.getFiles())) .build(); @@ -50,70 +53,77 @@ static AnalysisCentricDocument convertAnalysis(Analysis analysis, StudyRepositor return doc; } - public static List getDonors(@NonNull Analysis analysis){ - val groupedByDonorMap = analysis.getSamples() - .stream() + public static List getDonors(@NonNull Analysis analysis) { + val groupedByDonorMap = + analysis.getSamples().stream() .map(sample -> extractDonor(sample)) - .collect(Collectors.groupingBy(AnalysisCentricDonor ::getDonorId, Collectors.toList())); + .collect(Collectors.groupingBy(AnalysisCentricDonor::getDonorId, Collectors.toList())); - return groupedByDonorMap.values() - .stream() - .collect(Collectors.toList()) - .stream() - .map(donorList -> mergeDonorBySpecimen(donorList)) - .collect(Collectors.toList()); + return groupedByDonorMap.values().stream() + .collect(Collectors.toList()) + .stream() + .map(donorList -> mergeDonorBySpecimen(donorList)) + .collect(Collectors.toList()); } /** * Groups specimens belonging to a AnalysisCentricDonor + * * @param list a grouped list with each AnalysisCentricDonor element having exactly one Specimen - * @return a fully assembled AnalysisCentticDonor object with a list of specimens that belongs to the current donor + * @return a fully assembled AnalysisCentticDonor object with a list of specimens that belongs to + * the current donor */ - private static AnalysisCentricDonor mergeDonorBySpecimen(@NonNull List list){ + private static AnalysisCentricDonor mergeDonorBySpecimen( + @NonNull List list) { - checkNotFound(list.size() > 0, - "Failed to merge AnalysisCentricDonor by specimen: donor list is empty."); + checkNotFound( + list.size() > 0, "Failed to merge AnalysisCentricDonor by specimen: donor list is empty."); // Every element in list has the same donor, so just use the first donor val anyDonor = list.get(0); - checkNotFound(anyDonor.getSpecimens() != null && anyDonor.getSpecimens().size() > 0, - "Failed to merge AnalysisCentricDonor by specimen: donor doesn't have specimen,"); + checkNotFound( + anyDonor.getSpecimens() != null && anyDonor.getSpecimens().size() > 0, + "Failed to merge AnalysisCentricDonor by specimen: donor doesn't have specimen,"); // Each donor has only one specimen in the list - val specimenList = list.stream() + val specimenList = + list.stream() .map(analysisCentricDonor -> analysisCentricDonor.getSpecimens().get(0)) .collect(Collectors.toList()); - val specimenMap = specimenList.stream() - .collect( - Collectors.groupingBy(bio.overture.maestro.domain.entities.indexing.Specimen ::getSpecimenId, Collectors.toList())); + val specimenMap = + specimenList.stream() + .collect( + Collectors.groupingBy( + bio.overture.maestro.domain.entities.indexing.Specimen::getSpecimenId, + Collectors.toList())); - val specimens = specimenMap.values() - .stream() - .collect(Collectors.toList()) - .stream() - .map(speList -> groupSpecimensBySample(speList)) - .collect(Collectors.toList()); + val specimens = + specimenMap.values().stream() + .collect(Collectors.toList()) + .stream() + .map(speList -> groupSpecimensBySample(speList)) + .collect(Collectors.toList()); return AnalysisCentricDonor.builder() - .donorId(anyDonor.getDonorId()) - .submitterDonorId(anyDonor.getSubmitterDonorId()) - .gender(anyDonor.getGender()) - .specimens(specimens) - .build(); + .donorId(anyDonor.getDonorId()) + .submitterDonorId(anyDonor.getSubmitterDonorId()) + .gender(anyDonor.getGender()) + .specimens(specimens) + .build(); } - public static bio.overture.maestro.domain.entities.indexing.Specimen groupSpecimensBySample(@NonNull List list){ - checkNotFound(list.size() > 0, - "Failed to merge Specimen by Sample: Specimen list is empty."); + public static bio.overture.maestro.domain.entities.indexing.Specimen groupSpecimensBySample( + @NonNull List list) { + checkNotFound(list.size() > 0, "Failed to merge Specimen by Sample: Specimen list is empty."); val samples = new ArrayList(); list.stream().forEach(specimen -> samples.addAll(specimen.getSamples())); val specimen = list.get(0); // if there is more than one sample in the list, merge samples under one specimen - if(list.size() > 1) { + if (list.size() > 1) { return bio.overture.maestro.domain.entities.indexing.Specimen.builder() .samples(samples) .specimenId(specimen.getSpecimenId()) @@ -126,61 +136,63 @@ public static bio.overture.maestro.domain.entities.indexing.Specimen groupSpecim } /** - * Converts song metadata sample to AnalysisCentricDonor, - * each song Sample has one donor and one specimen. + * Converts song metadata sample to AnalysisCentricDonor, each song Sample has one donor and one + * specimen. + * * @param sample song metadata Sample object * @return converted AnalysisCentricDonor object */ - private static AnalysisCentricDonor extractDonor(@NonNull Sample sample){ + private static AnalysisCentricDonor extractDonor(@NonNull Sample sample) { val donor = sample.getDonor(); val specimen = sample.getSpecimen(); return AnalysisCentricDonor.builder() - .donorId(donor.getDonorId()) - .gender(donor.getGender()) - .submitterDonorId(donor.getSubmitterDonorId()) - .specimens(buildSpecimen(specimen, sample)) - .build(); + .donorId(donor.getDonorId()) + .gender(donor.getGender()) + .submitterDonorId(donor.getSubmitterDonorId()) + .specimens(buildSpecimen(specimen, sample)) + .build(); } /** * Converts metadata Specimen pojo and metadata Sample pojo to an indexing Specimen pojo. + * * @param specimen * @param sample * @return */ - public static List buildSpecimen(@NonNull Specimen specimen, - @NonNull Sample sample){ - return List.of(bio.overture.maestro.domain.entities.indexing.Specimen.builder() + public static List buildSpecimen( + @NonNull Specimen specimen, @NonNull Sample sample) { + return List.of( + bio.overture.maestro.domain.entities.indexing.Specimen.builder() .specimenId(specimen.getSpecimenId()) .submitterSpecimenId(specimen.getSubmitterSpecimenId()) .specimenType(specimen.getSpecimenType()) .specimenTissueSource(specimen.getSpecimenTissueSource()) .tumourNormalDesignation(specimen.getTumourNormalDesignation()) - .samples(List.of( - bio.overture.maestro.domain.entities.indexing.Sample.builder() - .sampleId(sample.getSampleId()) - .matchedNormalSubmitterSampleId(sample.getMatchedNormalSubmitterSampleId()) - .submitterSampleId(sample.getSubmitterSampleId()) - .sampleType(sample.getSampleType()) - .build())) + .samples( + List.of( + bio.overture.maestro.domain.entities.indexing.Sample.builder() + .sampleId(sample.getSampleId()) + .matchedNormalSubmitterSampleId(sample.getMatchedNormalSubmitterSampleId()) + .submitterSampleId(sample.getSubmitterSampleId()) + .sampleType(sample.getSampleType()) + .build())) .build()); } - private static List buildAnalysisCentricFiles(@NonNull List files){ - return files.stream() - .map(file -> fromFile(file)) - .collect(Collectors.toList()); + private static List buildAnalysisCentricFiles(@NonNull List files) { + return files.stream().map(file -> fromFile(file)).collect(Collectors.toList()); } - private static AnalysisCentricFile fromFile(@NonNull File file){ + private static AnalysisCentricFile fromFile(@NonNull File file) { return AnalysisCentricFile.builder() - .objectId(file.getObjectId()) - .fileAccess(file.getFileAccess()) - .dataType(file.getDataType()) - .md5Sum(file.getFileMd5sum()) - .name(file.getFileName()) - .size(file.getFileSize()) - .fileType(file.getFileType()) - .build(); + .objectId(file.getObjectId()) + .fileAccess(file.getFileAccess()) + .dataType(file.getDataType()) + .md5Sum(file.getFileMd5sum()) + .name(file.getFileName()) + .size(file.getFileSize()) + .fileType(file.getFileType()) + .build(); } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DefaultIndexer.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DefaultIndexer.java index 02e7470b..21cacbea 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DefaultIndexer.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DefaultIndexer.java @@ -17,6 +17,11 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.AnalysisCentricDocumentConverter.fromAnalysis; +import static bio.overture.maestro.domain.api.ExclusionRulesEvaluator.shouldExcludeAnalysis; +import static bio.overture.maestro.domain.utility.Exceptions.wrapWithIndexerException; +import static java.text.MessageFormat.format; + import bio.overture.maestro.domain.api.exception.FailureData; import bio.overture.maestro.domain.api.exception.IndexerException; import bio.overture.maestro.domain.api.message.*; @@ -41,748 +46,812 @@ import io.vavr.Tuple2; import io.vavr.control.Either; import io.vavr.control.Try; +import java.util.*; +import java.util.stream.Collectors; +import javax.inject.Inject; import lombok.*; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; -import static bio.overture.maestro.domain.api.AnalysisCentricDocumentConverter.fromAnalysis; -import static bio.overture.maestro.domain.api.ExclusionRulesEvaluator.shouldExcludeAnalysis; -import static bio.overture.maestro.domain.utility.Exceptions.wrapWithIndexerException; -import static java.text.MessageFormat.format; @Slf4j class DefaultIndexer implements Indexer { - static final String STUDY_ID = "studyId"; - static final String REPO_CODE = "repoCode"; - static final String ANALYSIS_ID = "analysisId"; - static final String ERR = "err"; - - private static final String REPO_URL = "repoUrl"; - private static final String FAILURE_DATA = "failureData"; - private static final String CONFLICTS = "conflicts"; - private static final String FILE_CENTRIC_INDEX = "file_centric_1.0"; - private static final String ANALYSIS_CENTRIC_INDEX = "analysis_centric_1.0"; - - private boolean isFileCentricEnabled; - private boolean isAnalysisCentricEnabled; - - private final FileCentricIndexAdapter fileCentricIndexAdapter; - private final AnalysisCentricIndexAdapter analysisCentricIndexAdapter; - private final StudyDAO studyDAO; - private final StudyRepositoryDAO studyRepositoryDao; - private final ExclusionRulesDAO exclusionRulesDAO; - private final Notifier notifier; - - @Inject - DefaultIndexer(FileCentricIndexAdapter fileCentricIndexAdapter, - AnalysisCentricIndexAdapter analysisCentricIndexAdapter, - StudyDAO studyDAO, - StudyRepositoryDAO studyRepositoryDao, - ExclusionRulesDAO exclusionRulesDAO, - Notifier notifier, - IndexEnabledProperties indexEnabled - ) { - this.fileCentricIndexAdapter = fileCentricIndexAdapter; - this.analysisCentricIndexAdapter = analysisCentricIndexAdapter; - this.studyDAO = studyDAO; - this.studyRepositoryDao = studyRepositoryDao; - this.exclusionRulesDAO = exclusionRulesDAO; - this.notifier = notifier; - this.isAnalysisCentricEnabled = indexEnabled.isAnalysisCentricEnabled(); - this.isFileCentricEnabled = indexEnabled.isFileCentricEnabled(); - } - - @Override - public Flux indexAnalysis(@NonNull IndexAnalysisCommand command) { - List > monos = new ArrayList<>(); - if (isFileCentricEnabled) { - monos.add(indexAnalysisToFileCentric(command)); - } - if(isAnalysisCentricEnabled){ - monos.add(indexAnalysisToAnalysisCentric(command)); - } - return Flux.merge(monos); - } - - public Mono indexAnalysisToAnalysisCentric(@NonNull IndexAnalysisCommand indexAnalysisCommand){ - val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); - - return prepareTuple(indexAnalysisCommand) - .flatMap(this :: getAnalysisCentricDocuments) - .flatMap(this :: batchUpsertAnalysesAndCollectFailtures) - .onErrorResume(IndexerException.class, (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) - .onErrorResume((e) -> handleIndexAnalysisError(e, analysisIdentifier, ANALYSIS_CENTRIC_INDEX)); - } - - public Mono indexAnalysisToFileCentric(@NonNull IndexAnalysisCommand indexAnalysisCommand) { - val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); - - return prepareTuple(indexAnalysisCommand) - .flatMap(this :: getFileCentricDocuments) - .flatMap(this :: batchUpsertFilesAndCollectFailures) - // this handles exceptions that were handled already and avoids them getting to the generic handler - // because we know if we get this exception it was already logged and notified so we don't want that again. - .onErrorResume(IndexerException.class, (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) - // this handler handles uncaught exceptions - .onErrorResume((e) -> handleIndexAnalysisError(e, analysisIdentifier, FILE_CENTRIC_INDEX)); - } - - @Override - public Mono removeAnalysis(@NonNull RemoveAnalysisCommand removeAnalysisCommand) { - val analysisIdentifier = removeAnalysisCommand.getAnalysisIdentifier(); - return this.fileCentricIndexAdapter.removeAnalysisFiles(analysisIdentifier.getAnalysisId()) - .thenReturn(IndexResult.builder().successful(true).build()) - .onErrorResume((e) -> handleRemoveAnalysisError(analysisIdentifier)); - } - - @Override - public Flux indexStudy(@NonNull IndexStudyCommand command) { - List > monos = new ArrayList<>(); - - Mono, StudyAndRepository>> mono = prepareStudyAndRepo(command) - .flatMap(studyAndRepository -> getFilteredAnalyses - (studyAndRepository.getStudyRepository().getUrl(), studyAndRepository.getStudy().getStudyId()) - .map(analyses -> - new Tuple2<>(analyses, studyAndRepository) - )); - - if (isFileCentricEnabled) { - monos.add(indexStudyToFileCentric(command, mono)); - } - if(isAnalysisCentricEnabled){ - monos.add(indexStudyToAnalysisCentric(command, mono)); - } - return Flux.merge(monos); - } - - private Mono indexStudyToFileCentric(@NonNull IndexStudyCommand command, - @NonNull Mono, StudyAndRepository>> tuple2) { - log.trace("in indexStudyToFileCentric, args: {} ", command); - return tuple2 - .map(t -> buildFileCentricDocuments(t._2().getStudyRepository(), t._1())) - .flatMap(this :: batchUpsertFilesAndCollectFailures) - .onErrorResume(IndexerException.class, (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) - .onErrorResume((e) -> handleIndexStudyError(e, command.getStudyId(), - command.getRepositoryCode(), FILE_CENTRIC_INDEX)); - } - - private Mono indexStudyToAnalysisCentric(@NonNull IndexStudyCommand command, - @NonNull Mono, StudyAndRepository>> tuple2) { - log.trace("in indexStudyToAnalysisCentric, args: {} ", command); - return tuple2 - .map( t -> buildAnalysisCentricDocuments(t._2().getStudyRepository(), t._1())) - .flatMap(this :: batchUpsertAnalysesAndCollectFailtures) - .onErrorResume(IndexerException.class, (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) - .onErrorResume((e) -> handleIndexStudyError(e, command.getStudyId(), - command.getRepositoryCode(), ANALYSIS_CENTRIC_INDEX)); - } - - @Override - public Mono indexRepository(@NonNull IndexStudyRepositoryCommand command) { - log.trace("in indexRepository, args: {} ", command); - return tryGetStudyRepository(command.getRepositoryCode()) - .flatMapMany(this :: getAllStudies) - .flatMap(studyAndRepository -> - this.indexStudy(IndexStudyCommand.builder() - .studyId(studyAndRepository.getStudy().getStudyId()) - .repositoryCode(studyAndRepository.studyRepository.getCode()) - .build()) - // I had to put this block inside this flatMap to allow these operations to bubble up their exceptions - // to this onErrorResume handler without interrupting the main flux, and terminating it with error signals. - // for example if fetchAnalyses down stream throws error for a studyId the studies flux will - // continue emitting studies + static final String STUDY_ID = "studyId"; + static final String REPO_CODE = "repoCode"; + static final String ANALYSIS_ID = "analysisId"; + static final String ERR = "err"; + + private static final String REPO_URL = "repoUrl"; + private static final String FAILURE_DATA = "failureData"; + private static final String CONFLICTS = "conflicts"; + private static final String FILE_CENTRIC_INDEX = "file_centric_1.0"; + private static final String ANALYSIS_CENTRIC_INDEX = "analysis_centric_1.0"; + + private boolean isFileCentricEnabled; + private boolean isAnalysisCentricEnabled; + + private final FileCentricIndexAdapter fileCentricIndexAdapter; + private final AnalysisCentricIndexAdapter analysisCentricIndexAdapter; + private final StudyDAO studyDAO; + private final StudyRepositoryDAO studyRepositoryDao; + private final ExclusionRulesDAO exclusionRulesDAO; + private final Notifier notifier; + + @Inject + DefaultIndexer( + FileCentricIndexAdapter fileCentricIndexAdapter, + AnalysisCentricIndexAdapter analysisCentricIndexAdapter, + StudyDAO studyDAO, + StudyRepositoryDAO studyRepositoryDao, + ExclusionRulesDAO exclusionRulesDAO, + Notifier notifier, + IndexEnabledProperties indexEnabled) { + this.fileCentricIndexAdapter = fileCentricIndexAdapter; + this.analysisCentricIndexAdapter = analysisCentricIndexAdapter; + this.studyDAO = studyDAO; + this.studyRepositoryDao = studyRepositoryDao; + this.exclusionRulesDAO = exclusionRulesDAO; + this.notifier = notifier; + this.isAnalysisCentricEnabled = indexEnabled.isAnalysisCentricEnabled(); + this.isFileCentricEnabled = indexEnabled.isFileCentricEnabled(); + } + + @Override + public Flux indexAnalysis(@NonNull IndexAnalysisCommand command) { + List> monos = new ArrayList<>(); + if (isFileCentricEnabled) { + monos.add(indexAnalysisToFileCentric(command)); + } + if (isAnalysisCentricEnabled) { + monos.add(indexAnalysisToAnalysisCentric(command)); + } + return Flux.merge(monos); + } + + public Mono indexAnalysisToAnalysisCentric( + @NonNull IndexAnalysisCommand indexAnalysisCommand) { + val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); + + return prepareTuple(indexAnalysisCommand) + .flatMap(this::getAnalysisCentricDocuments) + .flatMap(this::batchUpsertAnalysesAndCollectFailtures) + .onErrorResume( + IndexerException.class, + (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) + .onErrorResume( + (e) -> handleIndexAnalysisError(e, analysisIdentifier, ANALYSIS_CENTRIC_INDEX)); + } + + public Mono indexAnalysisToFileCentric( + @NonNull IndexAnalysisCommand indexAnalysisCommand) { + val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); + + return prepareTuple(indexAnalysisCommand) + .flatMap(this::getFileCentricDocuments) + .flatMap(this::batchUpsertFilesAndCollectFailures) + // this handles exceptions that were handled already and avoids them getting to the generic + // handler + // because we know if we get this exception it was already logged and notified so we don't + // want that again. + .onErrorResume( + IndexerException.class, + (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) + // this handler handles uncaught exceptions + .onErrorResume((e) -> handleIndexAnalysisError(e, analysisIdentifier, FILE_CENTRIC_INDEX)); + } + + @Override + public Mono removeAnalysis(@NonNull RemoveAnalysisCommand removeAnalysisCommand) { + val analysisIdentifier = removeAnalysisCommand.getAnalysisIdentifier(); + return this.fileCentricIndexAdapter + .removeAnalysisFiles(analysisIdentifier.getAnalysisId()) + .thenReturn(IndexResult.builder().successful(true).build()) + .onErrorResume((e) -> handleRemoveAnalysisError(analysisIdentifier)); + } + + @Override + public Flux indexStudy(@NonNull IndexStudyCommand command) { + List> monos = new ArrayList<>(); + + Mono, StudyAndRepository>> mono = + prepareStudyAndRepo(command) + .flatMap( + studyAndRepository -> + getFilteredAnalyses( + studyAndRepository.getStudyRepository().getUrl(), + studyAndRepository.getStudy().getStudyId()) + .map(analyses -> new Tuple2<>(analyses, studyAndRepository))); + + if (isFileCentricEnabled) { + monos.add(indexStudyToFileCentric(command, mono)); + } + if (isAnalysisCentricEnabled) { + monos.add(indexStudyToAnalysisCentric(command, mono)); + } + return Flux.merge(monos); + } + + private Mono indexStudyToFileCentric( + @NonNull IndexStudyCommand command, + @NonNull Mono, StudyAndRepository>> tuple2) { + log.trace("in indexStudyToFileCentric, args: {} ", command); + return tuple2 + .map(t -> buildFileCentricDocuments(t._2().getStudyRepository(), t._1())) + .flatMap(this::batchUpsertFilesAndCollectFailures) + .onErrorResume( + IndexerException.class, + (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) + .onErrorResume( + (e) -> + handleIndexStudyError( + e, command.getStudyId(), command.getRepositoryCode(), FILE_CENTRIC_INDEX)); + } + + private Mono indexStudyToAnalysisCentric( + @NonNull IndexStudyCommand command, + @NonNull Mono, StudyAndRepository>> tuple2) { + log.trace("in indexStudyToAnalysisCentric, args: {} ", command); + return tuple2 + .map(t -> buildAnalysisCentricDocuments(t._2().getStudyRepository(), t._1())) + .flatMap(this::batchUpsertAnalysesAndCollectFailtures) + .onErrorResume( + IndexerException.class, + (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) + .onErrorResume( + (e) -> + handleIndexStudyError( + e, command.getStudyId(), command.getRepositoryCode(), ANALYSIS_CENTRIC_INDEX)); + } + + @Override + public Mono indexRepository(@NonNull IndexStudyRepositoryCommand command) { + log.trace("in indexRepository, args: {} ", command); + return tryGetStudyRepository(command.getRepositoryCode()) + .flatMapMany(this::getAllStudies) + .flatMap( + studyAndRepository -> + this.indexStudy( + IndexStudyCommand.builder() + .studyId(studyAndRepository.getStudy().getStudyId()) + .repositoryCode(studyAndRepository.studyRepository.getCode()) + .build()) + // I had to put this block inside this flatMap to allow these operations to bubble up + // their exceptions + // to this onErrorResume handler without interrupting the main flux, and terminating it + // with error signals. + // for example if fetchAnalyses down stream throws error for a studyId the studies flux + // will + // continue emitting studies ) - .onErrorResume(IndexerException.class, (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) - .onErrorResume((e) -> handleIndexRepositoryError(e, command.getRepositoryCode())) - .reduce(this :: reduceIndexResult); - } - - @Override - public void addRule(AddRuleCommand addRuleCommand) { - throw new IndexerException("not implemented yet"); - } - - @Override - public void deleteRule(DeleteRuleCommand deleteRuleCommand) { - throw new IndexerException("not implemented yet"); - } - - @Override - public List getAllRules() { - throw new IndexerException("not implemented yet"); - } - - /* **************** * - * Private Methods * - * **************** */ - private Mono prepareStudyAndRepo(@NonNull IndexStudyCommand command){ - return tryGetStudyRepository(command.getRepositoryCode()) - .map(filesRepository -> toStudyAndRepositoryTuple(command, filesRepository)); - } - - private Mono prepareTuple(@NonNull IndexAnalysisCommand indexAnalysisCommand){ - val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); - return tryGetStudyRepository(analysisIdentifier.getRepositoryCode()) - .map(studyRepository -> - buildStudyAnalysisRepoTuple(analysisIdentifier, studyRepository)); - } - - private Mono handleRemoveAnalysisError(@NonNull AnalysisIdentifier analysisIdentifier) { - val failureInfo = Map.of(ANALYSIS_ID, Set.of(analysisIdentifier.getAnalysisId())); - this.notifier.notify(new IndexerNotification(NotificationName.FAILED_TO_REMOVE_ANALYSIS, failureInfo)); - return Mono.just(IndexResult.builder() + .onErrorResume( + IndexerException.class, + (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) + .onErrorResume((e) -> handleIndexRepositoryError(e, command.getRepositoryCode())) + .reduce(this::reduceIndexResult); + } + + @Override + public void addRule(AddRuleCommand addRuleCommand) { + throw new IndexerException("not implemented yet"); + } + + @Override + public void deleteRule(DeleteRuleCommand deleteRuleCommand) { + throw new IndexerException("not implemented yet"); + } + + @Override + public List getAllRules() { + throw new IndexerException("not implemented yet"); + } + + /* **************** * + * Private Methods * + * **************** */ + private Mono prepareStudyAndRepo(@NonNull IndexStudyCommand command) { + return tryGetStudyRepository(command.getRepositoryCode()) + .map(filesRepository -> toStudyAndRepositoryTuple(command, filesRepository)); + } + + private Mono prepareTuple( + @NonNull IndexAnalysisCommand indexAnalysisCommand) { + val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); + return tryGetStudyRepository(analysisIdentifier.getRepositoryCode()) + .map(studyRepository -> buildStudyAnalysisRepoTuple(analysisIdentifier, studyRepository)); + } + + private Mono handleRemoveAnalysisError( + @NonNull AnalysisIdentifier analysisIdentifier) { + val failureInfo = Map.of(ANALYSIS_ID, Set.of(analysisIdentifier.getAnalysisId())); + this.notifier.notify( + new IndexerNotification(NotificationName.FAILED_TO_REMOVE_ANALYSIS, failureInfo)); + return Mono.just( + IndexResult.builder() .failureData(FailureData.builder().failingIds(failureInfo).build()) - .build() - ); - } - - private Mono tryGetStudyRepository(@NonNull String repoCode) { - return this.studyRepositoryDao.getFilesRepository(repoCode) - .onErrorMap((e) -> handleGetStudyRepoError(repoCode, e)); - } - - private Throwable handleGetStudyRepoError(@NonNull String repoCode, Throwable e) { - val failure = Map.of(REPO_CODE, Set.of(repoCode)); - this.notifier.notify(new IndexerNotification(NotificationName.FAILED_TO_FETCH_REPOSITORY, failure)); - return wrapWithIndexerException(e, "failed getting repository", FailureData.builder() - .failingIds(failure).build()); - } - - private Flux getAllStudies(StudyRepository studyRepository) { - return this.studyDAO.getStudies(GetAllStudiesCommand.builder() - .filesRepositoryBaseUrl(studyRepository.getUrl()) - .build() - ).onErrorMap((e) -> handleGetStudiesError(studyRepository, e)) + .build()); + } + + private Mono tryGetStudyRepository(@NonNull String repoCode) { + return this.studyRepositoryDao + .getFilesRepository(repoCode) + .onErrorMap((e) -> handleGetStudyRepoError(repoCode, e)); + } + + private Throwable handleGetStudyRepoError(@NonNull String repoCode, Throwable e) { + val failure = Map.of(REPO_CODE, Set.of(repoCode)); + this.notifier.notify( + new IndexerNotification(NotificationName.FAILED_TO_FETCH_REPOSITORY, failure)); + return wrapWithIndexerException( + e, "failed getting repository", FailureData.builder().failingIds(failure).build()); + } + + private Flux getAllStudies(StudyRepository studyRepository) { + return this.studyDAO + .getStudies( + GetAllStudiesCommand.builder().filesRepositoryBaseUrl(studyRepository.getUrl()).build()) + .onErrorMap((e) -> handleGetStudiesError(studyRepository, e)) .map(study -> toStudyAndRepository(studyRepository, study)); - } - - @NotNull - private Throwable handleGetStudiesError(StudyRepository studyRepository, Throwable e) { - val errMsg = getErrorMessageOrType(e); - notifyFailedToFetchStudies(studyRepository.getCode(), errMsg); - return wrapWithIndexerException(e, "fetch studies failed", FailureData.builder() + } + + @NotNull + private Throwable handleGetStudiesError(StudyRepository studyRepository, Throwable e) { + val errMsg = getErrorMessageOrType(e); + notifyFailedToFetchStudies(studyRepository.getCode(), errMsg); + return wrapWithIndexerException( + e, + "fetch studies failed", + FailureData.builder() .failingIds(Map.of(REPO_CODE, Set.of(studyRepository.getCode()))) .build()); - } - - private void notifyFailedToFetchStudies(String code, String message) { - val attrs = Map.of(REPO_CODE, code, ERR, message); - val notification = - new IndexerNotification(NotificationName.FETCH_REPO_STUDIES_FAILED, attrs); - notifier.notify(notification); - } - - private StudyAndRepository toStudyAndRepository(StudyRepository studyRepository, Study studyEither) { - return StudyAndRepository.builder() - .study(studyEither) - .studyRepository(studyRepository) - .build(); - } - - private StudyAndRepository toStudyAndRepositoryTuple(@NonNull IndexStudyCommand indexStudyCommand, - @NonNull StudyRepository filesRepository) { - return StudyAndRepository.builder() - .study(Study.builder().studyId(indexStudyCommand.getStudyId()).build()) - .studyRepository(filesRepository).build(); - } - - private Mono> getFilteredAnalyses(@NonNull String repoBaseUrl, @NonNull String studyId) { - return fetchAnalyses(repoBaseUrl, studyId).flatMap(this :: getExclusionRulesAndFilter); - } - - private Mono> fetchAnalyses(@NonNull String studyRepositoryBaseUrl, @NonNull String studyId) { - val command = GetStudyAnalysesCommand.builder() + } + + private void notifyFailedToFetchStudies(String code, String message) { + val attrs = Map.of(REPO_CODE, code, ERR, message); + val notification = new IndexerNotification(NotificationName.FETCH_REPO_STUDIES_FAILED, attrs); + notifier.notify(notification); + } + + private StudyAndRepository toStudyAndRepository( + StudyRepository studyRepository, Study studyEither) { + return StudyAndRepository.builder().study(studyEither).studyRepository(studyRepository).build(); + } + + private StudyAndRepository toStudyAndRepositoryTuple( + @NonNull IndexStudyCommand indexStudyCommand, @NonNull StudyRepository filesRepository) { + return StudyAndRepository.builder() + .study(Study.builder().studyId(indexStudyCommand.getStudyId()).build()) + .studyRepository(filesRepository) + .build(); + } + + private Mono> getFilteredAnalyses( + @NonNull String repoBaseUrl, @NonNull String studyId) { + return fetchAnalyses(repoBaseUrl, studyId).flatMap(this::getExclusionRulesAndFilter); + } + + private Mono> fetchAnalyses( + @NonNull String studyRepositoryBaseUrl, @NonNull String studyId) { + val command = + GetStudyAnalysesCommand.builder() .filesRepositoryBaseUrl(studyRepositoryBaseUrl) .studyId(studyId) .build(); - return this.studyDAO - .getStudyAnalyses(command) - .onErrorMap(e -> handleFetchAnalysesError(studyRepositoryBaseUrl, studyId, command, e)); - } - - private Throwable handleFetchAnalysesError(String studyRepositoryBaseUrl, String studyId, - GetStudyAnalysesCommand command, Throwable e) { - notifyStudyFetchingError(studyId, studyRepositoryBaseUrl, e.getMessage()); - return wrapWithIndexerException(e, - format("failed fetching studyId analysis, command: {0}, retries exhausted", command), - FailureData.builder() - .failingIds(Map.of(STUDY_ID, Set.of(studyId))) - .build() - ); - } - - private void notifyStudyFetchingError(String studyId, String repoUrl, String excMsg) { - val attrs = Map.of(STUDY_ID, studyId, REPO_URL, repoUrl, ERR, excMsg); - val notification = - new IndexerNotification(NotificationName.STUDY_ANALYSES_FETCH_FAILED, attrs); - notifier.notify(notification); - } - - private StudyAnalysisRepositoryTuple buildStudyAnalysisRepoTuple(@NonNull AnalysisIdentifier indexAnalysisCommand, - StudyRepository filesRepository) { - return StudyAnalysisRepositoryTuple.builder() - .analysisId(indexAnalysisCommand.getAnalysisId()) - .study(Study.builder().studyId(indexAnalysisCommand.getStudyId()).build()) - .studyRepository(filesRepository) - .build(); - } - - private Mono> getAnalysisFromStudyRepository(StudyAnalysisRepositoryTuple tuple) { - return tryFetchAnalysis(tuple).flatMap(this :: getExclusionRulesAndFilter); - } - - private Mono>> - getFileCentricDocuments(StudyAnalysisRepositoryTuple tuple) { - return getAnalysisFromStudyRepository(tuple) - .map((analyses) -> buildFileCentricDocuments(tuple.studyRepository, analyses)); - } - - private Mono>> - getAnalysisCentricDocuments(StudyAnalysisRepositoryTuple tuple){ - return getAnalysisFromStudyRepository(tuple) - .map((analyses -> buildAnalysisCentricDocuments(tuple.studyRepository, analyses))); - } - - private Mono> tryFetchAnalysis(StudyAnalysisRepositoryTuple tuple) { - return this.studyDAO.getAnalysis(GetAnalysisCommand.builder() - .analysisId(tuple.getAnalysisId()) - .filesRepositoryBaseUrl(tuple.getStudyRepository().getUrl()) - .studyId(tuple.getStudy().getStudyId()) - .build() - ).map(List::of) + return this.studyDAO + .getStudyAnalyses(command) + .onErrorMap(e -> handleFetchAnalysesError(studyRepositoryBaseUrl, studyId, command, e)); + } + + private Throwable handleFetchAnalysesError( + String studyRepositoryBaseUrl, String studyId, GetStudyAnalysesCommand command, Throwable e) { + notifyStudyFetchingError(studyId, studyRepositoryBaseUrl, e.getMessage()); + return wrapWithIndexerException( + e, + format("failed fetching studyId analysis, command: {0}, retries exhausted", command), + FailureData.builder().failingIds(Map.of(STUDY_ID, Set.of(studyId))).build()); + } + + private void notifyStudyFetchingError(String studyId, String repoUrl, String excMsg) { + val attrs = Map.of(STUDY_ID, studyId, REPO_URL, repoUrl, ERR, excMsg); + val notification = new IndexerNotification(NotificationName.STUDY_ANALYSES_FETCH_FAILED, attrs); + notifier.notify(notification); + } + + private StudyAnalysisRepositoryTuple buildStudyAnalysisRepoTuple( + @NonNull AnalysisIdentifier indexAnalysisCommand, StudyRepository filesRepository) { + return StudyAnalysisRepositoryTuple.builder() + .analysisId(indexAnalysisCommand.getAnalysisId()) + .study(Study.builder().studyId(indexAnalysisCommand.getStudyId()).build()) + .studyRepository(filesRepository) + .build(); + } + + private Mono> getAnalysisFromStudyRepository(StudyAnalysisRepositoryTuple tuple) { + return tryFetchAnalysis(tuple).flatMap(this::getExclusionRulesAndFilter); + } + + private Mono>> getFileCentricDocuments( + StudyAnalysisRepositoryTuple tuple) { + return getAnalysisFromStudyRepository(tuple) + .map((analyses) -> buildFileCentricDocuments(tuple.studyRepository, analyses)); + } + + private Mono>> getAnalysisCentricDocuments( + StudyAnalysisRepositoryTuple tuple) { + return getAnalysisFromStudyRepository(tuple) + .map((analyses -> buildAnalysisCentricDocuments(tuple.studyRepository, analyses))); + } + + private Mono> tryFetchAnalysis(StudyAnalysisRepositoryTuple tuple) { + return this.studyDAO + .getAnalysis( + GetAnalysisCommand.builder() + .analysisId(tuple.getAnalysisId()) + .filesRepositoryBaseUrl(tuple.getStudyRepository().getUrl()) + .studyId(tuple.getStudy().getStudyId()) + .build()) + .map(List::of) .onErrorMap((e) -> handleFetchAnalysisError(tuple, e)); - } + } - private IndexResult convertIndexerExceptionToIndexResult(IndexerException e) { - return IndexResult.builder() - .failureData(e.getFailureData()) - .successful(false) - .build(); - } + private IndexResult convertIndexerExceptionToIndexResult(IndexerException e) { + return IndexResult.builder().failureData(e.getFailureData()).successful(false).build(); + } - private Mono handleIndexStudyError(Throwable e, String studyId, String repoCode, String indexName) { - val context = Map.of( + private Mono handleIndexStudyError( + Throwable e, String studyId, String repoCode, String indexName) { + val context = + Map.of( STUDY_ID, studyId, REPO_CODE, repoCode, - ERR, getErrorMessageOrType(e) - ); - val failingId = Map.of(STUDY_ID, Set.of(studyId)); - return notifyAndReturnFallback(failingId, context, indexName); - } - - private Mono handleIndexAnalysisError(Throwable e, @NonNull AnalysisIdentifier indexAnalysisCommand, @NonNull String indexName) { - log.error("unhandled exception while indexing analysis", e); - val failureContext = Map.of( + ERR, getErrorMessageOrType(e)); + val failingId = Map.of(STUDY_ID, Set.of(studyId)); + return notifyAndReturnFallback(failingId, context, indexName); + } + + private Mono handleIndexAnalysisError( + Throwable e, @NonNull AnalysisIdentifier indexAnalysisCommand, @NonNull String indexName) { + log.error("unhandled exception while indexing analysis", e); + val failureContext = + Map.of( ANALYSIS_ID, indexAnalysisCommand.getAnalysisId(), STUDY_ID, indexAnalysisCommand.getStudyId(), REPO_CODE, indexAnalysisCommand.getRepositoryCode(), - ERR, getErrorMessageOrType(e) - ); - val failedAnalysisId = Map.of(ANALYSIS_ID, Set.of(indexAnalysisCommand.getAnalysisId())); - return notifyAndReturnFallback(failedAnalysisId, failureContext, indexName); - } - - @NotNull - private Mono notifyAndReturnFallback(Map> failingIds, - Map contextInfo, String indexName) { - this.notifier.notify(new IndexerNotification(NotificationName.UNHANDLED_ERROR, contextInfo)); - return Mono.just(IndexResult.builder() + ERR, getErrorMessageOrType(e)); + val failedAnalysisId = Map.of(ANALYSIS_ID, Set.of(indexAnalysisCommand.getAnalysisId())); + return notifyAndReturnFallback(failedAnalysisId, failureContext, indexName); + } + + @NotNull + private Mono notifyAndReturnFallback( + Map> failingIds, + Map contextInfo, + String indexName) { + this.notifier.notify(new IndexerNotification(NotificationName.UNHANDLED_ERROR, contextInfo)); + return Mono.just( + IndexResult.builder() .indexName(indexName) .failureData(FailureData.builder().failingIds(failingIds).build()) .successful(false) - .build() - ); - } - - private Mono> getExclusionRulesAndFilter(List analyses) { - return this.exclusionRulesDAO.getExclusionRules() - .defaultIfEmpty(Map.of()) - .map(ruleMap -> AnalysisAndExclusions.builder() - .analyses(analyses) - .exclusionRulesMap(ruleMap) - .build() - ) - .map(analysisAndExclusions -> filterExcludedAnalyses(analyses, analysisAndExclusions)) - .onErrorMap((e) -> handleExclusionStepError(analyses, e)); - } - - private Throwable handleExclusionStepError(List analyses, Throwable e) { - val failureInfo = Map.of(ANALYSIS_ID, analyses.stream() - .map(Analysis::getAnalysisId) - .collect(Collectors.toUnmodifiableSet()) - ); - notifier.notify(new IndexerNotification(NotificationName.FAILED_TO_FETCH_ANALYSIS, failureInfo)); - return wrapWithIndexerException(e, - "failed filtering analysis", - FailureData.builder().failingIds(failureInfo).build() - ); - } - - private Tuple2> - buildFileCentricDocuments(StudyRepository repo, List analyses) { - - return analyses.stream() - .map(analysis -> buildFileDocuments(analysis, repo)) - .map(newEither -> newEither.fold( + .build()); + } + + private Mono> getExclusionRulesAndFilter(List analyses) { + return this.exclusionRulesDAO + .getExclusionRules() + .defaultIfEmpty(Map.of()) + .map( + ruleMap -> + AnalysisAndExclusions.builder() + .analyses(analyses) + .exclusionRulesMap(ruleMap) + .build()) + .map(analysisAndExclusions -> filterExcludedAnalyses(analyses, analysisAndExclusions)) + .onErrorMap((e) -> handleExclusionStepError(analyses, e)); + } + + private Throwable handleExclusionStepError(List analyses, Throwable e) { + val failureInfo = + Map.of( + ANALYSIS_ID, + analyses.stream().map(Analysis::getAnalysisId).collect(Collectors.toUnmodifiableSet())); + notifier.notify( + new IndexerNotification(NotificationName.FAILED_TO_FETCH_ANALYSIS, failureInfo)); + return wrapWithIndexerException( + e, "failed filtering analysis", FailureData.builder().failingIds(failureInfo).build()); + } + + private Tuple2> buildFileCentricDocuments( + StudyRepository repo, List analyses) { + + return analyses.stream() + .map(analysis -> buildFileDocuments(analysis, repo)) + .map( + newEither -> + newEither.fold( (left) -> new Tuple2<>(left.getFailureData(), List.of()), - (right) -> new Tuple2<>(FailureData.builder().build(), right) - ) - ).reduce((accumulated, current) -> { - accumulated._1().addFailures(current._1()); - val combined = new ArrayList<>(accumulated._2()); - combined.addAll(current._2()); - return new Tuple2<>(accumulated._1(), Collections.unmodifiableList(combined)); - }).orElseGet(() -> new Tuple2<>(FailureData.builder().build(), List.of())); - } - - private Tuple2> - buildAnalysisCentricDocuments(StudyRepository repo, List analyses){ - return analyses.stream() - .map(analysis -> buildAnalysisDocuments(analysis, repo)) - .map(newEither -> newEither.fold( - (left) -> new Tuple2<>(left.getFailureData(), List.of()), - (right) -> new Tuple2<>(FailureData.builder().build(), right) - ) - ).reduce((accumulated, current) -> { - accumulated._1().addFailures(current._1()); - val combined = new ArrayList<>(accumulated._2()); - combined.addAll(current._2()); - return new Tuple2<>(accumulated._1(), Collections.unmodifiableList(combined)); - }).orElseGet(() -> new Tuple2<>(FailureData.builder().build(), List.of())); - } - - private Mono batchUpsert(List files) { - return getAlreadyIndexed(files) - .map(storedFilesList -> findConflicts(files, storedFilesList)) - .flatMap(conflictsCheckResult -> { - handleConflicts(conflictsCheckResult); - return Mono.just(conflictsCheckResult); + (right) -> new Tuple2<>(FailureData.builder().build(), right))) + .reduce( + (accumulated, current) -> { + accumulated._1().addFailures(current._1()); + val combined = new ArrayList<>(accumulated._2()); + combined.addAll(current._2()); + return new Tuple2<>(accumulated._1(), Collections.unmodifiableList(combined)); }) - .map(conflictsCheckResult -> removeConflictingFromInputFilesList(files, conflictsCheckResult)) - .flatMap(this :: callBatchUpsert) - .doOnNext(this :: notifyIndexRequestFailures) - .onErrorResume( - (ex) -> ex instanceof IndexerException, - (ex) -> Mono.just(IndexResult.builder() - .indexName(FILE_CENTRIC_INDEX) - .successful(false) - .failureData(((IndexerException) ex).getFailureData()) - .build()) - ).doOnSuccess(indexResult -> log.trace("finished batchUpsert, list size {}, hashcode {}", files.size(), - Objects.hashCode(files)) - ); - } - - private Mono batchUpsertAnalysis(List analyses) { - return Mono.fromSupplier(() -> analyses).subscribeOn(Schedulers.elastic()) - .flatMap(this :: callBatchUpsertAnalysis) - .doOnNext(this :: notifyIndexRequestFailures) - .onErrorResume( - (ex) -> ex instanceof IndexerException, - (ex) -> Mono.just(IndexResult.builder() - .successful(false) - .failureData(((IndexerException) ex).getFailureData()) - .build()) - ).doOnSuccess(indexResult -> log.trace("finished batch upsert analysis, list size {}, hashcode {}", analyses.size(), - Objects.hashCode(analyses))); - } - - private List filterExcludedAnalyses(List analyses, AnalysisAndExclusions analysisAndExclusions) { - return analyses.stream() - .filter(analysis -> !shouldExcludeAnalysis(analysis, analysisAndExclusions.getExclusionRulesMap())) - .collect(Collectors.toList()); - } - - private Either> - buildFileDocuments(Analysis analysis, StudyRepository repository) { - - return Try.of(() -> FileCentricDocumentConverter.fromAnalysis(analysis, repository)) - .onFailure((e) -> notifyBuildDocumentFailure(NotificationName.CONVERT_ANALYSIS_TO_FILE_DOCS_FAILED, analysis, repository, e)) - .toEither() - .left() - .map((t) -> wrapBuildDocumentException(analysis, t)) - .toEither(); - } - - private Either> - buildAnalysisDocuments(Analysis analysis, StudyRepository repository){ - - return Try.of(() -> fromAnalysis(analysis, repository)) - .onFailure( (e) -> - notifyBuildDocumentFailure(NotificationName.CONVERT_ANALYSIS_TO_ANALYSIS_DOCS_FAILED, analysis, repository, e)) - .toEither() - .left() - .map( (t) -> wrapBuildDocumentException(analysis, t)) - .toEither(); - } - - // if there is already a record in another song - private Mono> getAlreadyIndexed(List files) { - return fileCentricIndexAdapter.fetchByIds(files.stream() - .map(FileCentricDocument :: getObjectId) - .collect(Collectors.toList()) - ).map((fetchResult) -> { - // we convert this list to a hash map to optimize performance for large lists when we lookup files by Ids - val idToFileMap = new HashMap(); - fetchResult.forEach(item -> idToFileMap.put(item.getObjectId(), item)); - return Collections.unmodifiableMap(idToFileMap); - } - ); - } - - private ConflictsCheckResult findConflicts(List filesToIndex, - Map storedFiles) { - val conflictingPairs = filesToIndex.stream() + .orElseGet(() -> new Tuple2<>(FailureData.builder().build(), List.of())); + } + + private Tuple2> buildAnalysisCentricDocuments( + StudyRepository repo, List analyses) { + return analyses.stream() + .map(analysis -> buildAnalysisDocuments(analysis, repo)) + .map( + newEither -> + newEither.fold( + (left) -> + new Tuple2<>(left.getFailureData(), List.of()), + (right) -> new Tuple2<>(FailureData.builder().build(), right))) + .reduce( + (accumulated, current) -> { + accumulated._1().addFailures(current._1()); + val combined = new ArrayList<>(accumulated._2()); + combined.addAll(current._2()); + return new Tuple2<>(accumulated._1(), Collections.unmodifiableList(combined)); + }) + .orElseGet(() -> new Tuple2<>(FailureData.builder().build(), List.of())); + } + + private Mono batchUpsert(List files) { + return getAlreadyIndexed(files) + .map(storedFilesList -> findConflicts(files, storedFilesList)) + .flatMap( + conflictsCheckResult -> { + handleConflicts(conflictsCheckResult); + return Mono.just(conflictsCheckResult); + }) + .map( + conflictsCheckResult -> + removeConflictingFromInputFilesList(files, conflictsCheckResult)) + .flatMap(this::callBatchUpsert) + .doOnNext(this::notifyIndexRequestFailures) + .onErrorResume( + (ex) -> ex instanceof IndexerException, + (ex) -> + Mono.just( + IndexResult.builder() + .indexName(FILE_CENTRIC_INDEX) + .successful(false) + .failureData(((IndexerException) ex).getFailureData()) + .build())) + .doOnSuccess( + indexResult -> + log.trace( + "finished batchUpsert, list size {}, hashcode {}", + files.size(), + Objects.hashCode(files))); + } + + private Mono batchUpsertAnalysis(List analyses) { + return Mono.fromSupplier(() -> analyses) + .subscribeOn(Schedulers.elastic()) + .flatMap(this::callBatchUpsertAnalysis) + .doOnNext(this::notifyIndexRequestFailures) + .onErrorResume( + (ex) -> ex instanceof IndexerException, + (ex) -> + Mono.just( + IndexResult.builder() + .successful(false) + .failureData(((IndexerException) ex).getFailureData()) + .build())) + .doOnSuccess( + indexResult -> + log.trace( + "finished batch upsert analysis, list size {}, hashcode {}", + analyses.size(), + Objects.hashCode(analyses))); + } + + private List filterExcludedAnalyses( + List analyses, AnalysisAndExclusions analysisAndExclusions) { + return analyses.stream() + .filter( + analysis -> + !shouldExcludeAnalysis(analysis, analysisAndExclusions.getExclusionRulesMap())) + .collect(Collectors.toList()); + } + + private Either> buildFileDocuments( + Analysis analysis, StudyRepository repository) { + + return Try.of(() -> FileCentricDocumentConverter.fromAnalysis(analysis, repository)) + .onFailure( + (e) -> + notifyBuildDocumentFailure( + NotificationName.CONVERT_ANALYSIS_TO_FILE_DOCS_FAILED, analysis, repository, e)) + .toEither() + .left() + .map((t) -> wrapBuildDocumentException(analysis, t)) + .toEither(); + } + + private Either> buildAnalysisDocuments( + Analysis analysis, StudyRepository repository) { + + return Try.of(() -> fromAnalysis(analysis, repository)) + .onFailure( + (e) -> + notifyBuildDocumentFailure( + NotificationName.CONVERT_ANALYSIS_TO_ANALYSIS_DOCS_FAILED, + analysis, + repository, + e)) + .toEither() + .left() + .map((t) -> wrapBuildDocumentException(analysis, t)) + .toEither(); + } + + // if there is already a record in another song + private Mono> getAlreadyIndexed( + List files) { + return fileCentricIndexAdapter + .fetchByIds( + files.stream().map(FileCentricDocument::getObjectId).collect(Collectors.toList())) + .map( + (fetchResult) -> { + // we convert this list to a hash map to optimize performance for large lists when we + // lookup files by Ids + val idToFileMap = new HashMap(); + fetchResult.forEach(item -> idToFileMap.put(item.getObjectId(), item)); + return Collections.unmodifiableMap(idToFileMap); + }); + } + + private ConflictsCheckResult findConflicts( + List filesToIndex, Map storedFiles) { + val conflictingPairs = + filesToIndex.stream() .map(fileToIndex -> findIfAnyStoredFileConflicts(storedFiles, fileToIndex)) .filter(Objects::nonNull) .collect(Collectors.toList()); - return ConflictsCheckResult.builder() - .conflictingFiles(conflictingPairs) - .build(); - } + return ConflictsCheckResult.builder().conflictingFiles(conflictingPairs).build(); + } - private Throwable handleFetchAnalysisError(StudyAnalysisRepositoryTuple tuple, Throwable e) { - log.error("failed to fetch analysis", e); - val notificationInfo = Map.of( + private Throwable handleFetchAnalysisError(StudyAnalysisRepositoryTuple tuple, Throwable e) { + log.error("failed to fetch analysis", e); + val notificationInfo = + Map.of( ANALYSIS_ID, tuple.getAnalysisId(), REPO_CODE, tuple.getStudyRepository().getCode(), STUDY_ID, tuple.getStudy().getStudyId(), - ERR, e.getMessage() - ); - val failureInfo = Map.of( - ANALYSIS_ID, Set.of(tuple.getAnalysisId()) - ); - notifier.notify(new IndexerNotification(NotificationName.FAILED_TO_FETCH_ANALYSIS, notificationInfo)); - return wrapWithIndexerException(e, "failed getting analysis", FailureData.builder() - .failingIds( - failureInfo - ).build() - ); - } - - private void handleConflicts(ConflictsCheckResult conflictingFiles) { - if (conflictingFiles == null || conflictingFiles.getConflictingFiles().isEmpty()) return; - this.notifyConflicts(conflictingFiles); - } - - private List removeConflictingFromInputFilesList(List files, - ConflictsCheckResult conflictsCheckResult) { - return files.stream() - .filter(fileCentricDocument -> !isInConflictsList(conflictsCheckResult, fileCentricDocument)) - .collect(Collectors.toUnmodifiableList()); - } - - private Mono callBatchUpsert(List conflictFreeFilesList) { - return this.fileCentricIndexAdapter.batchUpsertFileRepositories( - BatchIndexFilesCommand.builder().files(conflictFreeFilesList).build() - ); - } - - private Mono callBatchUpsertAnalysis(List analyses){ - return this.analysisCentricIndexAdapter.batchUpsertAnalysisRepositories( BatchIndexAnalysisCommand. - builder().analyses(analyses).build()); - } - - private void notifyIndexRequestFailures(IndexResult indexResult) { - if (!indexResult.isSuccessful()) { - notifier.notify( - new IndexerNotification( - NotificationName.INDEX_REQ_FAILED, - Map.of( - FAILURE_DATA, indexResult.getFailureData() - ) - ) - ); - } - } - - private void notifyBuildDocumentFailure(NotificationName notificationName, - Analysis analysis, StudyRepository repository, Throwable e) { - notifier.notify( - new IndexerNotification( - notificationName, - Map.of( - ANALYSIS_ID, analysis.getAnalysisId(), - STUDY_ID, analysis.getStudyId(), - REPO_CODE, repository.getCode(), - ERR, e.getMessage() - ) - ) - ); - } - - private IndexerException wrapBuildDocumentException(Analysis analysis, Throwable throwable) { - return wrapWithIndexerException(throwable, - format("buildFileDocuments failed for analysis : {0}, studyId: {1}", - analysis.getAnalysisId(), analysis.getStudyId()), - FailureData.builder() - .failingIds(Map.of(ANALYSIS_ID, Set.of(analysis.getAnalysisId()))) - .build()); - } - - private Tuple2 - findIfAnyStoredFileConflicts(Map storedFiles, FileCentricDocument fileToIndex) { - if (storedFiles.containsKey(fileToIndex.getObjectId())) { - val storedFile = storedFiles.get(fileToIndex.getObjectId()); - if (fileToIndex.isValidReplica(storedFile)) { - return null; - } - return new Tuple2<>(fileToIndex, storedFiles.get(fileToIndex.getObjectId())); - } else { - return null; - } - } - - private void notifyConflicts(ConflictsCheckResult conflictsCheckResult) { - val conflictingFileList = conflictsCheckResult.getConflictingFiles() - .stream() + ERR, e.getMessage()); + val failureInfo = Map.of(ANALYSIS_ID, Set.of(tuple.getAnalysisId())); + notifier.notify( + new IndexerNotification(NotificationName.FAILED_TO_FETCH_ANALYSIS, notificationInfo)); + return wrapWithIndexerException( + e, "failed getting analysis", FailureData.builder().failingIds(failureInfo).build()); + } + + private void handleConflicts(ConflictsCheckResult conflictingFiles) { + if (conflictingFiles == null || conflictingFiles.getConflictingFiles().isEmpty()) return; + this.notifyConflicts(conflictingFiles); + } + + private List removeConflictingFromInputFilesList( + List files, ConflictsCheckResult conflictsCheckResult) { + return files.stream() + .filter( + fileCentricDocument -> !isInConflictsList(conflictsCheckResult, fileCentricDocument)) + .collect(Collectors.toUnmodifiableList()); + } + + private Mono callBatchUpsert(List conflictFreeFilesList) { + return this.fileCentricIndexAdapter.batchUpsertFileRepositories( + BatchIndexFilesCommand.builder().files(conflictFreeFilesList).build()); + } + + private Mono callBatchUpsertAnalysis(List analyses) { + return this.analysisCentricIndexAdapter.batchUpsertAnalysisRepositories( + BatchIndexAnalysisCommand.builder().analyses(analyses).build()); + } + + private void notifyIndexRequestFailures(IndexResult indexResult) { + if (!indexResult.isSuccessful()) { + notifier.notify( + new IndexerNotification( + NotificationName.INDEX_REQ_FAILED, + Map.of(FAILURE_DATA, indexResult.getFailureData()))); + } + } + + private void notifyBuildDocumentFailure( + NotificationName notificationName, + Analysis analysis, + StudyRepository repository, + Throwable e) { + notifier.notify( + new IndexerNotification( + notificationName, + Map.of( + ANALYSIS_ID, analysis.getAnalysisId(), + STUDY_ID, analysis.getStudyId(), + REPO_CODE, repository.getCode(), + ERR, e.getMessage()))); + } + + private IndexerException wrapBuildDocumentException(Analysis analysis, Throwable throwable) { + return wrapWithIndexerException( + throwable, + format( + "buildFileDocuments failed for analysis : {0}, studyId: {1}", + analysis.getAnalysisId(), analysis.getStudyId()), + FailureData.builder() + .failingIds(Map.of(ANALYSIS_ID, Set.of(analysis.getAnalysisId()))) + .build()); + } + + private Tuple2 findIfAnyStoredFileConflicts( + Map storedFiles, FileCentricDocument fileToIndex) { + if (storedFiles.containsKey(fileToIndex.getObjectId())) { + val storedFile = storedFiles.get(fileToIndex.getObjectId()); + if (fileToIndex.isValidReplica(storedFile)) { + return null; + } + return new Tuple2<>(fileToIndex, storedFiles.get(fileToIndex.getObjectId())); + } else { + return null; + } + } + + private void notifyConflicts(ConflictsCheckResult conflictsCheckResult) { + val conflictingFileList = + conflictsCheckResult.getConflictingFiles().stream() .map(tuple -> tuple.apply(this::toFileConflict)) .collect(Collectors.toUnmodifiableList()); - val notification = new IndexerNotification(NotificationName.INDEX_FILE_CONFLICT, - Map.of(CONFLICTS, conflictingFileList)); - this.notifier.notify(notification); - } - - private boolean isInConflictsList(ConflictsCheckResult conflictsCheckResult, - FileCentricDocument fileCentricDocument) { - return conflictsCheckResult.getConflictingFiles() - .stream() - .map(Tuple2::_1) - .anyMatch(conflictingFile -> conflictingFile - .getObjectId() - .equals(fileCentricDocument.getObjectId()) - ); - } - - private FileConflict toFileConflict(FileCentricDocument f1, FileCentricDocument f2) { - return FileConflict.builder() - .newFile(ConflictingFile.builder() + val notification = + new IndexerNotification( + NotificationName.INDEX_FILE_CONFLICT, Map.of(CONFLICTS, conflictingFileList)); + this.notifier.notify(notification); + } + + private boolean isInConflictsList( + ConflictsCheckResult conflictsCheckResult, FileCentricDocument fileCentricDocument) { + return conflictsCheckResult.getConflictingFiles().stream() + .map(Tuple2::_1) + .anyMatch( + conflictingFile -> + conflictingFile.getObjectId().equals(fileCentricDocument.getObjectId())); + } + + private FileConflict toFileConflict(FileCentricDocument f1, FileCentricDocument f2) { + return FileConflict.builder() + .newFile( + ConflictingFile.builder() .objectId(f1.getObjectId()) .analysisId(f1.getAnalysis().getAnalysisId()) .studyId(f1.getStudyId()) - .repoCode(f1.getRepositories().stream() - .map(Repository::getCode) - .collect(Collectors.toUnmodifiableSet()) - ).build() - ).indexedFile(ConflictingFile.builder() + .repoCode( + f1.getRepositories().stream() + .map(Repository::getCode) + .collect(Collectors.toUnmodifiableSet())) + .build()) + .indexedFile( + ConflictingFile.builder() .objectId(f2.getObjectId()) .analysisId(f2.getAnalysis().getAnalysisId()) .studyId(f2.getStudyId()) - .repoCode(f2.getRepositories().stream() - .map(Repository::getCode) - .collect(Collectors.toUnmodifiableSet()) - ).build() - ).build(); - } - - private Mono - batchUpsertFilesAndCollectFailures(Tuple2> tuple) { - return this.batchUpsert( tuple._2() ) - .map( upsertResult -> + .repoCode( + f2.getRepositories().stream() + .map(Repository::getCode) + .collect(Collectors.toUnmodifiableSet())) + .build()) + .build(); + } + + private Mono batchUpsertFilesAndCollectFailures( + Tuple2> tuple) { + return this.batchUpsert(tuple._2()) + .map( + upsertResult -> reduceIndexResult( - IndexResult.builder() - .failureData(tuple._1()).build(), - upsertResult)); - } - - private Mono - batchUpsertAnalysesAndCollectFailtures(Tuple2> tuple) { - return this.batchUpsertAnalysis( tuple._2()) - .map( upsertResult -> + IndexResult.builder().failureData(tuple._1()).build(), upsertResult)); + } + + private Mono batchUpsertAnalysesAndCollectFailtures( + Tuple2> tuple) { + return this.batchUpsertAnalysis(tuple._2()) + .map( + upsertResult -> reduceIndexResult( - IndexResult.builder() - .failureData(tuple._1()).build(), - upsertResult)); - } - - private Mono handleIndexRepositoryError(Throwable e, String repositoryCode) { - val failingId = Map.of(REPO_CODE, Set.of(repositoryCode)); - val contextInfo = Map.of(REPO_CODE, repositoryCode, ERR, e.getMessage()); - return this.notifyAndReturnFallback(failingId, contextInfo, FILE_CENTRIC_INDEX + ANALYSIS_CENTRIC_INDEX); - } - - private IndexResult reduceIndexResult(IndexResult accumulatedResult, IndexResult newResult) { - log.trace("In reduceIndexResult, newResult {} ", newResult); - val both = FailureData.builder().build(); - if (!accumulatedResult.isSuccessful()) { - both.addFailures(accumulatedResult.getFailureData()); - } - if (!newResult.isSuccessful()) { - both.addFailures(newResult.getFailureData()); - } - return IndexResult.builder() - .indexName(newResult.getIndexName()) - .failureData(both) - .successful(both.getFailingIds().isEmpty()) - .build(); - } - - private String getErrorMessageOrType(Throwable e) { - return e.getMessage() == null ? e.getClass().getName() : e.getMessage(); - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - static class FileConflict { - private ConflictingFile newFile; - private ConflictingFile indexedFile; - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - static class ConflictingFile { - private String objectId; - private String analysisId; - private String studyId; - private Set repoCode; - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - private static class ConflictsCheckResult { - private List> conflictingFiles; - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - private static class StudyAnalysisRepositoryTuple { - private StudyRepository studyRepository; - private Study study; - private String analysisId; - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - public static class StudyAndRepository { - private StudyRepository studyRepository; - private Study study; - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - private static class AnalysisAndExclusions { - private List analyses; - private Map, List> exclusionRulesMap; - } - + IndexResult.builder().failureData(tuple._1()).build(), upsertResult)); + } + + private Mono handleIndexRepositoryError( + Throwable e, String repositoryCode) { + val failingId = Map.of(REPO_CODE, Set.of(repositoryCode)); + val contextInfo = Map.of(REPO_CODE, repositoryCode, ERR, e.getMessage()); + return this.notifyAndReturnFallback( + failingId, contextInfo, FILE_CENTRIC_INDEX + ANALYSIS_CENTRIC_INDEX); + } + + private IndexResult reduceIndexResult(IndexResult accumulatedResult, IndexResult newResult) { + log.trace("In reduceIndexResult, newResult {} ", newResult); + val both = FailureData.builder().build(); + if (!accumulatedResult.isSuccessful()) { + both.addFailures(accumulatedResult.getFailureData()); + } + if (!newResult.isSuccessful()) { + both.addFailures(newResult.getFailureData()); + } + return IndexResult.builder() + .indexName(newResult.getIndexName()) + .failureData(both) + .successful(both.getFailingIds().isEmpty()) + .build(); + } + + private String getErrorMessageOrType(Throwable e) { + return e.getMessage() == null ? e.getClass().getName() : e.getMessage(); + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + static class FileConflict { + private ConflictingFile newFile; + private ConflictingFile indexedFile; + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + static class ConflictingFile { + private String objectId; + private String analysisId; + private String studyId; + private Set repoCode; + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + private static class ConflictsCheckResult { + private List> conflictingFiles; + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + private static class StudyAnalysisRepositoryTuple { + private StudyRepository studyRepository; + private Study study; + private String analysisId; + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + public static class StudyAndRepository { + private StudyRepository studyRepository; + private Study study; + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + private static class AnalysisAndExclusions { + private List analyses; + private Map, List> exclusionRulesMap; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/ExclusionRulesEvaluator.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/ExclusionRulesEvaluator.java index d3f68c44..75a77ec7 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/ExclusionRulesEvaluator.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/ExclusionRulesEvaluator.java @@ -19,84 +19,75 @@ import bio.overture.maestro.domain.entities.indexing.rules.ExclusionRule; import bio.overture.maestro.domain.entities.metadata.study.*; - import java.util.List; import java.util.Map; /** - * Check analysis entities to see if any is marked for exclusion and hence this whole analysis - * to be excluded. - * - * currently this checks rules in the following order: - * - Study - * - Analysis - * - File - * - Sample - * - Specimen - * - Donor + * Check analysis entities to see if any is marked for exclusion and hence this whole analysis to be + * excluded. * + *

currently this checks rules in the following order: - Study - Analysis - File - Sample - + * Specimen - Donor */ class ExclusionRulesEvaluator { - static boolean shouldExcludeAnalysis(Analysis analysis, Map, List> exclusionRules) { - if (exclusionRules.isEmpty()) return false; - return isExcludedByStudy(analysis, exclusionRules) - || isExcludedByAnalysis(analysis, exclusionRules) - || isExcludedByFile(analysis, exclusionRules) - || isExcludedBySample(analysis, exclusionRules) - || isExcludedBySpecimen(analysis, exclusionRules) - || isExcludedByDonor(analysis, exclusionRules); - } + static boolean shouldExcludeAnalysis( + Analysis analysis, Map, List> exclusionRules) { + if (exclusionRules.isEmpty()) return false; + return isExcludedByStudy(analysis, exclusionRules) + || isExcludedByAnalysis(analysis, exclusionRules) + || isExcludedByFile(analysis, exclusionRules) + || isExcludedBySample(analysis, exclusionRules) + || isExcludedBySpecimen(analysis, exclusionRules) + || isExcludedByDonor(analysis, exclusionRules); + } - private static boolean isExcludedByStudy(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(Study.class) && exclusionRules.get(Study.class) - .stream() - .anyMatch(r -> r.applies( - Study.builder().studyId(analysis.getStudyId()).build() - ) - ); - } + private static boolean isExcludedByStudy( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(Study.class) + && exclusionRules.get(Study.class).stream() + .anyMatch(r -> r.applies(Study.builder().studyId(analysis.getStudyId()).build())); + } - private static boolean isExcludedByAnalysis(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(Analysis.class) && exclusionRules.get(Analysis.class) - .stream() - .anyMatch(r -> r.applies(analysis)); - } + private static boolean isExcludedByAnalysis( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(Analysis.class) + && exclusionRules.get(Analysis.class).stream().anyMatch(r -> r.applies(analysis)); + } - private static boolean isExcludedByFile(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(File.class) && analysis.getFiles().stream() - .anyMatch(file -> exclusionRules.get(File.class) - .stream() - .anyMatch(r -> r.applies(file)) - ); - } + private static boolean isExcludedByFile( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(File.class) + && analysis.getFiles().stream() + .anyMatch( + file -> exclusionRules.get(File.class).stream().anyMatch(r -> r.applies(file))); + } - private static boolean isExcludedBySample(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(Sample.class) && analysis.getSamples().stream() - .anyMatch(sample -> exclusionRules.get(Sample.class) - .stream() - .anyMatch(r -> r.applies(sample)) - ); - } + private static boolean isExcludedBySample( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(Sample.class) + && analysis.getSamples().stream() + .anyMatch( + sample -> + exclusionRules.get(Sample.class).stream().anyMatch(r -> r.applies(sample))); + } - private static boolean isExcludedBySpecimen(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(Specimen.class) && analysis.getSamples() - .stream() + private static boolean isExcludedBySpecimen( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(Specimen.class) + && analysis.getSamples().stream() .map(Sample::getSpecimen) - .anyMatch(specimen -> exclusionRules.get(Specimen.class) - .stream() - .anyMatch(r -> r.applies(specimen)) - ); - } + .anyMatch( + specimen -> + exclusionRules.get(Specimen.class).stream().anyMatch(r -> r.applies(specimen))); + } - private static boolean isExcludedByDonor(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(Donor.class) && analysis.getSamples() - .stream() + private static boolean isExcludedByDonor( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(Donor.class) + && analysis.getSamples().stream() .map(Sample::getDonor) - .anyMatch(donor -> exclusionRules.get(Donor.class) - .stream() - .anyMatch(r -> r.applies(donor)) - ); - } - + .anyMatch( + donor -> exclusionRules.get(Donor.class).stream().anyMatch(r -> r.applies(donor))); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/FileCentricDocumentConverter.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/FileCentricDocumentConverter.java index 8915eca0..81e254c7 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/FileCentricDocumentConverter.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/FileCentricDocumentConverter.java @@ -17,23 +17,22 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.AnalysisCentricDocumentConverter.buildSpecimen; +import static bio.overture.maestro.domain.api.AnalysisCentricDocumentConverter.groupSpecimensBySample; +import static bio.overture.maestro.domain.api.exception.NotFoundException.checkNotFound; + import bio.overture.maestro.domain.entities.indexing.*; import bio.overture.maestro.domain.entities.metadata.repository.StudyRepository; import bio.overture.maestro.domain.entities.metadata.study.Analysis; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static bio.overture.maestro.domain.api.AnalysisCentricDocumentConverter.buildSpecimen; -import static bio.overture.maestro.domain.api.AnalysisCentricDocumentConverter.groupSpecimensBySample; -import static bio.overture.maestro.domain.api.exception.NotFoundException.checkNotFound; - /** * This class holds the structural changes that the indexer applies to prepare the File documents @@ -179,68 +178,70 @@ private static Optional getDonors(@NonNull Analysis analysis) { - val groupedByDonormap = analysis.getSamples() - .stream() - .map(sample -> extractDonor(sample)) - .collect(Collectors.groupingBy(FileCentricDonor ::getDonorId, Collectors.toList())); - - return groupedByDonormap.values() - .stream() - .collect(Collectors.toList()) - .stream() - .map(donorList -> mergeDonorBySpecimen(donorList)) - .collect(Collectors.toList()); + val groupedByDonormap = + analysis.getSamples().stream() + .map(sample -> extractDonor(sample)) + .collect(Collectors.groupingBy(FileCentricDonor::getDonorId, Collectors.toList())); + + return groupedByDonormap.values().stream() + .collect(Collectors.toList()) + .stream() + .map(donorList -> mergeDonorBySpecimen(donorList)) + .collect(Collectors.toList()); } - /** - * Converts song metadata sample to FileCentricDonor, - * each song Sample has one donor and one specimen. - * @param sample song metadata Sample object - * @return converted FileCentricDonor object - */ - private static FileCentricDonor extractDonor(@NonNull bio.overture.maestro.domain.entities.metadata.study.Sample sample){ - val donor = sample.getDonor(); - val specimen = sample.getSpecimen(); - return FileCentricDonor.builder() - .donorId(donor.getDonorId()) - .submitterDonorId(donor.getSubmitterDonorId()) - .gender(donor.getGender()) - .specimens(buildSpecimen(specimen, sample)) - .build(); + /** + * Converts song metadata sample to FileCentricDonor, each song Sample has one donor and one + * specimen. + * + * @param sample song metadata Sample object + * @return converted FileCentricDonor object + */ + private static FileCentricDonor extractDonor( + @NonNull bio.overture.maestro.domain.entities.metadata.study.Sample sample) { + val donor = sample.getDonor(); + val specimen = sample.getSpecimen(); + return FileCentricDonor.builder() + .donorId(donor.getDonorId()) + .submitterDonorId(donor.getSubmitterDonorId()) + .gender(donor.getGender()) + .specimens(buildSpecimen(specimen, sample)) + .build(); } - private static FileCentricDonor mergeDonorBySpecimen(@NonNull List list){ - checkNotFound(list.size() > 0, - "Failed to merge FileCentricDonor by specimen: donor list is empty."); - - // Every element in list has the same donor, so just use the first donor - val anyDonor = list.get(0); - - checkNotFound(anyDonor.getSpecimens() != null && anyDonor.getSpecimens().size() > 0, - "Failed to merge FileCentricDonor by specimen: donor doesn't have specimen,"); - - val specimenList = list.stream() - .map(fileCentricDonor -> fileCentricDonor.getSpecimens().get(0)) - .collect(Collectors.toList()); - - val specimenMap = specimenList.stream() - .collect(Collectors.groupingBy( - Specimen ::getSpecimenId, Collectors.toList() - )); - - val specimens = specimenMap.values() - .stream() - .collect(Collectors.toList()) - .stream() - .map(speList -> groupSpecimensBySample(speList)) - .collect(Collectors.toList()); - - return FileCentricDonor.builder() - .donorId(anyDonor.getDonorId()) - .submitterDonorId(anyDonor.getSubmitterDonorId()) - .gender(anyDonor.getGender()) - .specimens(specimens) - .build(); + private static FileCentricDonor mergeDonorBySpecimen(@NonNull List list) { + checkNotFound( + list.size() > 0, "Failed to merge FileCentricDonor by specimen: donor list is empty."); + + // Every element in list has the same donor, so just use the first donor + val anyDonor = list.get(0); + + checkNotFound( + anyDonor.getSpecimens() != null && anyDonor.getSpecimens().size() > 0, + "Failed to merge FileCentricDonor by specimen: donor doesn't have specimen,"); + + val specimenList = + list.stream() + .map(fileCentricDonor -> fileCentricDonor.getSpecimens().get(0)) + .collect(Collectors.toList()); + + val specimenMap = + specimenList.stream() + .collect(Collectors.groupingBy(Specimen::getSpecimenId, Collectors.toList())); + + val specimens = + specimenMap.values().stream() + .collect(Collectors.toList()) + .stream() + .map(speList -> groupSpecimensBySample(speList)) + .collect(Collectors.toList()); + + return FileCentricDonor.builder() + .donorId(anyDonor.getDonorId()) + .submitterDonorId(anyDonor.getSubmitterDonorId()) + .gender(anyDonor.getGender()) + .specimens(specimens) + .build(); } private static boolean isDataFile(bio.overture.maestro.domain.entities.metadata.study.File f) { @@ -269,7 +270,10 @@ private static boolean isIDXFile(String filename) { } private static boolean isIndexFile(String filename) { - return isBAIFile(filename) || isCRAIFile(filename) || isIDXFile(filename) || isTBIFile(filename); + return isBAIFile(filename) + || isCRAIFile(filename) + || isIDXFile(filename) + || isTBIFile(filename); } private static String indexFileFormat(String fileName) { diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/IndexEnabledProperties.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/IndexEnabledProperties.java index 68792c3e..c48aa8ab 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/IndexEnabledProperties.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/IndexEnabledProperties.java @@ -3,6 +3,9 @@ import lombok.NonNull; public interface IndexEnabledProperties { - @NonNull boolean isFileCentricEnabled(); - @NonNull boolean isAnalysisCentricEnabled(); + @NonNull + boolean isFileCentricEnabled(); + + @NonNull + boolean isAnalysisCentricEnabled(); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Indexer.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Indexer.java index be3ee8e7..93ec56de 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Indexer.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Indexer.java @@ -19,47 +19,49 @@ import bio.overture.maestro.domain.api.message.*; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionRule; -import bio.overture.maestro.domain.port.outbound.indexing.FileCentricIndexAdapter; +import java.util.List; import lombok.NonNull; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.util.List; - -/** - * Main entry point for the Indexer API - */ +/** Main entry point for the Indexer API */ public interface Indexer { - /** - * A generic method to index a single analysis to all indices - * @param indexAnalysisCommand - * @return failure info and success flag of all indices - */ - Flux indexAnalysis(@NonNull IndexAnalysisCommand indexAnalysisCommand); + /** + * A generic method to index a single analysis to all indices + * + * @param indexAnalysisCommand + * @return failure info and success flag of all indices + */ + Flux indexAnalysis(@NonNull IndexAnalysisCommand indexAnalysisCommand); + + /** + * Used to remove all files documents for an analysis. + * + * @param removeAnalysisCommand specify repo studyId and analysis id + * @return flag indicating success and failure info if any + */ + Mono removeAnalysis(@NonNull RemoveAnalysisCommand removeAnalysisCommand); + + /** + * A generic method to index a study. + * + * @param command + * @return failure info and success flag of all indices + */ + Flux indexStudy(@NonNull IndexStudyCommand command); - /** - * Used to remove all files documents for an analysis. - * @param removeAnalysisCommand specify repo studyId and analysis id - * @return flag indicating success and failure info if any - */ - Mono removeAnalysis(@NonNull RemoveAnalysisCommand removeAnalysisCommand); + /** + * A generic method to index the entire repository to all indices. + * + * @param command contains repository code + * @return result indicating success/fail and failure information + */ + Mono indexRepository(@NonNull IndexStudyRepositoryCommand command); - /** - * A generic method to index a study. - * @param command - * @return failure info and success flag of all indices - */ - Flux indexStudy(@NonNull IndexStudyCommand command); + void addRule(AddRuleCommand addRuleCommand); - /** - * A generic method to index the entire repository to all indices. - * @param command contains repository code - * @return result indicating success/fail and failure information - */ - Mono indexRepository(@NonNull IndexStudyRepositoryCommand command); + void deleteRule(DeleteRuleCommand deleteRuleCommand); - void addRule(AddRuleCommand addRuleCommand); - void deleteRule(DeleteRuleCommand deleteRuleCommand); - List getAllRules(); + List getAllRules(); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationCategory.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationCategory.java index be6da1b8..2caac77b 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationCategory.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationCategory.java @@ -17,8 +17,8 @@ package bio.overture.maestro.domain.api; -public enum NotificationCategory { - ERROR, - INFO, - WARN, +public enum NotificationCategory { + ERROR, + INFO, + WARN, } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationChannel.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationChannel.java index 023d1316..5650b34f 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationChannel.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationChannel.java @@ -18,16 +18,16 @@ package bio.overture.maestro.domain.api; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; +import java.util.Set; import lombok.NonNull; import reactor.core.publisher.Mono; -import java.util.Set; - /** - * A channel is an abstraction of the technology infrastructure - * that this notification will be delivered through, can be email, web api call, filesystem or anything. + * A channel is an abstraction of the technology infrastructure that this notification will be + * delivered through, can be email, web api call, filesystem or anything. */ public interface NotificationChannel { - Mono send(@NonNull IndexerNotification notification); - Set subscriptions(); + Mono send(@NonNull IndexerNotification notification); + + Set subscriptions(); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationName.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationName.java index f86db6f9..61bf1882 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationName.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationName.java @@ -21,22 +21,21 @@ @Getter public enum NotificationName { - STUDY_ANALYSES_FETCH_FAILED(NotificationCategory.ERROR), - FETCH_REPO_STUDIES_FAILED(NotificationCategory.ERROR), - INDEX_REQ_FAILED(NotificationCategory.ERROR), - CONVERT_ANALYSIS_TO_FILE_DOCS_FAILED(NotificationCategory.ERROR), - CONVERT_ANALYSIS_TO_ANALYSIS_DOCS_FAILED(NotificationCategory.ERROR), - INDEX_FILE_CONFLICT(NotificationCategory.WARN), - ALL(null), - UNHANDLED_ERROR(NotificationCategory.ERROR), - FAILED_TO_FETCH_ANALYSIS(NotificationCategory.ERROR), - FAILED_TO_FETCH_REPOSITORY(NotificationCategory.ERROR), - FAILED_TO_REMOVE_ANALYSIS(NotificationCategory.ERROR); + STUDY_ANALYSES_FETCH_FAILED(NotificationCategory.ERROR), + FETCH_REPO_STUDIES_FAILED(NotificationCategory.ERROR), + INDEX_REQ_FAILED(NotificationCategory.ERROR), + CONVERT_ANALYSIS_TO_FILE_DOCS_FAILED(NotificationCategory.ERROR), + CONVERT_ANALYSIS_TO_ANALYSIS_DOCS_FAILED(NotificationCategory.ERROR), + INDEX_FILE_CONFLICT(NotificationCategory.WARN), + ALL(null), + UNHANDLED_ERROR(NotificationCategory.ERROR), + FAILED_TO_FETCH_ANALYSIS(NotificationCategory.ERROR), + FAILED_TO_FETCH_REPOSITORY(NotificationCategory.ERROR), + FAILED_TO_REMOVE_ANALYSIS(NotificationCategory.ERROR); - private final NotificationCategory category; - - NotificationName(NotificationCategory category) { - this.category = category; - } + private final NotificationCategory category; + NotificationName(NotificationCategory category) { + this.category = category; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Notifier.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Notifier.java index 2e90726b..b73ed132 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Notifier.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Notifier.java @@ -18,54 +18,65 @@ package bio.overture.maestro.domain.api; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; -import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import javax.inject.Inject; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import javax.inject.Inject; +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; @Slf4j class Notifier { - private final Set notificationChannels; + private final Set notificationChannels; - @Inject - public Notifier(Set notificationChannels) { - this.notificationChannels = notificationChannels; - } + @Inject + public Notifier(Set notificationChannels) { + this.notificationChannels = notificationChannels; + } - /** - * Asynchronously calls the eligible notification channels. - * @param notification the notification to send - */ - public void notify(IndexerNotification notification) { - // some of the channels may need async I/O execution (slack) but - // the caller (indexer) doesn't need to worry about what happens here so no need to return the flux and - // we subscribe here. - Flux.fromIterable(getEligibleChannels(notification)) - .flatMap(notificationChannel -> sendNotificationThroughChannel(notification, notificationChannel)) - .subscribe(); - } + /** + * Asynchronously calls the eligible notification channels. + * + * @param notification the notification to send + */ + public void notify(IndexerNotification notification) { + // some of the channels may need async I/O execution (slack) but + // the caller (indexer) doesn't need to worry about what happens here so no need to return the + // flux and + // we subscribe here. + Flux.fromIterable(getEligibleChannels(notification)) + .flatMap( + notificationChannel -> + sendNotificationThroughChannel(notification, notificationChannel)) + .subscribe(); + } - private Mono sendNotificationThroughChannel(IndexerNotification notification, NotificationChannel notificationChannel) { - return notificationChannel.send(notification) - .onErrorResume(e -> { - log.error("failed to deliver notification {} to channel {}", notification, notificationChannel, e); - return Mono.just(false); + private Mono sendNotificationThroughChannel( + IndexerNotification notification, NotificationChannel notificationChannel) { + return notificationChannel + .send(notification) + .onErrorResume( + e -> { + log.error( + "failed to deliver notification {} to channel {}", + notification, + notificationChannel, + e); + return Mono.just(false); }); - } + } - private List getEligibleChannels(IndexerNotification notification) { - return notificationChannels.stream() - .filter(notificationChannel -> shouldReceiveNotification(notification, notificationChannel)) - .collect(Collectors.toUnmodifiableList()); - } + private List getEligibleChannels(IndexerNotification notification) { + return notificationChannels.stream() + .filter(notificationChannel -> shouldReceiveNotification(notification, notificationChannel)) + .collect(Collectors.toUnmodifiableList()); + } - private boolean shouldReceiveNotification(IndexerNotification notification, NotificationChannel notificationChannel) { - return notificationChannel.subscriptions().contains(NotificationName.ALL) - || notificationChannel.subscriptions().contains(notification.getNotificationName()); - } + private boolean shouldReceiveNotification( + IndexerNotification notification, NotificationChannel notificationChannel) { + return notificationChannel.subscriptions().contains(NotificationName.ALL) + || notificationChannel.subscriptions().contains(notification.getNotificationName()); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/BadDataException.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/BadDataException.java index a68fd92c..4c5fd138 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/BadDataException.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/BadDataException.java @@ -17,11 +17,9 @@ package bio.overture.maestro.domain.api.exception; -/** - * This exception is to indicate that a data processing issue happened. - */ +/** This exception is to indicate that a data processing issue happened. */ public class BadDataException extends IndexerException { - public BadDataException(String message) { - super(message); - } + public BadDataException(String message) { + super(message); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/FailureData.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/FailureData.java index f9887630..a50d61b4 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/FailureData.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/FailureData.java @@ -17,40 +17,39 @@ package bio.overture.maestro.domain.api.exception; -import lombok.Builder; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; - import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; @Getter @Builder @ToString @EqualsAndHashCode public class FailureData { - @Builder.Default - private final Map> failingIds = new HashMap<>(); + @Builder.Default private final Map> failingIds = new HashMap<>(); - private void addFailures(String type, Set ids) { - if (failingIds.containsKey(type)) { - failingIds.put(type, - Stream.concat(failingIds.get(type).stream(), ids.stream()) - .collect(Collectors.toUnmodifiableSet())); - return; - } - failingIds.put(type, Set.copyOf(ids)); + private void addFailures(String type, Set ids) { + if (failingIds.containsKey(type)) { + failingIds.put( + type, + Stream.concat(failingIds.get(type).stream(), ids.stream()) + .collect(Collectors.toUnmodifiableSet())); + return; } + failingIds.put(type, Set.copyOf(ids)); + } - public Map> getFailingIds() { - return Map.copyOf(this.failingIds); - } + public Map> getFailingIds() { + return Map.copyOf(this.failingIds); + } - public void addFailures(FailureData failureData) { - failureData.getFailingIds().forEach(this::addFailures); - } + public void addFailures(FailureData failureData) { + failureData.getFailingIds().forEach(this::addFailures); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/IndexerException.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/IndexerException.java index f04595f6..e53c4300 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/IndexerException.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/IndexerException.java @@ -27,13 +27,14 @@ @NoArgsConstructor @AllArgsConstructor public class IndexerException extends RuntimeException { - protected FailureData failureData; - public IndexerException(String message) { - super(message); - } + protected FailureData failureData; - public IndexerException(String message, Throwable cause, FailureData failureData) { - super(message, cause); - this.failureData = failureData; - } + public IndexerException(String message) { + super(message); + } + + public IndexerException(String message, Throwable cause, FailureData failureData) { + super(message, cause); + this.failureData = failureData; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/NotFoundException.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/NotFoundException.java index e9b619c0..7c51a027 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/NotFoundException.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/NotFoundException.java @@ -17,29 +17,27 @@ package bio.overture.maestro.domain.api.exception; +import static java.lang.String.format; + import lombok.NoArgsConstructor; import lombok.NonNull; -import static java.lang.String.format; - -/** - * Indicates a required / requested resource / data is missing. - */ +/** Indicates a required / requested resource / data is missing. */ @NoArgsConstructor public class NotFoundException extends IndexerException { - public NotFoundException(String message) { - super(message); - } + public NotFoundException(String message) { + super(message); + } - public static NotFoundException buildNotFoundException( - @NonNull String formattedMessage, Object... args) { - return new NotFoundException(format(formattedMessage, args)); - } + public static NotFoundException buildNotFoundException( + @NonNull String formattedMessage, Object... args) { + return new NotFoundException(format(formattedMessage, args)); + } - public static void checkNotFound( - boolean expression, @NonNull String formattedMessage, @NonNull Object... args) { - if (!expression) { - throw new NotFoundException(format(formattedMessage, args)); - } + public static void checkNotFound( + boolean expression, @NonNull String formattedMessage, @NonNull Object... args) { + if (!expression) { + throw new NotFoundException(format(formattedMessage, args)); } + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AddRuleCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AddRuleCommand.java index be5a4c35..a0f622bc 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AddRuleCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AddRuleCommand.java @@ -17,5 +17,4 @@ package bio.overture.maestro.domain.api.message; -public class AddRuleCommand { -} +public class AddRuleCommand {} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AnalysisIdentifier.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AnalysisIdentifier.java index ba9d1c54..da176eb1 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AnalysisIdentifier.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AnalysisIdentifier.java @@ -26,10 +26,7 @@ @EqualsAndHashCode @AllArgsConstructor public class AnalysisIdentifier { - @NonNull - private String analysisId; - @NonNull - private String studyId; - @NonNull - private String repositoryCode; + @NonNull private String analysisId; + @NonNull private String studyId; + @NonNull private String repositoryCode; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/DeleteRuleCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/DeleteRuleCommand.java index 3b14457a..27b3856b 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/DeleteRuleCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/DeleteRuleCommand.java @@ -17,5 +17,4 @@ package bio.overture.maestro.domain.api.message; -public class DeleteRuleCommand { -} +public class DeleteRuleCommand {} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexAnalysisCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexAnalysisCommand.java index b3fd4350..4e2656e6 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexAnalysisCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexAnalysisCommand.java @@ -26,6 +26,5 @@ @EqualsAndHashCode @AllArgsConstructor public class IndexAnalysisCommand { - @NonNull - private AnalysisIdentifier analysisIdentifier; + @NonNull private AnalysisIdentifier analysisIdentifier; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexResult.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexResult.java index e30fcefc..4d2ad14b 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexResult.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexResult.java @@ -27,7 +27,7 @@ @EqualsAndHashCode @AllArgsConstructor public class IndexResult { - private String indexName; - @Builder.Default private FailureData failureData = FailureData.builder().build(); - private boolean successful; + private String indexName; + @Builder.Default private FailureData failureData = FailureData.builder().build(); + private boolean successful; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyCommand.java index 968da567..ccaadeb8 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyCommand.java @@ -26,8 +26,6 @@ @EqualsAndHashCode @AllArgsConstructor public class IndexStudyCommand { - @NonNull - private String studyId; - @NonNull - private String repositoryCode; + @NonNull private String studyId; + @NonNull private String repositoryCode; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyRepositoryCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyRepositoryCommand.java index f309656f..73cd4c97 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyRepositoryCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyRepositoryCommand.java @@ -26,6 +26,5 @@ @EqualsAndHashCode @AllArgsConstructor public class IndexStudyRepositoryCommand { - @NonNull - private String repositoryCode; + @NonNull private String repositoryCode; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/RemoveAnalysisCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/RemoveAnalysisCommand.java index 099db8f4..7df34b0a 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/RemoveAnalysisCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/RemoveAnalysisCommand.java @@ -26,6 +26,5 @@ @EqualsAndHashCode @AllArgsConstructor public class RemoveAnalysisCommand { - @NonNull - private AnalysisIdentifier analysisIdentifier; + @NonNull private AnalysisIdentifier analysisIdentifier; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/AnalysisType.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/AnalysisType.java index b7f547a1..dba76c7a 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/AnalysisType.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/AnalysisType.java @@ -8,6 +8,6 @@ @AllArgsConstructor @EqualsAndHashCode public class AnalysisType { - private String name; - private Integer version; + private String name; + private Integer version; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/File.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/File.java index ae78a942..4dfff28b 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/File.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/File.java @@ -26,11 +26,9 @@ @AllArgsConstructor @EqualsAndHashCode public class File { - @NonNull - private String name; - @NonNull - private String md5sum; - private Long size; - private String dataType; - private IndexFile indexFile; + @NonNull private String name; + @NonNull private String md5sum; + private Long size; + private String dataType; + private IndexFile indexFile; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricAnalysis.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricAnalysis.java index 3a574ae1..b48a4e24 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricAnalysis.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricAnalysis.java @@ -19,11 +19,10 @@ import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; -import lombok.*; -import lombok.experimental.FieldNameConstants; - import java.util.Map; import java.util.TreeMap; +import lombok.*; +import lombok.experimental.FieldNameConstants; @Getter @Builder @@ -34,37 +33,31 @@ @FieldNameConstants public class FileCentricAnalysis { - @NonNull - private String analysisId; - @NonNull - private String analysisType; - @NonNull - private Integer analysisVersion; - @NonNull - private String analysisState; - @NonNull - private Map experiment; - /** - * this field is to capture the dynamic fields in the analysis. - * it's the responsibility of the users to make sure the mapping is consistent with - * the different fields that they want to add/index, they are also responsibile - * to add the mappings of these fields or reindex appropriately. - */ - @NonNull - private final Map data = new TreeMap<>(); - - @JsonAnyGetter - public Map getData() { - return data; - } - - @JsonAnySetter - public void setData(String key, Object value) { - data.put(key, value); - } - - public void replaceData(Map data) { - this.data.clear(); - this.data.putAll(data); - } + @NonNull private String analysisId; + @NonNull private String analysisType; + @NonNull private Integer analysisVersion; + @NonNull private String analysisState; + @NonNull private Map experiment; + /** + * this field is to capture the dynamic fields in the analysis. it's the responsibility of the + * users to make sure the mapping is consistent with the different fields that they want to + * add/index, they are also responsibile to add the mappings of these fields or reindex + * appropriately. + */ + @NonNull private final Map data = new TreeMap<>(); + + @JsonAnyGetter + public Map getData() { + return data; + } + + @JsonAnySetter + public void setData(String key, Object value) { + data.put(key, value); + } + + public void replaceData(Map data) { + this.data.clear(); + this.data.putAll(data); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDocument.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDocument.java index 837fdd15..fff97dbb 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDocument.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDocument.java @@ -17,14 +17,11 @@ package bio.overture.maestro.domain.entities.indexing; +import java.util.List; import lombok.*; import lombok.experimental.FieldNameConstants; -import java.util.List; - -/** - * Represents the structure of the index document that corresponds to an analysis "File". - */ +/** Represents the structure of the index document that corresponds to an analysis "File". */ @Builder @Getter @ToString @@ -34,57 +31,48 @@ @FieldNameConstants public class FileCentricDocument { - @NonNull - private String objectId; + @NonNull private String objectId; - @NonNull - private String studyId; + @NonNull private String studyId; - private String dataType; + private String dataType; - private String fileType; + private String fileType; - private String fileAccess; + private String fileAccess; - @NonNull - private FileCentricAnalysis analysis; + @NonNull private FileCentricAnalysis analysis; - /** - * The actual genome analysis files information. - */ - @NonNull - private File file; + /** The actual genome analysis files information. */ + @NonNull private File file; - /** - * Each files can be hosted in more than one files repository, this references the other repositories (locations) - * where this files can be fetched from. - */ - @NonNull - private List repositories; + /** + * Each files can be hosted in more than one files repository, this references the other + * repositories (locations) where this files can be fetched from. + */ + @NonNull private List repositories; - @NonNull - private List donors; + @NonNull private List donors; - /** - * This method is to check if the files is a valid replica of another files. - * by replication we mean that an analysis can be copied to a different metadata repository to make downloading - * the files faster for different geographical locations. - * it checks all attributes except for the repository (since the repository is expected to be different) - * - * @param fileCentricDocument the other files we compare to - * @return flag indicates if this is a valid replica. - */ - public boolean isValidReplica(FileCentricDocument fileCentricDocument) { - if (fileCentricDocument == null) return false; - if (this.equals(fileCentricDocument)) return true; - return this.objectId.equals(fileCentricDocument.getObjectId()) - && this.studyId.equals(fileCentricDocument.getStudyId()) - && this.dataType.equals(fileCentricDocument.getDataType()) - && this.fileType.equals(fileCentricDocument.getFileType()) - && this.fileAccess.equals(fileCentricDocument.getFileAccess()) - && this.donors.equals(fileCentricDocument.getDonors()) - && this.analysis.equals(fileCentricDocument.getAnalysis()) - && this.file.equals(fileCentricDocument.getFile()); - } + /** + * This method is to check if the files is a valid replica of another files. by replication we + * mean that an analysis can be copied to a different metadata repository to make downloading the + * files faster for different geographical locations. it checks all attributes except for the + * repository (since the repository is expected to be different) + * + * @param fileCentricDocument the other files we compare to + * @return flag indicates if this is a valid replica. + */ + public boolean isValidReplica(FileCentricDocument fileCentricDocument) { + if (fileCentricDocument == null) return false; + if (this.equals(fileCentricDocument)) return true; + return this.objectId.equals(fileCentricDocument.getObjectId()) + && this.studyId.equals(fileCentricDocument.getStudyId()) + && this.dataType.equals(fileCentricDocument.getDataType()) + && this.fileType.equals(fileCentricDocument.getFileType()) + && this.fileAccess.equals(fileCentricDocument.getFileAccess()) + && this.donors.equals(fileCentricDocument.getDonors()) + && this.analysis.equals(fileCentricDocument.getAnalysis()) + && this.file.equals(fileCentricDocument.getFile()); + } } - diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDonor.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDonor.java index c193d939..daabd5a3 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDonor.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDonor.java @@ -17,8 +17,8 @@ package bio.overture.maestro.domain.entities.indexing; -import lombok.*; import java.util.List; +import lombok.*; @Builder @Getter @@ -27,15 +27,11 @@ @EqualsAndHashCode @AllArgsConstructor public class FileCentricDonor { - @NonNull - private String donorId; + @NonNull private String donorId; - @NonNull - private String gender; + @NonNull private String gender; - @NonNull - private String submitterDonorId; + @NonNull private String submitterDonorId; - @NonNull - private List specimens; + @NonNull private List specimens; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/IndexFile.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/IndexFile.java index c61e046d..6c74478c 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/IndexFile.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/IndexFile.java @@ -26,14 +26,10 @@ @AllArgsConstructor @EqualsAndHashCode public class IndexFile { - @NonNull - private String objectId; - @NonNull - private String name; - @NonNull - private String fileType; - @NonNull - private String md5sum; - private String dataType; - private Long size; + @NonNull private String objectId; + @NonNull private String name; + @NonNull private String fileType; + @NonNull private String md5sum; + private String dataType; + private Long size; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Repository.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Repository.java index bf2edcdc..60d95b03 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Repository.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Repository.java @@ -17,7 +17,6 @@ package bio.overture.maestro.domain.entities.indexing; - import lombok.*; @Builder @@ -27,20 +26,15 @@ @NoArgsConstructor @AllArgsConstructor public class Repository { - @NonNull - private String code; + @NonNull private String code; - @NonNull - private String organization; + @NonNull private String organization; - private String name; + private String name; - @NonNull - private String type; + @NonNull private String type; - @NonNull - private String country; + @NonNull private String country; - @NonNull - private String url; + @NonNull private String url; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Sample.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Sample.java index bba406df..5115917d 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Sample.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Sample.java @@ -20,8 +20,7 @@ import lombok.*; /** - * Many samples can belong to an Analysis, a samples represents - * a donor and a specimen composition. + * Many samples can belong to an Analysis, a samples represents a donor and a specimen composition. */ @Getter @Builder @@ -30,11 +29,8 @@ @AllArgsConstructor @EqualsAndHashCode public class Sample { - @NonNull - private String sampleId; - @NonNull - private String submitterSampleId; - @NonNull - private String sampleType; - private String matchedNormalSubmitterSampleId; + @NonNull private String sampleId; + @NonNull private String submitterSampleId; + @NonNull private String sampleType; + private String matchedNormalSubmitterSampleId; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Specimen.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Specimen.java index a9ca6379..5389b326 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Specimen.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Specimen.java @@ -18,8 +18,8 @@ package bio.overture.maestro.domain.entities.indexing; import com.fasterxml.jackson.annotation.JsonInclude; -import lombok.*; import java.util.List; +import lombok.*; @Getter @Builder @@ -29,15 +29,10 @@ @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) public class Specimen { - @NonNull - private String specimenId; - @NonNull - private String specimenType; - @NonNull - private String submitterSpecimenId; - @NonNull - private List samples; - private String tumourNormalDesignation; - private String specimenTissueSource; + @NonNull private String specimenId; + @NonNull private String specimenType; + @NonNull private String submitterSpecimenId; + @NonNull private List samples; + private String tumourNormalDesignation; + private String specimenTissueSource; } - diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/StorageType.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/StorageType.java index 768fee2a..6765d609 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/StorageType.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/StorageType.java @@ -18,5 +18,5 @@ package bio.overture.maestro.domain.entities.indexing; public enum StorageType { - S3 + S3 } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDocument.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDocument.java index 31616c1e..5fa69959 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDocument.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDocument.java @@ -3,11 +3,11 @@ import bio.overture.maestro.domain.entities.indexing.Repository; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; -import lombok.*; -import lombok.experimental.FieldNameConstants; import java.util.List; import java.util.Map; import java.util.TreeMap; +import lombok.*; +import lombok.experimental.FieldNameConstants; @Builder @Getter diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDonor.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDonor.java index 7757a7e8..5a81cceb 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDonor.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDonor.java @@ -1,11 +1,10 @@ package bio.overture.maestro.domain.entities.indexing.analysis; import bio.overture.maestro.domain.entities.indexing.Specimen; +import java.util.List; import lombok.*; import lombok.experimental.FieldNameConstants; -import java.util.List; - @Builder @Getter @ToString @@ -14,16 +13,11 @@ @EqualsAndHashCode @FieldNameConstants public class AnalysisCentricDonor { - @NonNull - private String donorId; - - @NonNull - private String submitterDonorId; + @NonNull private String donorId; - @NonNull - private String gender; + @NonNull private String submitterDonorId; - @NonNull - private List specimens; + @NonNull private String gender; + @NonNull private List specimens; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricFile.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricFile.java index 1584111f..a5688e3e 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricFile.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricFile.java @@ -10,7 +10,7 @@ @AllArgsConstructor @EqualsAndHashCode @FieldNameConstants -public class AnalysisCentricFile { +public class AnalysisCentricFile { @NonNull private String objectId; diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricSpecimen.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricSpecimen.java index da50b957..4f6f0042 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricSpecimen.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricSpecimen.java @@ -19,5 +19,4 @@ public class AnalysisCentricSpecimen { @NonNull private Sample samples; private String tumourNormalDesignation; private String specimenTissueSource; - } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionId.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionId.java index 18613172..6543a2fc 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionId.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionId.java @@ -23,9 +23,9 @@ import java.lang.annotation.Target; /** - * A metadata annotation to indicate a field annotated with this - * qualifies to be processed by {@link IDExclusionRule} + * A metadata annotation to indicate a field annotated with this qualifies to be processed by {@link + * IDExclusionRule} */ @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.FIELD}) -public @interface ExclusionId { } +public @interface ExclusionId {} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionRule.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionRule.java index d7cfebe6..d62e280d 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionRule.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionRule.java @@ -17,9 +17,7 @@ package bio.overture.maestro.domain.entities.indexing.rules; -/** - * A generic extendable rule to indicate if a rule applies to an instance. - */ +/** A generic extendable rule to indicate if a rule applies to an instance. */ public abstract class ExclusionRule { - abstract public boolean applies(Object instance); + public abstract boolean applies(Object instance); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/IDExclusionRule.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/IDExclusionRule.java index 798d1797..8eab68fc 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/IDExclusionRule.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/IDExclusionRule.java @@ -17,19 +17,18 @@ package bio.overture.maestro.domain.entities.indexing.rules; - -import lombok.*; -import lombok.extern.slf4j.Slf4j; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import lombok.*; +import lombok.extern.slf4j.Slf4j; /** - * When applied on an instance it checks if the field annotated with {@link ExclusionId} - * is in the list of ids that should be excluded. + * When applied on an instance it checks if the field annotated with {@link ExclusionId} is in the + * list of ids that should be excluded. * - * if multiple fields in the instance marked with this, first one (as returned in the class metadata) wins + *

if multiple fields in the instance marked with this, first one (as returned in the class + * metadata) wins */ @Slf4j @Getter @@ -40,46 +39,41 @@ @EqualsAndHashCode(callSuper = true) public class IDExclusionRule extends ExclusionRule { - /** - * the class this rules applies to. - */ - private Class clazz; + /** the class this rules applies to. */ + private Class clazz; - /** - * the list of ids to be excluded. - */ - @Builder.Default - private List ids = new ArrayList<>(); + /** the list of ids to be excluded. */ + @Builder.Default private List ids = new ArrayList<>(); - @SneakyThrows - public boolean applies(Object instance) { - log.trace("checking rule against : {}", instance); - if (ids.isEmpty() || !instance.getClass().equals(clazz)) return false; + @SneakyThrows + public boolean applies(Object instance) { + log.trace("checking rule against : {}", instance); + if (ids.isEmpty() || !instance.getClass().equals(clazz)) return false; - val idExclusionField = Arrays.stream(instance.getClass().getDeclaredFields()) + val idExclusionField = + Arrays.stream(instance.getClass().getDeclaredFields()) .filter(field -> field.getAnnotationsByType(ExclusionId.class).length > 0) .findFirst() .orElse(null); - if (idExclusionField == null) { - log.trace("idExclusionField is null"); - return false; - } + if (idExclusionField == null) { + log.trace("idExclusionField is null"); + return false; + } - idExclusionField.setAccessible(true); - val value = idExclusionField.get(instance); - if (value == null) { - log.trace("value is null is null"); - return false; - } + idExclusionField.setAccessible(true); + val value = idExclusionField.get(instance); + if (value == null) { + log.trace("value is null is null"); + return false; + } - val excluded = ids.contains(String.valueOf(value)); - log.trace("id exclusion rule for value {} = {}", value, excluded); + val excluded = ids.contains(String.valueOf(value)); + log.trace("id exclusion rule for value {} = {}", value, excluded); - if (excluded) { - log.info("id {} was excluded according to the rules", value); - } - return excluded; + if (excluded) { + log.info("id {} was excluded according to the rules", value); } - + return excluded; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/repository/StudyRepository.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/repository/StudyRepository.java index 550cc928..dd710497 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/repository/StudyRepository.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/repository/StudyRepository.java @@ -17,13 +17,12 @@ package bio.overture.maestro.domain.entities.metadata.repository; - import bio.overture.maestro.domain.entities.indexing.StorageType; import lombok.*; /** - * This represents a studyId (including analyses & files) metadata repository, holds information about sources where this - * indexer can pull metadata from. + * This represents a studyId (including analyses & files) metadata repository, holds information + * about sources where this indexer can pull metadata from. */ @Builder @Getter @@ -32,39 +31,21 @@ @EqualsAndHashCode @AllArgsConstructor public class StudyRepository { - /** - * display name of the repository - */ - @NonNull - private String name; + /** display name of the repository */ + @NonNull private String name; - /** - * a unique code for the repository - */ - @NonNull - private String code; + /** a unique code for the repository */ + @NonNull private String code; - /** - * the country where this files repository resides - */ - @NonNull - private String country; + /** the country where this files repository resides */ + @NonNull private String country; - /** - * based url of the host of this repository metadata - */ - @NonNull - private String url; + /** based url of the host of this repository metadata */ + @NonNull private String url; - /** - * the block storage type of files (s3 usually) - */ - @NonNull - private StorageType storageType; + /** the block storage type of files (s3 usually) */ + @NonNull private StorageType storageType; - /** - * the organization the owns this files repository - */ - @NonNull - private String organization; + /** the organization the owns this files repository */ + @NonNull private String organization; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Analysis.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Analysis.java index 44341149..36d51d67 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Analysis.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Analysis.java @@ -20,16 +20,12 @@ import bio.overture.maestro.domain.entities.indexing.rules.ExclusionId; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; -import lombok.*; - import java.util.List; import java.util.Map; import java.util.TreeMap; +import lombok.*; -/** - * This corresponds to the analysis entity in the file metadata repository. - * - */ +/** This corresponds to the analysis entity in the file metadata repository. */ @Getter @Builder @ToString @@ -38,60 +34,41 @@ @EqualsAndHashCode public class Analysis { - @NonNull - @ExclusionId - private String analysisId; + @NonNull @ExclusionId private String analysisId; - /** - * Method used in this analysis (variantCall, sequenceRead) - */ - @NonNull - private AnalysisTypeId analysisType; + /** Method used in this analysis (variantCall, sequenceRead) */ + @NonNull private AnalysisTypeId analysisType; - /** - * the status of the analysis (published or other values) - */ - @NonNull - private String analysisState; + /** the status of the analysis (published or other values) */ + @NonNull private String analysisState; - /** - * the studyId Id that this analysis belongs to. - */ - @NonNull - private String studyId; + /** the studyId Id that this analysis belongs to. */ + @NonNull private String studyId; - /** - * multiple files belong to an analysis, files can be related (bam, bai, xml) - */ - @NonNull - private List files; + /** multiple files belong to an analysis, files can be related (bam, bai, xml) */ + @NonNull private List files; - /** - * An analysis can have one or more samples - */ - @NonNull - private List samples; + /** An analysis can have one or more samples */ + @NonNull private List samples; - /** - * extra information about the analysis type. - * this will contain attributes that change by the analysis type. - *

- * see the source repository api for more info if needed. - */ - private Map experiment; + /** + * extra information about the analysis type. this will contain attributes that change by the + * analysis type. + * + *

see the source repository api for more info if needed. + */ + private Map experiment; - /** - * data represents song dynamic fields. - */ - @NonNull private final Map data = new TreeMap<>(); + /** data represents song dynamic fields. */ + @NonNull private final Map data = new TreeMap<>(); - @JsonAnyGetter - public Map getData() { - return data; - } + @JsonAnyGetter + public Map getData() { + return data; + } - @JsonAnySetter - public void setData(String key, Object value) { - data.put(key, value); - } + @JsonAnySetter + public void setData(String key, Object value) { + data.put(key, value); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Donor.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Donor.java index 326169b8..d8371b15 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Donor.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Donor.java @@ -18,14 +18,10 @@ package bio.overture.maestro.domain.entities.metadata.study; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionId; -import lombok.*; - import java.util.Map; +import lombok.*; -/** - * This represents the samples donor - * A donor is part of the {@link Sample} entity. - */ +/** This represents the samples donor A donor is part of the {@link Sample} entity. */ @Getter @Builder @ToString @@ -33,33 +29,18 @@ @AllArgsConstructor @EqualsAndHashCode public class Donor { - /** - * The id of this donor - */ - @NonNull - @ExclusionId - private String donorId; + /** The id of this donor */ + @NonNull @ExclusionId private String donorId; - /** - * the id as submitted by the analysis creator - */ - @NonNull - private String submitterDonorId; + /** the id as submitted by the analysis creator */ + @NonNull private String submitterDonorId; - /** - * the studyId which this donor belongs to - */ - @NonNull - private String studyId; + /** the studyId which this donor belongs to */ + @NonNull private String studyId; - /** - * can be Male, Female, Other - */ - @NonNull - private String gender; + /** can be Male, Female, Other */ + @NonNull private String gender; - /** - * for extra information if any. - */ - private Map info; + /** for extra information if any. */ + private Map info; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/File.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/File.java index 0ce67baf..e76a2a76 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/File.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/File.java @@ -18,16 +18,15 @@ package bio.overture.maestro.domain.entities.metadata.study; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionId; -import lombok.*; - import java.util.Map; +import lombok.*; /** * A file represents an analysis output that results from the experiment on the Donor specimen. * multiple files belong to one Analysis and reside in an object store. * - * A file can reside in multiple repositories and it can have relations to other files in a single analysis - * like BAM and its index file BAI. + *

A file can reside in multiple repositories and it can have relations to other files in a + * single analysis like BAM and its index file BAI. */ @Getter @Builder @@ -36,22 +35,14 @@ @AllArgsConstructor @EqualsAndHashCode public class File { - @NonNull - @ExclusionId - private String objectId; - @NonNull - private String studyId; - @NonNull - private String analysisId; - @NonNull - private String fileName; - @NonNull - private String fileType; - @NonNull - private String fileMd5sum; - @NonNull - private String fileAccess; - private String dataType; - private long fileSize; - private Map info; + @NonNull @ExclusionId private String objectId; + @NonNull private String studyId; + @NonNull private String analysisId; + @NonNull private String fileName; + @NonNull private String fileType; + @NonNull private String fileMd5sum; + @NonNull private String fileAccess; + private String dataType; + private long fileSize; + private Map info; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Sample.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Sample.java index d667bfee..e272ac5e 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Sample.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Sample.java @@ -18,13 +18,11 @@ package bio.overture.maestro.domain.entities.metadata.study; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionId; -import lombok.*; - import java.util.Map; +import lombok.*; /** - * Many samples can belong to an Analysis, a samples represents - * a donor and a specimen composition. + * Many samples can belong to an Analysis, a samples represents a donor and a specimen composition. */ @Getter @Builder @@ -33,13 +31,12 @@ @EqualsAndHashCode @AllArgsConstructor public class Sample { - @ExclusionId - private String sampleId; - private String specimenId; - private String submitterSampleId; - private String matchedNormalSubmitterSampleId; - private String sampleType; - private Donor donor; - private Specimen specimen; - private Map info; + @ExclusionId private String sampleId; + private String specimenId; + private String submitterSampleId; + private String matchedNormalSubmitterSampleId; + private String sampleType; + private Donor donor; + private Specimen specimen; + private Map info; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Specimen.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Specimen.java index 3989c212..8d80ebef 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Specimen.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Specimen.java @@ -18,14 +18,10 @@ package bio.overture.maestro.domain.entities.metadata.study; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionId; -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.*; - import java.util.Map; +import lombok.*; -/** - * A Specimen provides information about the source of the samples - */ +/** A Specimen provides information about the source of the samples */ @Getter @Builder @ToString @@ -33,12 +29,11 @@ @AllArgsConstructor @EqualsAndHashCode public class Specimen { - @ExclusionId - private String specimenId; - private String donorId; - private String submitterSpecimenId; - private String tumourNormalDesignation; - private String specimenTissueSource; - private String specimenType; - private Map info; + @ExclusionId private String specimenId; + private String donorId; + private String submitterSpecimenId; + private String tumourNormalDesignation; + private String specimenTissueSource; + private String specimenType; + private Map info; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Study.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Study.java index 92015eb0..148dcd66 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Study.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Study.java @@ -27,7 +27,5 @@ @EqualsAndHashCode @AllArgsConstructor public class Study { - @NonNull - @ExclusionId - private String studyId; + @NonNull @ExclusionId private String studyId; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/AnalysisCentricIndexAdapter.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/AnalysisCentricIndexAdapter.java index 5470e375..27892799 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/AnalysisCentricIndexAdapter.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/AnalysisCentricIndexAdapter.java @@ -4,8 +4,8 @@ import lombok.NonNull; import reactor.core.publisher.Mono; -public interface AnalysisCentricIndexAdapter { - - Mono batchUpsertAnalysisRepositories(@NonNull BatchIndexAnalysisCommand batchIndexAnalysisCommand); +public interface AnalysisCentricIndexAdapter { + Mono batchUpsertAnalysisRepositories( + @NonNull BatchIndexAnalysisCommand batchIndexAnalysisCommand); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexAnalysisCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexAnalysisCommand.java index 1b9faae2..cbab11d6 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexAnalysisCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexAnalysisCommand.java @@ -1,9 +1,8 @@ package bio.overture.maestro.domain.port.outbound.indexing; import bio.overture.maestro.domain.entities.indexing.analysis.AnalysisCentricDocument; -import lombok.*; - import java.util.List; +import lombok.*; @Getter @Builder @@ -12,8 +11,7 @@ @EqualsAndHashCode public class BatchIndexAnalysisCommand { - @NonNull - private List analyses; + @NonNull private List analyses; public String toString() { val size = analyses == null ? "null" : String.valueOf(analyses.size()); diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexFilesCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexFilesCommand.java index 666a753a..57ca1708 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexFilesCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexFilesCommand.java @@ -18,9 +18,8 @@ package bio.overture.maestro.domain.port.outbound.indexing; import bio.overture.maestro.domain.entities.indexing.FileCentricDocument; -import lombok.*; - import java.util.List; +import lombok.*; @Getter @Builder @@ -29,12 +28,11 @@ @EqualsAndHashCode public class BatchIndexFilesCommand { - @NonNull - private List files; + @NonNull private List files; - // avoid dumping all files info as that's too much - public String toString() { - val size = files == null ? "null" : String.valueOf(files.size()); - return super.toString() + "[files = " + size + "]"; - } + // avoid dumping all files info as that's too much + public String toString() { + val size = files == null ? "null" : String.valueOf(files.size()); + return super.toString() + "[files = " + size + "]"; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/FileCentricIndexAdapter.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/FileCentricIndexAdapter.java index 62af9b7f..ca0f0bcd 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/FileCentricIndexAdapter.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/FileCentricIndexAdapter.java @@ -19,43 +19,42 @@ import bio.overture.maestro.domain.api.message.IndexResult; import bio.overture.maestro.domain.entities.indexing.FileCentricDocument; -import lombok.NonNull; -import reactor.core.publisher.Mono; import java.util.List; import java.util.Set; +import lombok.NonNull; +import reactor.core.publisher.Mono; /** - * Adapter for the indexing server client, this provides the indexer with needed APIs - * to index files centric documents in the index server + * Adapter for the indexing server client, this provides the indexer with needed APIs to index files + * centric documents in the index server */ public interface FileCentricIndexAdapter { - /** - * Updates a fileDocument repositories field, or indexes the whole document if doesn't exist. - * - * @param batchIndexFilesCommand requires the full document to insert if doesn't exist. - * @return flag indicating if the operation was successful. - */ - Mono batchUpsertFileRepositories(@NonNull BatchIndexFilesCommand batchIndexFilesCommand); - - /** - * Batch fetch documents from the index by the specified ids. - * @param ids a list of ids to fetch documents by from elastic search. - * @return List contains found documents. - */ - Mono> fetchByIds(List ids); - - /** - * Method to delete files documents from the files centric index - * - * @param fileCentricDocumentIds the list of files to delete - * @return indexer exception instance contains the list of failures. - */ - Mono removeFiles(Set fileCentricDocumentIds); - - /** - * Remove all files documents related to the specified analysisId - */ - Mono removeAnalysisFiles(String analysisId); - + /** + * Updates a fileDocument repositories field, or indexes the whole document if doesn't exist. + * + * @param batchIndexFilesCommand requires the full document to insert if doesn't exist. + * @return flag indicating if the operation was successful. + */ + Mono batchUpsertFileRepositories( + @NonNull BatchIndexFilesCommand batchIndexFilesCommand); + + /** + * Batch fetch documents from the index by the specified ids. + * + * @param ids a list of ids to fetch documents by from elastic search. + * @return List contains found documents. + */ + Mono> fetchByIds(List ids); + + /** + * Method to delete files documents from the files centric index + * + * @param fileCentricDocumentIds the list of files to delete + * @return indexer exception instance contains the list of failures. + */ + Mono removeFiles(Set fileCentricDocumentIds); + + /** Remove all files documents related to the specified analysisId */ + Mono removeAnalysisFiles(String analysisId); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/rules/ExclusionRulesDAO.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/rules/ExclusionRulesDAO.java index ba9194aa..82adaab7 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/rules/ExclusionRulesDAO.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/rules/ExclusionRulesDAO.java @@ -18,11 +18,10 @@ package bio.overture.maestro.domain.port.outbound.indexing.rules; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionRule; -import reactor.core.publisher.Mono; - import java.util.List; import java.util.Map; +import reactor.core.publisher.Mono; public interface ExclusionRulesDAO { - Mono, List>> getExclusionRules(); + Mono, List>> getExclusionRules(); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/repository/StudyRepositoryDAO.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/repository/StudyRepositoryDAO.java index 931577d3..03ab2fc5 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/repository/StudyRepositoryDAO.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/repository/StudyRepositoryDAO.java @@ -22,14 +22,16 @@ import reactor.core.publisher.Mono; /** - * Provides access to data about the studies repositories that the indexer should read metadata from to get information - * like: url, repository storage type, urls etc + * Provides access to data about the studies repositories that the indexer should read metadata from + * to get information like: url, repository storage type, urls etc */ public interface StudyRepositoryDAO { - /** - * Gets a files repository by code - * @param code the unique code of the repository - * @return the repository or empty null if not found - */ - @NonNull Mono getFilesRepository(@NonNull String code); + /** + * Gets a files repository by code + * + * @param code the unique code of the repository + * @return the repository or empty null if not found + */ + @NonNull + Mono getFilesRepository(@NonNull String code); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAllStudiesCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAllStudiesCommand.java index 069edddf..e58385eb 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAllStudiesCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAllStudiesCommand.java @@ -26,6 +26,5 @@ @EqualsAndHashCode @AllArgsConstructor public class GetAllStudiesCommand { - @NonNull - private String filesRepositoryBaseUrl; + @NonNull private String filesRepositoryBaseUrl; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAnalysisCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAnalysisCommand.java index b33e4b9f..d0198a28 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAnalysisCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAnalysisCommand.java @@ -26,10 +26,7 @@ @AllArgsConstructor @EqualsAndHashCode public class GetAnalysisCommand { - @NonNull - private String analysisId; - @NonNull - private String studyId; - @NonNull - private String filesRepositoryBaseUrl; + @NonNull private String analysisId; + @NonNull private String studyId; + @NonNull private String filesRepositoryBaseUrl; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetStudyAnalysesCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetStudyAnalysesCommand.java index 77d6dd1a..49e39254 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetStudyAnalysesCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetStudyAnalysesCommand.java @@ -26,8 +26,6 @@ @EqualsAndHashCode @AllArgsConstructor public class GetStudyAnalysesCommand { - @NonNull - private String studyId; - @NonNull - private String filesRepositoryBaseUrl; + @NonNull private String studyId; + @NonNull private String filesRepositoryBaseUrl; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/StudyDAO.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/StudyDAO.java index 6ed06025..4a71e7a9 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/StudyDAO.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/StudyDAO.java @@ -19,38 +19,43 @@ import bio.overture.maestro.domain.entities.metadata.study.Analysis; import bio.overture.maestro.domain.entities.metadata.study.Study; +import java.util.List; import lombok.NonNull; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.util.List; - /** - * The studyId repository, which provides APIs to read Studies, analyses, etc information from a StudyRepository + * The studyId repository, which provides APIs to read Studies, analyses, etc information from a + * StudyRepository */ public interface StudyDAO { - /** - * loads analyses for a single studyId from a single repository - * - * @param getStudyAnalysesCommand contains studyId and repository base url - * @return a mono of all analyses found related to a studyId. - * @throws bio.overture.maestro.domain.api.exception.NotFoundException - * in case the studyId wasn't found. - */ - @NonNull Mono> getStudyAnalyses(@NonNull GetStudyAnalysesCommand getStudyAnalysesCommand); + /** + * loads analyses for a single studyId from a single repository + * + * @param getStudyAnalysesCommand contains studyId and repository base url + * @return a mono of all analyses found related to a studyId. + * @throws bio.overture.maestro.domain.api.exception.NotFoundException in case the studyId wasn't + * found. + */ + @NonNull + Mono> getStudyAnalyses(@NonNull GetStudyAnalysesCommand getStudyAnalysesCommand); - /** - * loads all studies in a repository - * @param getStudyAnalysesCommand contains repository url - * @return a flux of all the studies in the specified repository - */ - @NonNull Flux getStudies(@NonNull GetAllStudiesCommand getStudyAnalysesCommand); + /** + * loads all studies in a repository + * + * @param getStudyAnalysesCommand contains repository url + * @return a flux of all the studies in the specified repository + */ + @NonNull + Flux getStudies(@NonNull GetAllStudiesCommand getStudyAnalysesCommand); - /** - * load an analysis of a studyId from a repository - * @param command specifies the analysis id, the studyId and the repository url - * @return the analysis mono - */ - @NonNull Mono getAnalysis(@NonNull GetAnalysisCommand command); + /** + * load an analysis of a studyId from a repository + * + * @param command specifies the analysis id, the studyId and the repository url + * @return the analysis mono + */ + @NonNull + Mono getAnalysis(@NonNull GetAnalysisCommand command); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/notification/IndexerNotification.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/notification/IndexerNotification.java index 47e9d758..3c4200fe 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/notification/IndexerNotification.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/notification/IndexerNotification.java @@ -17,24 +17,22 @@ package bio.overture.maestro.domain.port.outbound.notification; +import static java.text.MessageFormat.format; import bio.overture.maestro.domain.api.NotificationName; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; -import java.util.Map; - -import static java.text.MessageFormat.format; - @Getter @EqualsAndHashCode @AllArgsConstructor public class IndexerNotification { - private final NotificationName notificationName; - private final Map attributes; + private final NotificationName notificationName; + private final Map attributes; - public String toString() { - return format("{0} | {1}", notificationName.name().toUpperCase(), attributes); - } + public String toString() { + return format("{0} | {1}", notificationName.name().toUpperCase(), attributes); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/CollectionsUtil.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/CollectionsUtil.java index 11bc083d..339350ee 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/CollectionsUtil.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/CollectionsUtil.java @@ -17,22 +17,20 @@ package bio.overture.maestro.domain.utility; -import lombok.experimental.UtilityClass; +import static java.lang.Math.min; +import static java.util.stream.Collectors.toMap; import java.util.List; import java.util.Map; import java.util.stream.IntStream; - -import static java.lang.Math.min; -import static java.util.stream.Collectors.toMap; +import lombok.experimental.UtilityClass; @UtilityClass public class CollectionsUtil { - public static Map> partitionList(List list, int partSize) { - return IntStream.iterate(0, i -> i + partSize) - .limit((list.size() + partSize - 1) / partSize) - .boxed() - .collect(toMap(i -> i / partSize, - i -> list.subList(i, min(i + partSize, list.size())))); - } + public static Map> partitionList(List list, int partSize) { + return IntStream.iterate(0, i -> i + partSize) + .limit((list.size() + partSize - 1) / partSize) + .boxed() + .collect(toMap(i -> i / partSize, i -> list.subList(i, min(i + partSize, list.size())))); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Exceptions.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Exceptions.java index ef7ae86d..51d30a4a 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Exceptions.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Exceptions.java @@ -17,27 +17,28 @@ package bio.overture.maestro.domain.utility; +import static java.text.MessageFormat.format; + import bio.overture.maestro.domain.api.exception.BadDataException; import bio.overture.maestro.domain.api.exception.FailureData; import bio.overture.maestro.domain.api.exception.IndexerException; import bio.overture.maestro.domain.api.exception.NotFoundException; import lombok.experimental.UtilityClass; -import static java.text.MessageFormat.format; - @UtilityClass public final class Exceptions { - public static Exception notFound(String msg, Object ...args) { - return new NotFoundException(format(msg, args)); - } + public static Exception notFound(String msg, Object... args) { + return new NotFoundException(format(msg, args)); + } - public static Exception badData(String msg, Object ...args) { - return new BadDataException(format(msg, args)); - } + public static Exception badData(String msg, Object... args) { + return new BadDataException(format(msg, args)); + } - public static IndexerException wrapWithIndexerException(Throwable e, String message, FailureData failureData) { - if (e instanceof IndexerException) return (IndexerException) e; - return new IndexerException(message, e, failureData); - } + public static IndexerException wrapWithIndexerException( + Throwable e, String message, FailureData failureData) { + if (e instanceof IndexerException) return (IndexerException) e; + return new IndexerException(message, e, failureData); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Parallel.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Parallel.java index b595153a..b06df40f 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Parallel.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Parallel.java @@ -17,57 +17,62 @@ package bio.overture.maestro.domain.utility; -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; -import lombok.extern.slf4j.Slf4j; -import lombok.val; +import static bio.overture.maestro.domain.utility.CollectionsUtil.partitionList; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.stream.Collectors; - -import static bio.overture.maestro.domain.utility.CollectionsUtil.partitionList; +import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; +import lombok.val; @Slf4j @UtilityClass public final class Parallel { - /** - * A BLOCKING (i.e. should be executed on a reactor scheduler) scatter gather to parallelize execution - * of suppliers, currently it uses completeable futures but may be transformed to reactor publishers. - * it handles paritioning an input list to a batch and create the necessary number of workers. - * it uses the default completeable future work stealing ForkJoin pool. - * - * @param inputList the list of input parameters that we want to split and pass to the supplier - * @param batchSize the size of the batch each supplier will handler - * @param supplier the function to take an input batch and supply the result - * @param the type parameter for the inputs - * @param the return type of the results - * @return a list of R - * @throws Exception this is here to force callers to handle the exceptions that can be thrown - * this method will only rethrow any exceptiona and fails fast. - * if another behaviour is desired suppliers should handle their exceptions. - */ - @SneakyThrows - public static List blockingScatterGather(List inputList, int batchSize, - Function>, R> supplier) throws Exception { - // scatter - val futures = partitionList(inputList, batchSize).entrySet().stream() + /** + * A BLOCKING (i.e. should be executed on a reactor scheduler) scatter gather to parallelize + * execution of suppliers, currently it uses completeable futures but may be transformed to + * reactor publishers. it handles paritioning an input list to a batch and create the necessary + * number of workers. it uses the default completeable future work stealing ForkJoin pool. + * + * @param inputList the list of input parameters that we want to split and pass to the supplier + * @param batchSize the size of the batch each supplier will handler + * @param supplier the function to take an input batch and supply the result + * @param the type parameter for the inputs + * @param the return type of the results + * @return a list of R + * @throws Exception this is here to force callers to handle the exceptions that can be thrown + * this method will only rethrow any exceptiona and fails fast. if another behaviour is + * desired suppliers should handle their exceptions. + */ + @SneakyThrows + public static List blockingScatterGather( + List inputList, int batchSize, Function>, R> supplier) + throws Exception { + // scatter + val futures = + partitionList(inputList, batchSize).entrySet().stream() .map((entry) -> CompletableFuture.supplyAsync(() -> supplier.apply(entry))) .collect(Collectors.toUnmodifiableList()); - //gather - val joinedFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) - .thenApply(v-> futures.stream().map(CompletableFuture::join).collect(Collectors.toUnmodifiableList())); + // gather + val joinedFutures = + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply( + v -> + futures.stream() + .map(CompletableFuture::join) + .collect(Collectors.toUnmodifiableList())); - try { - return joinedFutures.get(); - } catch (Exception e) { - log.error(e.getMessage()); - throw e; - } + try { + return joinedFutures.get(); + } catch (Exception e) { + log.error(e.getMessage()); + throw e; } - + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/StringUtilities.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/StringUtilities.java index 3ee2e112..c7c35729 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/StringUtilities.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/StringUtilities.java @@ -17,32 +17,28 @@ package bio.overture.maestro.domain.utility; +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; import lombok.NonNull; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; import lombok.val; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; - -import static java.nio.charset.StandardCharsets.UTF_8; - @UtilityClass public final class StringUtilities { - /** - * loads a string out of input stream. - */ - @SneakyThrows - public static String inputStreamToString(@NonNull InputStream inputStream) { - try (ByteArrayOutputStream result = new ByteArrayOutputStream()) { - val buffer = new byte[1024]; - int length; - while ((length = inputStream.read(buffer)) != -1) { - result.write(buffer, 0, length); - } - return result.toString(UTF_8); - } + /** loads a string out of input stream. */ + @SneakyThrows + public static String inputStreamToString(@NonNull InputStream inputStream) { + try (ByteArrayOutputStream result = new ByteArrayOutputStream()) { + val buffer = new byte[1024]; + int length; + while ((length = inputStream.read(buffer)) != -1) { + result.write(buffer, 0, length); + } + return result.toString(UTF_8); } - + } } diff --git a/maestro-domain/src/main/java/bio/overture/masestro/test/Fixture.java b/maestro-domain/src/main/java/bio/overture/masestro/test/Fixture.java index 0bb67ef9..b63469a6 100644 --- a/maestro-domain/src/main/java/bio/overture/masestro/test/Fixture.java +++ b/maestro-domain/src/main/java/bio/overture/masestro/test/Fixture.java @@ -17,124 +17,134 @@ package bio.overture.masestro.test; +import static bio.overture.maestro.domain.utility.StringUtilities.inputStreamToString; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; -import lombok.*; -import lombok.experimental.UtilityClass; import java.io.File; import java.io.IOException; import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; - -import static bio.overture.maestro.domain.utility.StringUtilities.inputStreamToString; +import lombok.*; +import lombok.experimental.UtilityClass; /** - * Helper to load test fixtures from resources. - * This class is only for testing purposes not to be used for non test code. + * Helper to load test fixtures from resources. This class is only for testing purposes not to be + * used for non test code. */ @UtilityClass public class Fixture { - private final static String FIXTURE_PATH = "src/test/resources/"; - private final static String BASE_PATH = "fixtures" + File.separator; - private final static ObjectMapper MAPPER = new ObjectMapper(); + private static final String FIXTURE_PATH = "src/test/resources/"; + private static final String BASE_PATH = "fixtures" + File.separator; + private static final ObjectMapper MAPPER = new ObjectMapper(); - @SneakyThrows - public static T loadJsonFixture(Class clazz, String fileName, Class targetClass) { - return loadJsonFixture(clazz, fileName, targetClass, MAPPER); - } + @SneakyThrows + public static T loadJsonFixture(Class clazz, String fileName, Class targetClass) { + return loadJsonFixture(clazz, fileName, targetClass, MAPPER); + } - public static T loadConverterTestFixture(String fileName, Class targetClass) { - return readConverterFixture(fileName, targetClass, MAPPER); - } + public static T loadConverterTestFixture(String fileName, Class targetClass) { + return readConverterFixture(fileName, targetClass, MAPPER); + } - @SneakyThrows - public static T loadJsonFixtureSnakeCase(Class clazz, String fileName, Class targetClass) { - ObjectMapper mapper = new ObjectMapper().setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); - return loadJsonFixture(clazz, fileName, targetClass, mapper); - } + @SneakyThrows + public static T loadJsonFixtureSnakeCase(Class clazz, String fileName, Class targetClass) { + ObjectMapper mapper = + new ObjectMapper().setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); + return loadJsonFixture(clazz, fileName, targetClass, mapper); + } - /** - * Use this overload for generics - */ - @SneakyThrows - public static T loadJsonFixture(Class clazz, String fileName, TypeReference type) { - return loadJsonFixture(clazz, fileName, type, MAPPER); - } - /** - * this overload can be used to load json files and convert it to the target class using a custom mapper - * - * @param clazz will be used to obtain a class loader to get the resources for. - * @param fileName the fixture files we want to load - * @param targetClass the target java type we want to convert the json to - * @param customMapper in case you want to pre configure a mapper (property name case for example) - * @param type parameter of the target class - * @param templateParams parameters map to be replaced in the json files - * use this if you have dynamic values that change each test run - * the placeholder should be ##key## and will be replaced with the value : - * templateParams.get(key) - * - * @return the converted json files as java type - * - */ - @SneakyThrows - public static T loadJsonFixture(Class clazz, - String fileName, - Class targetClass, - ObjectMapper customMapper, - Map templateParams) { - String json = loadJsonString(clazz, fileName); - TemplateResult replaceResult = new TemplateResult(); - replaceResult.setResult(json); - templateParams.forEach((name, value) -> replaceResult.setResult(replaceResult.getResult() - .replaceAll(Pattern.quote("##" + name + "##"), value))); + /** Use this overload for generics */ + @SneakyThrows + public static T loadJsonFixture(Class clazz, String fileName, TypeReference type) { + return loadJsonFixture(clazz, fileName, type, MAPPER); + } + /** + * this overload can be used to load json files and convert it to the target class using a custom + * mapper + * + * @param clazz will be used to obtain a class loader to get the resources for. + * @param fileName the fixture files we want to load + * @param targetClass the target java type we want to convert the json to + * @param customMapper in case you want to pre configure a mapper (property name case for example) + * @param type parameter of the target class + * @param templateParams parameters map to be replaced in the json files use this if you have + * dynamic values that change each test run the placeholder should be ##key## and will be + * replaced with the value : templateParams.get(key) + * @return the converted json files as java type + */ + @SneakyThrows + public static T loadJsonFixture( + Class clazz, + String fileName, + Class targetClass, + ObjectMapper customMapper, + Map templateParams) { + String json = loadJsonString(clazz, fileName); + TemplateResult replaceResult = new TemplateResult(); + replaceResult.setResult(json); + templateParams.forEach( + (name, value) -> + replaceResult.setResult( + replaceResult.getResult().replaceAll(Pattern.quote("##" + name + "##"), value))); - return customMapper.readValue(replaceResult.getResult(), targetClass); - } + return customMapper.readValue(replaceResult.getResult(), targetClass); + } - /** - * this overload can be used to load json files and convert it to the target class using a custom mapper - * - * @param clazz will be used to obtain a class loader to get the resources for. - * @param fileName the fixture files we want to load - * @param targetClass the target java type we want to convert the json to - * @param customMapper in case you want to pre configure a mapper (property name case for example) - * @param type parameter of the target class - * - * @return the converted json files as java type - * - */ - @SneakyThrows - public static T loadJsonFixture(Class clazz, String fileName, Class targetClass, ObjectMapper customMapper) { - String json = loadJsonString(clazz, fileName); - return customMapper.readValue(json, targetClass); - } + /** + * this overload can be used to load json files and convert it to the target class using a custom + * mapper + * + * @param clazz will be used to obtain a class loader to get the resources for. + * @param fileName the fixture files we want to load + * @param targetClass the target java type we want to convert the json to + * @param customMapper in case you want to pre configure a mapper (property name case for example) + * @param type parameter of the target class + * @return the converted json files as java type + */ + @SneakyThrows + public static T loadJsonFixture( + Class clazz, String fileName, Class targetClass, ObjectMapper customMapper) { + String json = loadJsonString(clazz, fileName); + return customMapper.readValue(json, targetClass); + } - @SneakyThrows - public static T loadJsonFixture(Class clazz, String fileName, TypeReference targetClass, ObjectMapper customMapper) { - String json = loadJsonString(clazz, fileName); - return customMapper.readValue(json, targetClass); - } + @SneakyThrows + public static T loadJsonFixture( + Class clazz, String fileName, TypeReference targetClass, ObjectMapper customMapper) { + String json = loadJsonString(clazz, fileName); + return customMapper.readValue(json, targetClass); + } - @SneakyThrows - public static T readConverterFixture(String fileName, Class targetClass, ObjectMapper customMapper) { - return customMapper.readValue(new File( FIXTURE_PATH + BASE_PATH + "DocumentConverterTest" + File.separator + fileName), targetClass); - } + @SneakyThrows + public static T readConverterFixture( + String fileName, Class targetClass, ObjectMapper customMapper) { + return customMapper.readValue( + new File(FIXTURE_PATH + BASE_PATH + "DocumentConverterTest" + File.separator + fileName), + targetClass); + } - public static String loadJsonString(Class clazz, String fileName) throws IOException { - return inputStreamToString( - Optional.ofNullable(clazz.getClassLoader() - .getResource(BASE_PATH + clazz.getSimpleName() + File.separator + fileName) - ).orElseThrow(() -> new RuntimeException("fixture not found. make sure you created the correct " + - "folder if this is a new class or if you renamed the class")).openStream()); - } + public static String loadJsonString(Class clazz, String fileName) throws IOException { + return inputStreamToString( + Optional.ofNullable( + clazz + .getClassLoader() + .getResource(BASE_PATH + clazz.getSimpleName() + File.separator + fileName)) + .orElseThrow( + () -> + new RuntimeException( + "fixture not found. make sure you created the correct " + + "folder if this is a new class or if you renamed the class")) + .openStream()); + } - @Getter - @Setter - @NoArgsConstructor - private static class TemplateResult { - private String result; - } + @Getter + @Setter + @NoArgsConstructor + private static class TemplateResult { + private String result; + } } diff --git a/maestro-domain/src/main/java/bio/overture/masestro/test/TestCategory.java b/maestro-domain/src/main/java/bio/overture/masestro/test/TestCategory.java index 64006977..fccc52de 100644 --- a/maestro-domain/src/main/java/bio/overture/masestro/test/TestCategory.java +++ b/maestro-domain/src/main/java/bio/overture/masestro/test/TestCategory.java @@ -21,6 +21,6 @@ @UtilityClass public class TestCategory { - public final static String UNIT_TEST = "unit_test"; - public final static String INT_TEST = "int_test"; + public static final String UNIT_TEST = "unit_test"; + public static final String INT_TEST = "int_test"; } diff --git a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverterTest.java b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverterTest.java index befd6c0a..9f14bd0c 100644 --- a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverterTest.java +++ b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverterTest.java @@ -1,5 +1,10 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.EntityGenerator.*; +import static bio.overture.masestro.test.Fixture.loadConverterTestFixture; +import static bio.overture.masestro.test.TestCategory.UNIT_TEST; +import static org.junit.jupiter.api.Assertions.*; + import bio.overture.maestro.domain.entities.indexing.analysis.AnalysisCentricDonor; import bio.overture.maestro.domain.entities.metadata.study.*; import lombok.val; @@ -8,11 +13,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import static bio.overture.maestro.domain.api.EntityGenerator.*; -import static bio.overture.masestro.test.Fixture.loadConverterTestFixture; -import static bio.overture.masestro.test.TestCategory.UNIT_TEST; -import static org.junit.jupiter.api.Assertions.*; - @ExtendWith(MockitoExtension.class) @Tag(UNIT_TEST) public class AnalysisCentricDocumentConverterTest { @@ -22,12 +22,13 @@ void testGetDonor() { val analysisObj = loadConverterTestFixture("TEST-CA.analysis.json", Analysis.class); // expected: - val donor = AnalysisCentricDonor.builder() - .donorId("DO1") - .gender("Female") - .submitterDonorId("MDT-AP-0749") - .specimens(buildSpecimenListForDonor()) - .build(); + val donor = + AnalysisCentricDonor.builder() + .donorId("DO1") + .gender("Female") + .submitterDonorId("MDT-AP-0749") + .specimens(buildSpecimenListForDonor()) + .build(); val results = AnalysisCentricDocumentConverter.getDonors(analysisObj); @@ -36,7 +37,7 @@ void testGetDonor() { } @Test - void testGetDonors_multi_donor(){ + void testGetDonors_multi_donor() { // Expected AnalysisCentricDonor data structure: // Analysis => d1 -> sp1 -> [sa1] // d1 -> sp2 -> [sa2] @@ -45,14 +46,16 @@ void testGetDonors_multi_donor(){ val analysisObj = loadConverterTestFixture("TEST-CA.analysis.multi-donor.json", Analysis.class); // expected results: - val donor_1 = AnalysisCentricDonor.builder() + val donor_1 = + AnalysisCentricDonor.builder() .donorId("DO1") .gender("Female") .submitterDonorId("MDT-AP-0749") .specimens(buildSpecimenListForDonor1()) .build(); - val donor_2 = AnalysisCentricDonor.builder() + val donor_2 = + AnalysisCentricDonor.builder() .donorId("DO2") .gender("Female") .submitterDonorId("MDT-AP-0749") diff --git a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DefaultIndexerTest.java b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DefaultIndexerTest.java index 9f88483a..425a2d36 100644 --- a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DefaultIndexerTest.java +++ b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DefaultIndexerTest.java @@ -17,6 +17,16 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.DefaultIndexer.REPO_CODE; +import static bio.overture.masestro.test.Fixture.loadJsonFixture; +import static bio.overture.masestro.test.Fixture.loadJsonFixtureSnakeCase; +import static bio.overture.masestro.test.TestCategory.UNIT_TEST; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; + import bio.overture.maestro.domain.api.exception.FailureData; import bio.overture.maestro.domain.api.exception.IndexerException; import bio.overture.maestro.domain.api.message.*; @@ -39,6 +49,11 @@ import bio.overture.maestro.domain.port.outbound.metadata.study.GetStudyAnalysesCommand; import bio.overture.maestro.domain.port.outbound.metadata.study.StudyDAO; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import lombok.SneakyThrows; import lombok.val; import org.jetbrains.annotations.NotNull; @@ -54,22 +69,6 @@ import reactor.core.publisher.Mono; import reactor.test.StepVerifier; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static bio.overture.maestro.domain.api.DefaultIndexer.REPO_CODE; -import static bio.overture.masestro.test.Fixture.loadJsonFixture; -import static bio.overture.masestro.test.Fixture.loadJsonFixtureSnakeCase; -import static bio.overture.masestro.test.TestCategory.UNIT_TEST; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.BDDMockito.given; -import static org.mockito.BDDMockito.then; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.times; - @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.WARN) @Tag(UNIT_TEST) diff --git a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/EntityGenerator.java b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/EntityGenerator.java index c084fb09..7990412c 100644 --- a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/EntityGenerator.java +++ b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/EntityGenerator.java @@ -2,63 +2,66 @@ import bio.overture.maestro.domain.entities.indexing.Sample; import bio.overture.maestro.domain.entities.indexing.Specimen; -import lombok.val; import java.util.ArrayList; import java.util.List; +import lombok.val; -/** - * Helper class for AnalysisCentricDocumentConverterTest and - * FileCentricDocumentConverterTest. - */ +/** Helper class for AnalysisCentricDocumentConverterTest and FileCentricDocumentConverterTest. */ public class EntityGenerator { public static List buildSpecimenListForDonor() { - val specimen = Specimen.builder() - .specimenId("SP1") - .specimenTissueSource("Other") - .submitterSpecimenId("MDT-AP-0749_tumor_specimen") - .tumourNormalDesignation("Tumour") - .specimenType("Primary tumour - solid tissue") - .samples(List.of( - Sample.builder() - .sampleId("SA1") - .sampleType("DNA") - .submitterSampleId("MDT-AP-0749_tumor") - .matchedNormalSubmitterSampleId("PCSI_0216_St_R") - .build())) - .build(); + val specimen = + Specimen.builder() + .specimenId("SP1") + .specimenTissueSource("Other") + .submitterSpecimenId("MDT-AP-0749_tumor_specimen") + .tumourNormalDesignation("Tumour") + .specimenType("Primary tumour - solid tissue") + .samples( + List.of( + Sample.builder() + .sampleId("SA1") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build())) + .build(); return List.of(specimen); } public static List buildSpecimenListForDonor1() { - val specimen_1 = Specimen.builder() - .specimenId("SP1") - .specimenTissueSource("Other") - .submitterSpecimenId("MDT-AP-0749_tumor_specimen") - .tumourNormalDesignation("Tumour") - .specimenType("Primary tumour - solid tissue") - .samples(List.of( - Sample.builder() - .sampleId("SA1") - .sampleType("DNA") - .submitterSampleId("MDT-AP-0749_tumor") - .matchedNormalSubmitterSampleId("PCSI_0216_St_R") - .build())) - .build(); + val specimen_1 = + Specimen.builder() + .specimenId("SP1") + .specimenTissueSource("Other") + .submitterSpecimenId("MDT-AP-0749_tumor_specimen") + .tumourNormalDesignation("Tumour") + .specimenType("Primary tumour - solid tissue") + .samples( + List.of( + Sample.builder() + .sampleId("SA1") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build())) + .build(); - val specimen_2 = Specimen.builder() - .specimenId("SP2") - .specimenTissueSource("Other") - .submitterSpecimenId("MDT-AP-0749_tumor_specimen") - .tumourNormalDesignation("Tumour") - .specimenType("Primary tumour - solid tissue") - .samples(List.of( - Sample.builder() - .sampleId("SA2") - .sampleType("DNA") - .submitterSampleId("MDT-AP-0749_tumor") - .matchedNormalSubmitterSampleId("PCSI_0216_St_R") - .build())) - .build(); + val specimen_2 = + Specimen.builder() + .specimenId("SP2") + .specimenTissueSource("Other") + .submitterSpecimenId("MDT-AP-0749_tumor_specimen") + .tumourNormalDesignation("Tumour") + .specimenType("Primary tumour - solid tissue") + .samples( + List.of( + Sample.builder() + .sampleId("SA2") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build())) + .build(); val list = new ArrayList(); list.add(specimen_2); @@ -67,22 +70,24 @@ public static List buildSpecimenListForDonor1() { } public static List buildSpecimenListForDonor2() { - val specimen_3 = Specimen.builder() - .specimenId("SP3") - .specimenTissueSource("Other") - .submitterSpecimenId("MDT-AP-0749_tumor_specimen") - .tumourNormalDesignation("Tumour") - .specimenType("Primary tumour - solid tissue") - .samples(buuldSamplesForDonor2_sp3()) - .build(); - val specimen_4 = Specimen.builder() - .specimenId("SP4") - .specimenTissueSource("Other") - .submitterSpecimenId("MDT-AP-0749_tumor_specimen") - .tumourNormalDesignation("Tumour") - .specimenType("Primary tumour - solid tissue") - .samples(buildSamplesForDonor2_sp4()) - .build(); + val specimen_3 = + Specimen.builder() + .specimenId("SP3") + .specimenTissueSource("Other") + .submitterSpecimenId("MDT-AP-0749_tumor_specimen") + .tumourNormalDesignation("Tumour") + .specimenType("Primary tumour - solid tissue") + .samples(buuldSamplesForDonor2_sp3()) + .build(); + val specimen_4 = + Specimen.builder() + .specimenId("SP4") + .specimenTissueSource("Other") + .submitterSpecimenId("MDT-AP-0749_tumor_specimen") + .tumourNormalDesignation("Tumour") + .specimenType("Primary tumour - solid tissue") + .samples(buildSamplesForDonor2_sp4()) + .build(); val list = new ArrayList(); list.add(specimen_4); @@ -91,34 +96,38 @@ public static List buildSpecimenListForDonor2() { } public static List buuldSamplesForDonor2_sp3() { - val sample_3 = Sample.builder() - .sampleId("SA3") - .sampleType("DNA") - .submitterSampleId("MDT-AP-0749_tumor") - .matchedNormalSubmitterSampleId("PCSI_0216_St_R") - .build(); - val sample_4 = Sample.builder() - .sampleId("SA4") - .sampleType("DNA") - .submitterSampleId("MDT-AP-0749_tumor") - .matchedNormalSubmitterSampleId("PCSI_0216_St_R") - .build(); + val sample_3 = + Sample.builder() + .sampleId("SA3") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build(); + val sample_4 = + Sample.builder() + .sampleId("SA4") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build(); return List.of(sample_3, sample_4); } public static List buildSamplesForDonor2_sp4() { - val sample_5 = Sample.builder() - .sampleId("SA5") - .sampleType("DNA") - .submitterSampleId("MDT-AP-0749_tumor") - .matchedNormalSubmitterSampleId("PCSI_0216_St_R") - .build(); - val sample_6 = Sample.builder() - .sampleId("SA6") - .sampleType("DNA") - .submitterSampleId("MDT-AP-0749_tumor") - .matchedNormalSubmitterSampleId("PCSI_0216_St_R") - .build(); + val sample_5 = + Sample.builder() + .sampleId("SA5") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build(); + val sample_6 = + Sample.builder() + .sampleId("SA6") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build(); return List.of(sample_5, sample_6); } } diff --git a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/FileCentricDocumentConverterTest.java b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/FileCentricDocumentConverterTest.java index 9c21dd97..e42a460e 100644 --- a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/FileCentricDocumentConverterTest.java +++ b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/FileCentricDocumentConverterTest.java @@ -1,5 +1,10 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.EntityGenerator.*; +import static bio.overture.masestro.test.Fixture.loadConverterTestFixture; +import static bio.overture.masestro.test.TestCategory.UNIT_TEST; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import bio.overture.maestro.domain.entities.indexing.FileCentricDonor; import bio.overture.maestro.domain.entities.metadata.study.Analysis; @@ -9,12 +14,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; -import static bio.overture.maestro.domain.api.EntityGenerator.*; -import static bio.overture.masestro.test.Fixture.loadConverterTestFixture; -import static bio.overture.masestro.test.TestCategory.UNIT_TEST; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - @ExtendWith(MockitoExtension.class) @Tag(UNIT_TEST) public class FileCentricDocumentConverterTest { @@ -25,12 +24,13 @@ void testGetDonor() { val analysisObj = loadConverterTestFixture("TEST-CA.analysis.json", Analysis.class); // expected: - val donor = FileCentricDonor.builder() - .donorId("DO1") - .gender("Female") - .submitterDonorId("MDT-AP-0749") - .specimens(buildSpecimenListForDonor()) - .build(); + val donor = + FileCentricDonor.builder() + .donorId("DO1") + .gender("Female") + .submitterDonorId("MDT-AP-0749") + .specimens(buildSpecimenListForDonor()) + .build(); val results = FileCentricDocumentConverter.getDonors(analysisObj); @@ -39,7 +39,7 @@ void testGetDonor() { } @Test - void testGetDonors_multi_donor(){ + void testGetDonors_multi_donor() { // Expected FileCentricDonor data structure: // Analysis => d1 -> sp1 -> [sa1] // d1 -> sp2 -> [sa2] @@ -48,19 +48,21 @@ void testGetDonors_multi_donor(){ val analysisObj = loadConverterTestFixture("TEST-CA.analysis.multi-donor.json", Analysis.class); // expected results: - val donor_1 = FileCentricDonor.builder() - .donorId("DO1") - .gender("Female") - .submitterDonorId("MDT-AP-0749") - .specimens(buildSpecimenListForDonor1()) - .build(); + val donor_1 = + FileCentricDonor.builder() + .donorId("DO1") + .gender("Female") + .submitterDonorId("MDT-AP-0749") + .specimens(buildSpecimenListForDonor1()) + .build(); - val donor_2 = FileCentricDonor.builder() - .donorId("DO2") - .gender("Female") - .submitterDonorId("MDT-AP-0749") - .specimens(buildSpecimenListForDonor2()) - .build(); + val donor_2 = + FileCentricDonor.builder() + .donorId("DO2") + .gender("Female") + .submitterDonorId("MDT-AP-0749") + .specimens(buildSpecimenListForDonor2()) + .build(); val results = FileCentricDocumentConverter.getDonors(analysisObj); diff --git a/pom.xml b/pom.xml index 65bfa85d..8fb3e11b 100644 --- a/pom.xml +++ b/pom.xml @@ -371,6 +371,18 @@ + + com.coveo + fmt-maven-plugin + 2.9 + + + + format + + + +