From f81be6e8207b5f5d2f487f2c502278e0852d85ea Mon Sep 17 00:00:00 2001 From: mkr Date: Wed, 6 Nov 2024 11:44:37 +0100 Subject: [PATCH] Change the Solr rule deployment script to support the Querqy 5 rule API. The example smui2solr.sh was still using ssh based file upload to deploy Querqy rules, there is a new way with Querqy 5: the Querqy rewriter API. * New smui2solr.sh deployment script that uses the Querqy 5 API. It is triggered with 4 parameters 1) the path to the common rules file 2) the path to the replace rules file 3) the target system, i.e. PRELIVE or LIVE 4) the Solr core or collection name (taken from the rules collection) * The script expects all other configuration as environment variables: - SMUI_QUERQY_REWRITER_COMMON_RULES: the name of the common rules rewriter the rules should be deployed to - SMUI_QUERQY_REWRITER_COMMON_RULES_DECOMPOUND: the name of the common rules rewriter for decompound rules (required, if decompound rules shall be exported separately) - SMUI_QUERQY_REWRITER_REPLACE: the name of the replace rules rewriter (required, if spelling/replacement rules are used) - SMUI_2SOLR_SOLR_HOST or SMUI_DEPLOY_LIVE_SOLR_HOST: the live Solr host - SMUI_DEPLOY_PRELIVE_SOLR_HOST: the prelive Solr host, if prelive deployment is enabled - SMUI_DEPLOY_(PRE)LIVE_SOLR_(USER|PASSWORD): the live/prelive user/password if Solr basic authentication is used - SMUI_DEPLOY_PRELIVE_SOLR_ALLOW_INSECURE: if the --insecure flag should be used when authenticating (i.e. no cert validation) * Removing all configuration related to the *target* file names of the common rules, optional decompound common rules and replace rules file names as they are no longer copied to a target. * Changing the decision to trigger a GIT based rule deployment to a dedicated toggle / environment variable SMUI_DEPLOYMENT_GIT_ENABLE which needs to be set to true. No longer relying on the magic name "GIT" as the rules file name on the deployment target. * Add jq to SMUI Docker image to facilitate JSON based Querqy request recreation in smui2solr.sh script. --- Dockerfile | 2 +- app/controllers/ApiController.scala | 2 +- app/models/FeatureToggleModel.scala | 10 +- app/services/RulesTxtDeploymentService.scala | 178 ++++++----------- build.sbt | 4 +- conf/application.conf | 32 ++- conf/smui2solr.sh | 185 +++++++++++------- test/resources/smui2test.sh | 4 - ...tDeploymentServiceConfigVariantsSpec.scala | 157 +++------------ .../RulesTxtDeploymentServiceSimpleSpec.scala | 6 +- 10 files changed, 219 insertions(+), 361 deletions(-) diff --git a/Dockerfile b/Dockerfile index 53a73a36..fe5c262e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN --mount=target=/root/.ivy2,type=cache sbt "set assembly / test := {}" clean FROM eclipse-temurin:17-jre-focal RUN apt-get update \ - && apt-get install -y --no-install-recommends openssh-client sshpass bash curl git \ + && apt-get install -y --no-install-recommends openssh-client sshpass bash curl git jq \ && rm -rf /var/lib/apt/lists/* ARG VERSION diff --git a/app/controllers/ApiController.scala b/app/controllers/ApiController.scala index 4d50e046..6b1a800d 100644 --- a/app/controllers/ApiController.scala +++ b/app/controllers/ApiController.scala @@ -326,7 +326,7 @@ class ApiController @Inject()(val controllerComponents: SecurityComponents, logger.debug("In ApiController :: updateRulesTxtForSolrIndexAndTargetPlatform") // generate rules.txt(s) - val rulesFiles = rulesTxtDeploymentService.generateRulesTxtContentWithFilenames(SolrIndexId(solrIndexId), targetSystem) + val rulesFiles = rulesTxtDeploymentService.generateRulesTxtContentWithFilenames(SolrIndexId(solrIndexId)) // validate every generated rules.txt rulesTxtDeploymentService.validateCompleteRulesTxts(rulesFiles) match { diff --git a/app/models/FeatureToggleModel.scala b/app/models/FeatureToggleModel.scala index 8545ff2a..0fb21d13 100644 --- a/app/models/FeatureToggleModel.scala +++ b/app/models/FeatureToggleModel.scala @@ -50,10 +50,10 @@ package object FeatureToggleModel extends Logging { private val FEATURE_TOGGLE_UI_LIST_LIMIT_ITEMS_TO = "toggle.ui-list.limit-items-to" private val FEATURE_TOGGLE_RULE_DEPLOYMENT_LOG_RULE_ID = "toggle.rule-deployment.log-rule-id" private val FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT = "toggle.rule-deployment.split-decompound-rules-txt" - private val FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO" private val FEATURE_TOGGLE_RULE_DEPLOYMENT_PRE_LIVE_PRESENT = "toggle.rule-deployment.pre-live.present" private val FEATURE_TOGGLE_RULE_DEPLOYMENT_CUSTOM_SCRIPT = "toggle.rule-deployment.custom-script" private val FEATURE_TOGGLE_RULE_DEPLOYMENT_CUSTOM_SCRIPT_SMUI2SOLR_SH_PATH = "toggle.rule-deployment.custom-script-SMUI2SOLR-SH_PATH" + private val FEATURE_TOGGLE_RULE_DEPLOYMENT_ENABLE_GIT = "toggle.rule-deployment.git.enable" private val FEATURE_TOGGLE_HEADLINE = "toggle.headline" private val FEATURE_TOGGLE_DEPLOYMENT_LABEL = "toggle.rule-deployment-label" private val FEATURE_TOGGLE_DEPLOYMENT_PRELIVE_LABEL = "toggle.deploy-prelive-fn-label" @@ -173,6 +173,10 @@ package object FeatureToggleModel extends Logging { ) } + def getToggleRuleDeploymentGitEnabled: Boolean = { + appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_ENABLE_GIT).getOrElse(false) + } + def getToggleRuleDeploymentLogRuleId: Boolean = { appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_LOG_RULE_ID).getOrElse(false) } @@ -181,10 +185,6 @@ package object FeatureToggleModel extends Logging { appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT).getOrElse(false) } - def getToggleRuleDeploymentSplitDecompoundRulesTxtDstCpFileTo: String = { - appConfig.getOptional[String](FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT_DST_CP_FILE_TO).getOrElse("") - } - def getToggleRuleDeploymentCustomScript: Boolean = { appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_CUSTOM_SCRIPT).getOrElse(false) } diff --git a/app/services/RulesTxtDeploymentService.scala b/app/services/RulesTxtDeploymentService.scala index 3ff2374c..d7511661 100644 --- a/app/services/RulesTxtDeploymentService.scala +++ b/app/services/RulesTxtDeploymentService.scala @@ -20,19 +20,18 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT environment: Environment) extends Logging { case class RulesTxtsForSolrIndex(solrIndexId: SolrIndexId, - regularRules: RulesTxtWithFileNames, - decompoundRules: Option[RulesTxtWithFileNames], - replaceRules: Option[RulesTxtWithFileNames]) { + regularRules: RulesTxtWithFileName, + decompoundRules: Option[RulesTxtWithFileName], + replaceRules: Option[RulesTxtWithFileName]) { - def regularAndDecompoundFiles: List[RulesTxtWithFileNames] = List(regularRules) ++ decompoundRules + def regularAndDecompoundFiles: List[RulesTxtWithFileName] = List(regularRules) ++ decompoundRules - def allFiles: List[RulesTxtWithFileNames] = regularAndDecompoundFiles ++ replaceRules + def allFiles: List[RulesTxtWithFileName] = regularAndDecompoundFiles ++ replaceRules } - case class RulesTxtWithFileNames(content: String, - sourceFileName: String, - destinationFileName: String) + case class RulesTxtWithFileName(content: String, + sourceFileName: String) /** * Generates a list of source to destination filenames containing the rules.txt(s) according to current application settings. @@ -40,78 +39,45 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT * @param solrIndexId Solr Index Id to generate the output for. */ // TODO evaluate, if logDebug should be used to prevent verbose logging of the whole generated rules.txt (for zip download especially) - def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): RulesTxtsForSolrIndex = { + def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, logDebug: Boolean = true): RulesTxtsForSolrIndex = { - // SMUI config for (regular) LIVE deployment val SRC_TMP_FILE = appConfig.get[String]("smui2solr.SRC_TMP_FILE") - val DST_CP_FILE_TO = appConfig.get[String]("smui2solr.DST_CP_FILE_TO") val DO_SPLIT_DECOMPOUND_RULES_TXT = featureToggleService.getToggleRuleDeploymentSplitDecompoundRulesTxt - val DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = featureToggleService.getToggleRuleDeploymentSplitDecompoundRulesTxtDstCpFileTo - // (additional) SMUI config for PRELIVE deployment - val SMUI_DEPLOY_PRELIVE_FN_RULES_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-rules-txt") - val SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-decompound-txt") - - // Replace rules (spelling) - val EXPORT_REPLACE_RULES = featureToggleService.getToggleActivateSpelling + val DO_EXPORT_REPLACE_RULES = featureToggleService.getToggleActivateSpelling val REPLACE_RULES_SRC_TMP_FILE = appConfig.get[String]("smui2solr.replace-rules-tmp-file") - val REPLACE_RULES_DST_CP_FILE_TO = appConfig.get[String]("smui2solr.replace-rules-dst-cp-file-to") - val SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-replace-txt") if (logDebug) { logger.debug( s""":: generateRulesTxtContentWithFilenames config |:: SRC_TMP_FILE = $SRC_TMP_FILE - |:: DST_CP_FILE_TO = $DST_CP_FILE_TO |:: DO_SPLIT_DECOMPOUND_RULES_TXT = $DO_SPLIT_DECOMPOUND_RULES_TXT - |:: DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = $DECOMPOUND_RULES_TXT_DST_CP_FILE_TO - |:: SMUI_DEPLOY_PRELIVE_FN_RULES_TXT = $SMUI_DEPLOY_PRELIVE_FN_RULES_TXT - |:: SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT = $SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT - |:: EXPORT_REPLACE_RULES = $EXPORT_REPLACE_RULES + |:: EXPORT_REPLACE_RULES = $DO_EXPORT_REPLACE_RULES |:: REPLACE_RULES_SRC_TMP_FILE = $REPLACE_RULES_SRC_TMP_FILE - |:: REPLACE_RULES_DST_CP_FILE_TO = $REPLACE_RULES_DST_CP_FILE_TO - |:: SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT = $SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT """.stripMargin) } // generate one rules.txt by default or two separated, if decompound instructions are supposed to be split - // TODO test correct generation in different scenarios (one vs. two rules.txts, etc.) - val dstCpFileTo = if (targetSystem == "PRELIVE") - SMUI_DEPLOY_PRELIVE_FN_RULES_TXT - else // targetSystem == "LIVE" - DST_CP_FILE_TO - - val replaceRulesDstCpFileTo = - if (targetSystem == "PRELIVE") SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT - else REPLACE_RULES_DST_CP_FILE_TO - - val replaceRules = - if (EXPORT_REPLACE_RULES) { + val optReplaceRules = + if (DO_EXPORT_REPLACE_RULES) { val allCanonicalSpellings = searchManagementRepository.listAllSpellingsWithAlternatives(solrIndexId) - Some(RulesTxtWithFileNames( - QuerqyReplaceRulesGenerator.renderAllCanonicalSpellingsToReplaceRules(allCanonicalSpellings), - REPLACE_RULES_SRC_TMP_FILE, - replaceRulesDstCpFileTo - )) + val spellingsRendered = QuerqyReplaceRulesGenerator.renderAllCanonicalSpellingsToReplaceRules(allCanonicalSpellings) + Some(RulesTxtWithFileName(spellingsRendered, REPLACE_RULES_SRC_TMP_FILE)) } else None if (!DO_SPLIT_DECOMPOUND_RULES_TXT) { RulesTxtsForSolrIndex(solrIndexId, - RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId), SRC_TMP_FILE, dstCpFileTo), + RulesTxtWithFileName(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId), SRC_TMP_FILE), None, - replaceRules + optReplaceRules ) } else { - val decompoundDstCpFileTo = if (targetSystem == "PRELIVE") - SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT - else // targetSystem == "LIVE" - DECOMPOUND_RULES_TXT_DST_CP_FILE_TO + val rulesWithoutDecompounds = querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false) + val decompoundRules = querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true) RulesTxtsForSolrIndex(solrIndexId, - RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false), SRC_TMP_FILE, dstCpFileTo), - Some(RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true), - SRC_TMP_FILE + "-2", decompoundDstCpFileTo) - ), - replaceRules + RulesTxtWithFileName(rulesWithoutDecompounds, SRC_TMP_FILE), + Some(RulesTxtWithFileName(decompoundRules, SRC_TMP_FILE + "-2")), + optReplaceRules ) } } @@ -123,7 +89,7 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT def validateCompleteRulesTxts(rulesTxts: RulesTxtsForSolrIndex, logDebug: Boolean = true): List[String] = { val rulesValidation = rulesTxts.regularAndDecompoundFiles.flatMap { rulesFile => if (logDebug) { - logger.debug(":: validateCompleteRulesTxts for src = " + rulesFile.sourceFileName + " dst = " + rulesFile.destinationFileName) + logger.debug(":: validateCompleteRulesTxts for src = " + rulesFile.sourceFileName) logger.debug(":: rulesTxt = <<<" + rulesFile.content + ">>>") } val validationResult = querqyRulesTxtGenerator.validateQuerqyRulesTxtToErrMsg(rulesFile.content) @@ -158,43 +124,34 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT } - def interfaceSmui2SolrSh(scriptPath: String, srcTmpFile: String, dstCpFileTo: String, solrHost: String, - solrCoreName: String, decompoundDstCpFileTo: String, targetSystem: String, - replaceRulesSrcTmpFile: String, replaceRulesDstCpFileTo: String + def interfaceSmui2SolrSh(scriptPath: String, + srcTmpFile: String, + replaceRulesSrcTmpFile: String, + targetSystem: String, + solrCoreName: String ): DeploymentScriptResult = { logger.info( s""":: interfaceSmui2SolrSh |:: scriptPath = $scriptPath |:: srcTmpFile = $srcTmpFile - |:: dstCpFileTo = $dstCpFileTo - |:: solrHost = $solrHost + |:: replaceRulesSrcTmpFile = $replaceRulesSrcTmpFile |:: solrCoreName = $solrCoreName - |:: decompoundDstCpFileTo = $decompoundDstCpFileTo |:: targetSystem = $targetSystem - |:: replaceRulesSrcTmpFile = $replaceRulesSrcTmpFile - |:: replaceRulesDstCpFileTo = $replaceRulesDstCpFileTo """.stripMargin) - val scriptCall = + val scriptCall = List( // define call for regular smui2solr (default or custom script) and add parameters to the script (in expected order, see smui2solr.sh) - scriptPath + " " + + scriptPath, // SRC_TMP_FILE=$1 - srcTmpFile + " " + - // DST_CP_FILE_TO=$2 - dstCpFileTo + " " + - // SOLR_HOST=$3 - solrHost + " " + + srcTmpFile, + // REPLACE_RULES_SRC_TMP_FILE=$2 + replaceRulesSrcTmpFile, + // TARGET_SYSTEM=$3 + targetSystem, // SOLR_CORE_NAME=$4 - solrCoreName + " " + - // DECOMPOUND_DST_CP_FILE_TO=$5 - decompoundDstCpFileTo + " " + - // TARGET_SYSTEM=$6 - targetSystem + " " + - // REPLACE_RULES_SRC_TMP_FILE=$7 - replaceRulesSrcTmpFile + " " + - // REPLACE_RULES_DST_CP_FILE_TO=$8 - replaceRulesDstCpFileTo + solrCoreName + ).mkString(" ") interfaceDeploymentScript(scriptCall) } @@ -233,58 +190,37 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT else environment.rootPath.getAbsolutePath + "/conf/smui2solr.sh" + val deployToGitConfigured = featureToggleService.getToggleRuleDeploymentGitEnabled + val srcTmpFile = rulesTxts.regularRules.sourceFileName - val dstCpFileTo = rulesTxts.regularRules.destinationFileName - val decompoundDstCpFileTo = if (rulesTxts.decompoundRules.isDefined) - rulesTxts.decompoundRules.get.destinationFileName - else - "NONE" - - // host for (optional) core reload - val SMUI_DEPLOY_PRELIVE_SOLR_HOST = appConfig.get[String]("smui2solr.deploy-prelive-solr-host") - val SMUI_DEPLOY_LIVE_SOLR_HOST = appConfig.get[String]("smui2solr.SOLR_HOST") - val solrHost = if (targetSystem == "PRELIVE") - if (SMUI_DEPLOY_PRELIVE_SOLR_HOST.isEmpty) - "NONE" - else - SMUI_DEPLOY_PRELIVE_SOLR_HOST - else // targetSystem == "LIVE" - if (SMUI_DEPLOY_LIVE_SOLR_HOST.isEmpty) - "NONE" - else - SMUI_DEPLOY_LIVE_SOLR_HOST - // core name from repo (optional, for core reload as well) val solrCoreName = searchManagementRepository.getSolrIndexName(rulesTxts.solrIndexId) - val replaceRulesSrcTmpFile = rulesTxts.replaceRules.map(_.sourceFileName).getOrElse("NONE") - val replaceRulesDstCpFileTo = rulesTxts.replaceRules.map(_.destinationFileName).getOrElse("NONE") // execute script // TODO currently only git deployment for LIVE instance available - val deployToGit = targetSystem.equals("LIVE") && appConfig.get[String]("smui2solr.DST_CP_FILE_TO").equals("GIT") - val result = (if(!deployToGit) { + val result = if (deployToGitConfigured) { + if (targetSystem == "LIVE") { + logger.info(":: executeDeploymentScript :: GIT configured calling interfaceSmui2GitSh") + // TODO support further rule files (decompound / replace) and maybe solrCoreName and/or targetSystem for git branch? + interfaceSmui2GitSh( + environment.rootPath.getAbsolutePath + "/conf/smui2git.sh", + srcTmpFile, + featureToggleService.getSmuiDeploymentGitRepoUrl, + featureToggleService.getSmuiDeploymentGitFilenameCommonRulesTxt, + ) + } else { + DeploymentScriptResult(exitCode = -1, output = "Git deployment only support for LIVE target") + } + } else { logger.info(s":: executeDeploymentScript :: regular script configured calling interfaceSmui2SolrSh(scriptPath = '$scriptPath')") interfaceSmui2SolrSh( scriptPath, srcTmpFile, - dstCpFileTo, - solrHost, - solrCoreName, - decompoundDstCpFileTo, - targetSystem, replaceRulesSrcTmpFile, - replaceRulesDstCpFileTo - ) - } else { - logger.info(":: executeDeploymentScript :: GIT configured calling interfaceSmui2GitSh") - // TODO support further rule files (decompound / replace) and maybe solrCoreName and/or targetSystem for git branch? - interfaceSmui2GitSh( - environment.rootPath.getAbsolutePath + "/conf/smui2git.sh", - srcTmpFile, - featureToggleService.getSmuiDeploymentGitRepoUrl, - featureToggleService.getSmuiDeploymentGitFilenameCommonRulesTxt, + targetSystem, + solrCoreName ) - }) + } if (result.success) { logger.info(s"Rules.txt deployment successful:\n${result.output}") } else { @@ -323,7 +259,7 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT try { for (index <- searchManagementRepository.listAllSolrIndexes) { // TODO make targetSystem configurable from ApiController.downloadAllRulesTxtFiles ... go with "LIVE" from now (as there exist no different revisions of the search management content)! - val rules = generateRulesTxtContentWithFilenames(index.id, "LIVE", logDebug = false) + val rules = generateRulesTxtContentWithFilenames(index.id, logDebug = false) zipStream.putNextEntry(new ZipEntry(s"rules_${index.name}.txt")) zipStream.write(rules.regularRules.content.getBytes("UTF-8")) zipStream.closeEntry() diff --git a/build.sbt b/build.sbt index f983f6dd..fb7c2b71 100644 --- a/build.sbt +++ b/build.sbt @@ -1,10 +1,10 @@ import com.typesafe.sbt.GitBranchPrompt name := "search-management-ui" -version := "4.3.1" +version := "4.4.1" maintainer := "Contact productful.io " -scalaVersion := "2.13.14" +scalaVersion := "2.13.15" ThisBuild / evictionErrorLevel := Level.Info diff --git a/conf/application.conf b/conf/application.conf index b8065618..f584d5c1 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -31,17 +31,6 @@ db.default.password=${?SMUI_DB_PASSWORD} smui2solr.SRC_TMP_FILE="/tmp/search-management-ui_rules-txt.tmp" smui2solr.SRC_TMP_FILE=${?SMUI_2SOLR_SRC_TMP_FILE} -smui2solr.DST_CP_FILE_TO="/usr/bin/solr/liveCore/conf/rules.txt" -smui2solr.DST_CP_FILE_TO=${?SMUI_2SOLR_DST_CP_FILE_TO} - -smui2solr.SOLR_HOST="localhost:8983" -smui2solr.SOLR_HOST=${?SMUI_2SOLR_SOLR_HOST} - -smui.deployment.git.repo-url="ssh://git@localhost/git-server/repos/smui_rulestxt_repo.git" -smui.deployment.git.repo-url=${?SMUI_DEPLOYMENT_GIT_REPO_URL} -smui2solr.deployment.git.filename.common-rules-txt="rules.txt" -smui2solr.deployment.git.filename.common-rules-txt=${?SMUI_DEPLOYMENT_GIT_FN_COMMON_RULES_TXT} - # Application Feature Toggles # ~~~~~ toggle.activate-spelling=false @@ -56,30 +45,31 @@ toggle.ui-concept.all-rules.with-solr-fields=${?SMUI_TOGGLE_UI_WITH_SOLR_FIELDS} toggle.ui-list.limit-items-to=-1 # set -1 for no limitation for displayed items in list toggle.ui-list.limit-items-to=${?SMUI_TOGGLE_UI_LIMIT_LIST_ITEMS_TO} +# For rule deployment to a Git repo +smui.deployment.git.repo-url="ssh://git@localhost/git-server/repos/smui_rulestxt_repo.git" +smui.deployment.git.repo-url=${?SMUI_DEPLOYMENT_GIT_REPO_URL} +smui2solr.deployment.git.filename.common-rules-txt="rules.txt" +smui2solr.deployment.git.filename.common-rules-txt=${?SMUI_DEPLOYMENT_GIT_FN_COMMON_RULES_TXT} + +# whether the special Git repo export should happen during rule deployment +toggle.rule-deployment.git.enable=false +toggle.rule-deployment.git.enable=${?SMUI_DEPLOYMENT_GIT_ENABLE} + +# whether the SMUI generated input id should be exported as part of the rules.txt toggle.rule-deployment.log-rule-id=false toggle.rule-deployment.log-rule-id=${?SMUI_TOGGLE_RULE_ID_LOGGING} toggle.rule-deployment.split-decompound-rules-txt=false toggle.rule-deployment.split-decompound-rules-txt=${?SMUI_TOGGLE_DEPL_SPLIT_DECOMPOUND} -toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO="/usr/bin/solr/liveCore/conf/decompound-rules.txt" -toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO=${?SMUI_TOGGLE_DEPL_DECOMPOUND_DST} smui2solr.replace-rules-tmp-file="/tmp/search-management-ui_replace-rules-txt.tmp" smui2solr.replace-rules-tmp-file=${?SMUI_2SOLR_REPLACE_RULES_TMP_FILE} -smui2solr.replace-rules-dst-cp-file-to="/usr/bin/solr/liveCore/conf/replace-rules.txt" -smui2solr.replace-rules-dst-cp-file-to=${?SMUI_2SOLR_REPLACE_RULES_DST_CP_FILE_TO} toggle.rule-deployment.pre-live.present=false toggle.rule-deployment.pre-live.present=${?SMUI_TOGGLE_DEPL_PRELIVE} # in case pre-live is present, describe PRELIVE deployment -smui2solr.deploy-prelive-fn-rules-txt="/usr/bin/solr/preliveCore/conf/rules.txt" -smui2solr.deploy-prelive-fn-rules-txt=${?SMUI_DEPLOY_PRELIVE_FN_RULES_TXT} smui2solr.deploy-prelive-solr-host="" smui2solr.deploy-prelive-solr-host=${?SMUI_DEPLOY_PRELIVE_SOLR_HOST} -smui2solr.deploy-prelive-fn-decompound-txt="/usr/bin/solr/preliveCore/conf/rules-decompound.txt" -smui2solr.deploy-prelive-fn-decompound-txt=${?SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT} -smui2solr.deploy-prelive-fn-replace-txt="/usr/bin/solr/preliveCore/conf/replace-rules.txt" -smui2solr.deploy-prelive-fn-replace-txt=${?SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT} toggle.rule-deployment.custom-script=false toggle.rule-deployment.custom-script=${?SMUI_TOGGLE_DEPL_CUSTOM_SCRIPT} diff --git a/conf/smui2solr.sh b/conf/smui2solr.sh index 711d3341..7bd5f1fb 100755 --- a/conf/smui2solr.sh +++ b/conf/smui2solr.sh @@ -2,86 +2,135 @@ set -euo pipefail -SRC_TMP_FILE=$1 -DST_CP_FILE_TO=$2 -SOLR_HOST=$3 -SOLR_CORE_NAME=$4 -DECOMPOUND_DST_CP_FILE_TO=$5 -TARGET_SYSTEM=$6 -REPLACE_RULES_SRC_TMP_FILE=$7 -REPLACE_RULES_DST_CP_FILE_TO=$8 - -echo "In smui2solr.sh - script performing rules.txt update and core reload" -echo "^-- SRC_TMP_FILE = $SRC_TMP_FILE" -echo "^-- DST_CP_FILE_TO = $DST_CP_FILE_TO" -echo "^-- SOLR_HOST = $SOLR_HOST" -echo "^-- SOLR_CORE_NAME: $SOLR_CORE_NAME" -echo "^-- DECOMPOUND_DST_CP_FILE_TO = $DECOMPOUND_DST_CP_FILE_TO" -echo "^-- TARGET_SYSTEM = $TARGET_SYSTEM" -echo "^-- REPLACE_RULES_SRC_TMP_FILE = $REPLACE_RULES_SRC_TMP_FILE" -echo "^-- REPLACE_RULES_DST_CP_FILE_TO = $REPLACE_RULES_DST_CP_FILE_TO" +# Expected command line parameters +common_rules_file=$1 +decompound_rules_file="${common_rules_file}-2" +replace_rules_file=$2 +target_system=$3 +solr_core_or_collection_name=$4 -# DEPLOYMENT -##### +# Read in environment variables -echo "^-- Perform rules.txt deployment (decompound-rules.txt eventually)" - -# $1 - from_filename -# $2 - to_filename (might be local or remote) -function deploy_rules_txt { - # e.g.: remote_user:remote_pass@remote_host:/path/to/live/solr/defaultCore/conf/rules.txt - if [[ $2 =~ (.*):(.*)@(.*) ]] - then - echo "^-- ... matched remote target (regex). proceeding with remote copy ..." - sshpass -p "${BASH_REMATCH[2]}" scp -v -o StrictHostKeyChecking=no $1 ${BASH_REMATCH[1]}@${BASH_REMATCH[3]} - else - echo "^-- ... no match (regex). proceeding with regular cp ..." - cp $1 $2 - fi -} +# Querqy rewriter names for rules, (optionally when exported separately) the decompound rules and spelling/replacement rules +common_rules_rewriter_name="${SMUI_QUERQY_REWRITER_COMMON_RULES:-common_rules}" +common_rules_decompound_rewriter_name="${SMUI_QUERQY_REWRITER_COMMON_RULES_DECOMPOUND:-common_rules_decompound}" +replace_rules_rewriter_name="${SMUI_QUERQY_REWRITER_REPLACE:-replace_rules}" + +# (legacy) the live Solr host the rules should be deployed to +SMUI_2SOLR_SOLR_HOST=${SMUI_2SOLR_SOLR_HOST:-} + +# (alternatively) the live Solr host the rules should be deployed to +SMUI_DEPLOY_LIVE_SOLR_HOST=${SMUI_DEPLOY_LIVE_SOLR_HOST:-} + +# (optional, if basic authentication is used): Solr user for non-prelive deployment +SMUI_DEPLOY_LIVE_SOLR_USER=${SMUI_DEPLOY_LIVE_SOLR_USER:-} + +# (optional, if basic authentication is used): Solr password for non-prelive deployment +SMUI_DEPLOY_LIVE_SOLR_PASSWORD=${SMUI_DEPLOY_LIVE_SOLR_PASSWORD:-} -echo "^-- ... rules.txt" -deploy_rules_txt $SRC_TMP_FILE $DST_CP_FILE_TO +# (optional): whether certificate validation should be turned off when connecting to (live) Solr host +SMUI_DEPLOY_LIVE_SOLR_ALLOW_INSECURE=${SMUI_DEPLOY_LIVE_SOLR_ALLOW_INSECURE:-} -echo "^-- ... decompound-rules.txt" -if ! [[ $DECOMPOUND_DST_CP_FILE_TO == "NONE" ]] +# (optional, if PRELIVE is used as a deployment target): the prelive Solr host the rules should be deployed to +SMUI_DEPLOY_PRELIVE_SOLR_HOST=${SMUI_DEPLOY_PRELIVE_SOLR_HOST:-} + +# (optional, if basic authentication is used): Prelive Solr user +SMUI_DEPLOY_PRELIVE_SOLR_USER=${SMUI_DEPLOY_PRELIVE_SOLR_USER:-} + +# (optional, if basic authentication is used): Prelive Solr user +SMUI_DEPLOY_PRELIVE_SOLR_PASSWORD=${SMUI_DEPLOY_PRELIVE_SOLR_PASSWORD:-} + +# (optional): whether certificate validation should be turned off when connecting to prelive Solr host +SMUI_DEPLOY_PRELIVE_SOLR_ALLOW_INSECURE=${SMUI_DEPLOY_PRELIVE_SOLR_ALLOW_INSECURE:-} + +# Solr (basic) authentication +solr_host="" +solr_basic_auth_user="" +solr_basic_auth_password="" +curl_solr_insecure_flag="" + +if [ "${target_system}" == "PRELIVE" ] ; then - deploy_rules_txt "$SRC_TMP_FILE-2" $DECOMPOUND_DST_CP_FILE_TO + solr_host="${SMUI_DEPLOY_PRELIVE_SOLR_HOST:-}" + if [ -n "${SMUI_DEPLOY_PRELIVE_SOLR_USER}" ] && [ -n "${SMUI_DEPLOY_PRELIVE_SOLR_PASSWORD}" ]; then + solr_basic_auth_user=${SMUI_DEPLOY_PRELIVE_SOLR_USER} + solr_basic_auth_password=${SMUI_DEPLOY_PRELIVE_SOLR_PASSWORD} + fi + if [ "${SMUI_DEPLOY_PRELIVE_SOLR_ALLOW_INSECURE,,}" == "true" ] ; + then + curl_solr_insecure_flag="--insecure" + fi fi -echo "^-- ... replace-rules.txt" -if ! [[ $REPLACE_RULES_SRC_TMP_FILE == "NONE" && $REPLACE_RULES_DST_CP_FILE_TO == "NONE" ]] +if [ "${target_system}" == "LIVE" ] ; then - deploy_rules_txt $REPLACE_RULES_SRC_TMP_FILE $REPLACE_RULES_DST_CP_FILE_TO + solr_host="${SMUI_2SOLR_SOLR_HOST:-${SMUI_DEPLOY_LIVE_SOLR_HOST:-}}" + if [ -n "${SMUI_DEPLOY_LIVE_SOLR_USER}" ] && [ -n "${SMUI_DEPLOY_LIVE_SOLR_PASSWORD}" ]; then + solr_basic_auth_user=${SMUI_DEPLOY_LIVE_SOLR_USER} + solr_basic_auth_password=${SMUI_DEPLOY_LIVE_SOLR_PASSWORD} + fi + if [ "${SMUI_DEPLOY_LIVE_SOLR_ALLOW_INSECURE,,}" == "true" ] ; + then + curl_solr_insecure_flag="--insecure" + fi fi -# CORE RELOAD +save_rewriter() { + local querqy_rewriter_name="$1" + local querqy_rewriter_data="$2" + + # read into array + IFS=' ' read -r -a curl_command <<< "curl -X POST -H \"Content-Type: application/json\" --fail" + + # add payload + curl_command+=("-d" "${querqy_rewriter_data}") + + # optional basic auth username + password and curl --insecure flag + if [ -n "${solr_basic_auth_user}" ] && [ -n "${solr_basic_auth_password}" ]; then + curl_command+=("--user" "${solr_basic_auth_user}:${solr_basic_auth_password}") + fi + if [ -n "${curl_solr_insecure_flag:-}" ]; then + curl_command+=("${curl_solr_insecure_flag}") + fi + + # add url + curl_command+=("${solr_host}/solr/${solr_core_or_collection_name}/querqy/rewriter/${querqy_rewriter_name}?action=save") + + echo "${curl_command[@]}" + + "${curl_command[@]}" +} + +echo "In smui2solr.sh - script for updating rules in Solr using an API call to the Querqy (>=v5) plugin " +echo "^-- common_rules_file = ${common_rules_file}" +echo "^-- decompound_rules_file = ${common_rules_file}-2" +echo "^-- solr_host = ${solr_host}" +echo "^-- target_system: ${target_system}" +echo "^-- solr_core_or_collection_name = ${solr_core_or_collection_name}" + +# DEPLOYMENT ##### -echo "^-- Perform core reload for SOLR_HOST = $SOLR_HOST, SOLR_CORE_NAME = $SOLR_CORE_NAME" +# common rules +if [ -s "${common_rules_file}" ] && [ -n "${common_rules_rewriter_name}" ]; then + # build the rewriter request payload by embedding the rules text into JSON + common_rules_rewriter_data=$(jq -Rs '{class: "querqy.solr.rewriter.commonrules.CommonRulesRewriterFactory", config: { rules: . }}' "${common_rules_file}") + echo "Deploying common rules for core ${solr_core_or_collection_name} to rewriter ${common_rules_rewriter_name} at ${solr_host}..." + save_rewriter "${common_rules_rewriter_name}" "${common_rules_rewriter_data}" +fi + +# optional: decompound rules to a separate common rules rewriter +if [ -s "${decompound_rules_file}" ] && [ -n "${common_rules_decompound_rewriter_name}" ]; then + common_rules_decompound_rewriter_data=$(jq -Rs '{class: "querqy.solr.rewriter.commonrules.CommonRulesRewriterFactory", config: { rules: . }}' "${decompound_rules_file}") + echo "Deploying decompound common rules for core ${solr_core_or_collection_name} to rewriter ${common_rules_decompound_rewriter_name} at ${solr_host}..." + save_rewriter "${common_rules_decompound_rewriter_name}" "${common_rules_decompound_rewriter_data}" +fi -if ! [[ $SOLR_HOST == "NONE" ]] -then - # TODO only core reload over http possible. make configurable. - SOLR_STATUS=$(curl -s -i -XGET "http://$SOLR_HOST/solr/admin/cores?wt=xml&action=RELOAD&core=$SOLR_CORE_NAME") - - if [ $? -ne 0 ]; then - exit 16 - fi - - if ! [[ $SOLR_STATUS == *"200 OK"* ]] - then - >&2 echo "Error reloading Solr core: $SOLR_STATUS" - exit 17 - fi - - if ! [[ $SOLR_STATUS == *"0"* ]] - then - >&2 echo "Error reloading Solr core: $SOLR_STATUS" - exit 18 - fi +# deploy replace / spelling correction rules to a separate replace rewriter if they exist +if [ -s "${replace_rules_file}" ] && [ -n "${replace_rules_rewriter_name}" ]; then + replace_rules_rewriter_data=$(jq -Rs '{class: "querqy.solr.rewriter.replace.ReplaceRewriterFactory", config: { ignoreCase: true, inputDelimiter: ";", rules: . }}' "${replace_rules_file}") + echo "Deploying replace rules for core ${solr_core_or_collection_name} to rewriter ${replace_rules_rewriter_name} at ${solr_host}..." + save_rewriter "${replace_rules_rewriter_name}" "${replace_rules_rewriter_data}" fi -# all ok -echo "smui2solr.sh - ok" -exit 0 +echo "Done" diff --git a/test/resources/smui2test.sh b/test/resources/smui2test.sh index f11c799c..881f29fb 100755 --- a/test/resources/smui2test.sh +++ b/test/resources/smui2test.sh @@ -8,7 +8,3 @@ echo "\$1 = >>>$1" echo "\$2 = >>>$2" echo "\$3 = >>>$3" echo "\$4 = >>>$4" -echo "\$5 = >>>$5" -echo "\$6 = >>>$6" -echo "\$7 = >>>$7" -echo "\$8 = >>>$8" diff --git a/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala b/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala index dc7fa1a7..a927c016 100644 --- a/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala +++ b/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala @@ -63,33 +63,18 @@ class RulesTxtOnlyDeploymentConfigVariantSpec extends AnyFlatSpec with Matchers override protected lazy val additionalAppConfig = Seq( "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", - "smui2solr.DST_CP_FILE_TO" -> "/deployment-path-live/common-rules.txt", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt" ) - "RulesTxtDeploymentService" should "provide only the (common) rules.txt for PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) + "RulesTxtDeploymentService" should "provide only the (common) rules.txt" in { + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) deploymentDescriptor.solrIndexId shouldBe core1Id deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-prelive/common-rules.txt" deploymentDescriptor.replaceRules shouldBe None deploymentDescriptor.decompoundRules shouldBe None } - - "RulesTxtDeploymentService" should "provide only the (common) rules.txt for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) - - deploymentDescriptor.solrIndexId shouldBe core1Id - deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content - deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-live/common-rules.txt" - deploymentDescriptor.replaceRules shouldBe None - deploymentDescriptor.decompoundRules shouldBe None - } - } class RulesAndReplaceTxtDeploymentConfigVariantSpec extends AnyFlatSpec with Matchers with CommonRulesTxtDeploymentServiceConfigVariantsSpecBase { @@ -98,42 +83,21 @@ class RulesAndReplaceTxtDeploymentConfigVariantSpec extends AnyFlatSpec with Mat override protected lazy val additionalAppConfig = Seq( "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", - "smui2solr.DST_CP_FILE_TO" -> "/deployment-path-live/common-rules.txt", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt", // spelling is activated (@see /smui/test/models/ApplicationTestBase.scala) - "smui2solr.replace-rules-tmp-file" -> "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp", - "smui2solr.replace-rules-dst-cp-file-to" -> "/deployment-path-live/replace-rules.txt", - "smui2solr.deploy-prelive-fn-replace-txt" -> "/deployment-path-prelive/replace-rules.txt" + "smui2solr.replace-rules-tmp-file" -> "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp" ) - "RulesTxtDeploymentService" should "provide the (common) rules.txt and a replace-rules.txt for PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) + "RulesTxtDeploymentService" should "provide the (common) rules.txt and a replace-rules.txt" in { + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) deploymentDescriptor.solrIndexId shouldBe core1Id deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-prelive/common-rules.txt" val replaceRules = deploymentDescriptor.replaceRules.get replaceRules.content should include ("freezer") // simply cross check content replaceRules.sourceFileName shouldBe "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp" - replaceRules.destinationFileName shouldBe "/deployment-path-prelive/replace-rules.txt" - deploymentDescriptor.decompoundRules shouldBe None - } - - "RulesTxtDeploymentService" should "provide the (common) rules.txt and a replace-rules.txt for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) - - deploymentDescriptor.solrIndexId shouldBe core1Id - deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content - deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-live/common-rules.txt" - - val replaceRules = deploymentDescriptor.replaceRules.get - replaceRules.content should include ("freezer") // simply cross check content - replaceRules.sourceFileName shouldBe "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp" - replaceRules.destinationFileName shouldBe "/deployment-path-live/replace-rules.txt" deploymentDescriptor.decompoundRules shouldBe None } @@ -145,12 +109,8 @@ class RulesAndDecompoundTxtDeploymentConfigVariantSpec extends AnyFlatSpec with override protected lazy val additionalAppConfig = Seq( "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", - "smui2solr.DST_CP_FILE_TO" -> "/deployment-path-live/common-rules.txt", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt", - "toggle.rule-deployment.split-decompound-rules-txt" -> true, - "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO" -> "/deployment-path-live/decompound-rules.txt", - "smui2solr.deploy-prelive-fn-decompound-txt" -> "/deployment-path-prelive/decompound-rules.txt" + "toggle.rule-deployment.split-decompound-rules-txt" -> true ) override protected def beforeAll(): Unit = { @@ -158,34 +118,17 @@ class RulesAndDecompoundTxtDeploymentConfigVariantSpec extends AnyFlatSpec with createDecompoundRule() } - "RulesTxtDeploymentService" should "provide the (common) rules.txt and a decompound-rules.txt for PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) + "RulesTxtDeploymentService" should "provide the (common) rules.txt and a decompound-rules.txt" in { + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) deploymentDescriptor.solrIndexId shouldBe core1Id deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-prelive/common-rules.txt" deploymentDescriptor.replaceRules shouldBe None val decompoundRules = deploymentDescriptor.decompoundRules.get decompoundRules.content should include ("damen") decompoundRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp-2" // auto generated (by spec) - decompoundRules.destinationFileName shouldBe "/deployment-path-prelive/decompound-rules.txt" - } - - "RulesTxtDeploymentService" should "provide the (common) rules.txt and a decompound-rules.txt for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) - - deploymentDescriptor.solrIndexId shouldBe core1Id - deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content - deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-live/common-rules.txt" - deploymentDescriptor.replaceRules shouldBe None - - val decompoundRules = deploymentDescriptor.decompoundRules.get - decompoundRules.content should include ("damen") - decompoundRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp-2" // auto generated (by spec) - decompoundRules.destinationFileName shouldBe "/deployment-path-live/decompound-rules.txt" } } @@ -197,17 +140,11 @@ class RulesReplaceAndDecompoundTxtDeploymentConfigVariantSpec extends AnyFlatSpe override protected lazy val additionalAppConfig = Seq( // (common) rules.txt config "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", - "smui2solr.DST_CP_FILE_TO" -> "/deployment-path-live/common-rules.txt", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt", // replace-rules.txt config "smui2solr.replace-rules-tmp-file" -> "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp", - "smui2solr.replace-rules-dst-cp-file-to" -> "/deployment-path-live/replace-rules.txt", - "smui2solr.deploy-prelive-fn-replace-txt" -> "/deployment-path-prelive/replace-rules.txt", // decompound-rules.txt config - "toggle.rule-deployment.split-decompound-rules-txt" -> true, - "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO" -> "/deployment-path-live/decompound-rules.txt", - "smui2solr.deploy-prelive-fn-decompound-txt" -> "/deployment-path-prelive/decompound-rules.txt" + "toggle.rule-deployment.split-decompound-rules-txt" -> true ) override protected def beforeAll(): Unit = { @@ -215,42 +152,20 @@ class RulesReplaceAndDecompoundTxtDeploymentConfigVariantSpec extends AnyFlatSpe createDecompoundRule() } - "RulesTxtDeploymentService" should "provide the (common) rules.txt, replace-rules.txt and a decompound-rules.txt for PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) - - deploymentDescriptor.solrIndexId shouldBe core1Id - deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content - deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-prelive/common-rules.txt" - - val replaceRules = deploymentDescriptor.replaceRules.get - replaceRules.content should include ("freezer") // simply cross check content - replaceRules.sourceFileName shouldBe "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp" - replaceRules.destinationFileName shouldBe "/deployment-path-prelive/replace-rules.txt" - - val decompoundRules = deploymentDescriptor.decompoundRules.get - decompoundRules.content should include ("damen") - decompoundRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp-2" // auto generated (by spec) - decompoundRules.destinationFileName shouldBe "/deployment-path-prelive/decompound-rules.txt" - } - - "RulesTxtDeploymentService" should "provide the (common) rules.txt, replace-rules.txt and a decompound-rules.txt for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) + "RulesTxtDeploymentService" should "provide the (common) rules.txt, replace-rules.txt and a decompound-rules.txt" in { + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) deploymentDescriptor.solrIndexId shouldBe core1Id deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-live/common-rules.txt" val replaceRules = deploymentDescriptor.replaceRules.get replaceRules.content should include ("freezer") // simply cross check content replaceRules.sourceFileName shouldBe "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp" - replaceRules.destinationFileName shouldBe "/deployment-path-live/replace-rules.txt" val decompoundRules = deploymentDescriptor.decompoundRules.get decompoundRules.content should include ("damen") decompoundRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp-2" // auto generated (by spec) - decompoundRules.destinationFileName shouldBe "/deployment-path-live/decompound-rules.txt" } } @@ -266,20 +181,11 @@ class RulesTxtDeploymentRegularTargetSpec extends AnyFlatSpec with Matchers with override protected lazy val additionalAppConfig = Seq( // (common) rules.txt config "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", - "smui2solr.DST_CP_FILE_TO" -> "/deployment-path-live/common-rules.txt", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt", // replace-rules.txt config "smui2solr.replace-rules-tmp-file" -> "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp", - "smui2solr.replace-rules-dst-cp-file-to" -> "/deployment-path-live/replace-rules.txt", - "smui2solr.deploy-prelive-fn-replace-txt" -> "/deployment-path-prelive/replace-rules.txt", // decompound-rules.txt config "toggle.rule-deployment.split-decompound-rules-txt" -> true, - "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO" -> "/deployment-path-live/decompound-rules.txt", - "smui2solr.deploy-prelive-fn-decompound-txt" -> "/deployment-path-prelive/decompound-rules.txt", - // Solr host config complete - "smui2solr.SOLR_HOST" -> "live.solr.instance:8983", - "smui2solr.deploy-prelive-solr-host" -> "prelive.solr.instance:8983", // test script "toggle.rule-deployment.custom-script" -> true, "toggle.rule-deployment.custom-script-SMUI2SOLR-SH_PATH" -> "test/resources/smui2test.sh" @@ -291,32 +197,24 @@ class RulesTxtDeploymentRegularTargetSpec extends AnyFlatSpec with Matchers with } "interfaceSmui2SolrSh" should "interface the test script should return all rules.txts for PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) val res = service.executeDeploymentScript(deploymentDescriptor, "PRELIVE") res.success shouldBe true res.output shouldBe s"""$$1 = >>>/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp - |$$2 = >>>/deployment-path-prelive/common-rules.txt - |$$3 = >>>prelive.solr.instance:8983 + |$$2 = >>>/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp + |$$3 = >>>PRELIVE |$$4 = >>>core1 - |$$5 = >>>/deployment-path-prelive/decompound-rules.txt - |$$6 = >>>PRELIVE - |$$7 = >>>/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp - |$$8 = >>>/deployment-path-prelive/replace-rules.txt |""".stripMargin } "interfaceSmui2SolrSh" should "interface the test script should return all rules.txts for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) val res = service.executeDeploymentScript(deploymentDescriptor, "LIVE") res.success shouldBe true res.output shouldBe s"""$$1 = >>>/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp - |$$2 = >>>/deployment-path-live/common-rules.txt - |$$3 = >>>live.solr.instance:8983 + |$$2 = >>>/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp + |$$3 = >>>LIVE |$$4 = >>>core1 - |$$5 = >>>/deployment-path-live/decompound-rules.txt - |$$6 = >>>LIVE - |$$7 = >>>/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp - |$$8 = >>>/deployment-path-live/replace-rules.txt |""".stripMargin } @@ -328,24 +226,16 @@ class RulesTxtDeploymentGitTargetSpec extends AnyFlatSpec with Matchers with Com override protected lazy val additionalAppConfig = Seq( // switch to GIT for LIVE deployment - "smui2solr.DST_CP_FILE_TO" -> "GIT", + "toggle.rule-deployment.git.enable" -> true, "smui.deployment.git.repo-url" -> "ssh://git@changed-git-server.tld/repos/smui_rulestxt_repo.git", "smui2solr.deployment.git.filename.common-rules-txt" -> "common-rules.txt", // (common) rules.txt config "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt", // replace-rules.txt config "smui2solr.replace-rules-tmp-file" -> "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp", - "smui2solr.replace-rules-dst-cp-file-to" -> "/deployment-path-live/replace-rules.txt", - "smui2solr.deploy-prelive-fn-replace-txt" -> "/deployment-path-prelive/replace-rules.txt", // decompound-rules.txt config - "toggle.rule-deployment.split-decompound-rules-txt" -> true, - "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO" -> "/deployment-path-live/decompound-rules.txt", - "smui2solr.deploy-prelive-fn-decompound-txt" -> "/deployment-path-prelive/decompound-rules.txt", - // Solr host config complete - "smui2solr.SOLR_HOST" -> "live.solr.instance:8983", - "smui2solr.deploy-prelive-solr-host" -> "prelive.solr.instance:8983" + "toggle.rule-deployment.split-decompound-rules-txt" -> true ) override protected def beforeAll(): Unit = { @@ -358,7 +248,7 @@ class RulesTxtDeploymentGitTargetSpec extends AnyFlatSpec with Matchers with Com "interfaceSmui2GitSh" should "interface the test script should return all rules.txts for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) val res = service.executeDeploymentScript(deploymentDescriptor, "LIVE") // TODO script result itself failed - this is cheesy, but cristal clear, as we don't have a local git server running or a test script instead @@ -375,14 +265,13 @@ class RulesTxtDeploymentGitTargetSpec extends AnyFlatSpec with Matchers with Com "interfacing git configured SMUI" should "stick with file copy deployment configuration PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) val res = service.executeDeploymentScript(deploymentDescriptor, "PRELIVE") // TODO script result itself failed - this is cheesy, but cristal clear, as we don't have a local git server running or a test script instead - res.output should include ("/deployment-path-prelive/common-rules.txt") - res.output should include ("/deployment-path-prelive/replace-rules.txt") - res.output should include ("/deployment-path-prelive/decompound-rules.txt") + res.exitCode shouldBe -1 + res.output shouldBe "Git deployment only support for LIVE target" // TODO As of v3.11.7 there is no option: // TODO ... to deploy to different git hosts / repos / branches (it all makes sense) diff --git a/test/services/RulesTxtDeploymentServiceSimpleSpec.scala b/test/services/RulesTxtDeploymentServiceSimpleSpec.scala index 606d4ee8..5886b441 100644 --- a/test/services/RulesTxtDeploymentServiceSimpleSpec.scala +++ b/test/services/RulesTxtDeploymentServiceSimpleSpec.scala @@ -53,21 +53,19 @@ class RulesTxtDeploymentServiceSpec extends AnyFlatSpec with Matchers with Appli |mechine => machine""".stripMargin "RulesTxtDeploymentService" should "generate rules files with correct file names" in { - val rulesTxt = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) + val rulesTxt = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) rulesTxt.solrIndexId shouldBe core1Id rulesTxt.decompoundRules shouldBe empty rulesTxt.regularRules.content.trim shouldBe rulesFileContent(inputIds) rulesTxt.regularRules.sourceFileName shouldBe "/tmp/search-management-ui_rules-txt.tmp" - rulesTxt.regularRules.destinationFileName shouldBe "/usr/bin/solr/liveCore/conf/rules.txt" rulesTxt.replaceRules.get.content.trim shouldBe replaceRulesFileContent() rulesTxt.replaceRules.get.sourceFileName shouldBe "/tmp/search-management-ui_replace-rules-txt.tmp" - rulesTxt.replaceRules.get.destinationFileName shouldBe "/usr/bin/solr/liveCore/conf/replace-rules.txt" } it should "validate the rules files correctly" in { - val rulesTxt = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) + val rulesTxt = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) service.validateCompleteRulesTxts(rulesTxt, logDebug = false) shouldBe empty val badRulesTxt = rulesTxt.copy(regularRules = rulesTxt.regularRules.copy(content = "a very bad rules file"))