Merge branch 'opensearch-project:main' into changelog-verifier

.ci/bwcVersions

@@ -48,3 +48,5 @@ BWC_VERSION:
   - "2.1.0"
   - "2.1.1"
   - "2.2.0"
+  - "2.2.1"
+  - "2.3.0"

.github/workflows/auto-release.yml

@@ -3,7 +3,7 @@ name: Releases
 on:
   push:
     tags:
-      - '*.*.*'
+      - '*'
 
 jobs:
 
@@ -12,11 +12,18 @@ jobs:
     permissions:
       contents: write
     steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/github-app-token@v1.5.0
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1
       - uses: actions/checkout@v2
       - uses: ncipollo/release-action@v1
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          github_token: ${{ steps.github_app_token.outputs.token }}
           bodyFile: release-notes/opensearch.release-notes-${{steps.tag.outputs.tag}}.md

.github/workflows/gradle-check.yml

@@ -2,9 +2,9 @@ name: Gradle Check (Jenkins)
 on:
   push:
     branches-ignore:
-      - 'backport/*'
-      - 'create-pull-request/*'
-      - 'dependabot/*'
+      - 'backport/**'
+      - 'create-pull-request/**'
+      - 'dependabot/**'
   pull_request_target:
     types: [opened, synchronize, reopened]
 

DEVELOPER_GUIDE.md

@@ -4,14 +4,16 @@
     - [Install Prerequisites](#install-prerequisites)
       - [JDK 11](#jdk-11)
       - [JDK 14](#jdk-14)
-      - [Runtime JDK](#runtime-jdk)
+      - [JDK 17](#jdk-17)
+      - [Custom Runtime JDK](#custom-runtime-jdk)
       - [Windows](#windows)
       - [Docker](#docker)
     - [Build](#build)
     - [Run Tests](#run-tests)
     - [Run OpenSearch](#run-opensearch)
   - [Use an Editor](#use-an-editor)
     - [IntelliJ IDEA](#intellij-idea)
+      - [Remote development using JetBrains Gateway](#remote-development-using-jetbrains-gateway)
     - [Visual Studio Code](#visual-studio-code)
     - [Eclipse](#eclipse)
   - [Project Layout](#project-layout)
@@ -35,6 +37,7 @@
       - [testImplementation](#testimplementation)
     - [Gradle Plugins](#gradle-plugins)
       - [Distribution Download Plugin](#distribution-download-plugin)
+    - [Creating fat-JAR of a Module](#creating-fat-jar-of-a-module)
   - [Misc](#misc)
     - [git-secrets](#git-secrets)
       - [Installation](#installation)
@@ -49,7 +52,7 @@
   - [Submitting Changes](#submitting-changes)
   - [Backports](#backports)
   - [LineLint](#linelint)
-  - [Lucene Snapshots](#lucene-snapshots)
+- [Lucene Snapshots](#lucene-snapshots)
 
 # Developer Guide
 
@@ -374,6 +377,42 @@ The Distribution Download plugin downloads the latest version of OpenSearch by d
 ./gradlew integTest -PcustomDistributionUrl="https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/1127/linux/x64/dist/opensearch-1.2.0-linux-x64.tar.gz"
 ```
 
+### Creating fat-JAR of a Module
+
+A fat-JAR (or an uber-JAR) is the JAR, which contains classes from all the libraries, on which your project depends and, of course, the classes of current project.
+
+There might be cases where a developer would like to add some custom logic to the code of a module (or multiple modules) and generate a fat-JAR that can be directly used by the dependency management tool. For example, in [#3665](https://github.com/opensearch-project/OpenSearch/pull/3665) a developer wanted to provide a tentative patch as a fat-JAR to a consumer for changes made in the high level REST client. 
+
+Use [Gradle Shadow plugin](https://imperceptiblethoughts.com/shadow/).
+Add the following to the `build.gradle` file of the module for which you want to create the fat-JAR, e.g. `client/rest-high-level/build.gradle`:
+
+```
+apply plugin: 'com.github.johnrengelman.shadow'
+```
+
+Run the `shadowJar` command using:
+```
+./gradlew :client:rest-high-level:shadowJar
+```
+
+This will generate a fat-JAR in the `build/distributions` folder of the module, e.g. .`/client/rest-high-level/build/distributions/opensearch-rest-high-level-client-1.4.0-SNAPSHOT.jar`.
+
+You can further customize your fat-JAR by customising the plugin, More information about shadow plugin can be found [here](https://imperceptiblethoughts.com/shadow/).
+
+To use the generated JAR, install the JAR locally, e.g. 
+```
+mvn install:install-file -Dfile=src/main/resources/opensearch-rest-high-level-client-1.4.0-SNAPSHOT.jar -DgroupId=org.opensearch.client -DartifactId=opensearch-rest-high-level-client -Dversion=1.4.0-SNAPSHOT -Dpackaging=jar -DgeneratePom=true
+```
+
+Refer the installed JAR as any other maven artifact, e.g.
+
+```
+<dependency>
+    <groupId>org.opensearch.client</groupId>
+    <artifactId>opensearch-rest-high-level-client</artifactId>
+    <version>1.4.0-SNAPSHOT</version>
+</dependency>
+```
 
 ## Misc
 

build.gradle

@@ -55,7 +55,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.9.0" apply false
+  id "com.diffplug.spotless" version "6.9.1" apply false
   id "org.gradle.test-retry" version "1.4.0" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -75,9 +75,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "11.0.15+10";
+    private static final String SYSTEM_JDK_VERSION = "11.0.16+8";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "17.0.3+7";
+    private static final String GRADLE_JDK_VERSION = "17.0.4+8";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchCluster.java

@@ -84,6 +84,8 @@ public class OpenSearchCluster implements TestClusterConfiguration, Named {
     private final ArchiveOperations archiveOperations;
     private int nodeIndex = 0;
 
+    private int zoneCount = 1;
+
     public OpenSearchCluster(
         String clusterName,
         Project project,
@@ -104,13 +106,21 @@ public OpenSearchCluster(
         this.bwcJdk = bwcJdk;
 
         // Always add the first node
-        addNode(clusterName + "-0");
+        String zone = hasZoneProperty() ? "zone-1" : "";
+        addNode(clusterName + "-0", zone);
         // configure the cluster name eagerly so all nodes know about it
         this.nodes.all((node) -> node.defaultConfig.put("cluster.name", safeName(clusterName)));
 
         addWaitForClusterHealth();
     }
 
+    public void setNumberOfZones(int zoneCount) {
+        if (zoneCount < 1) {
+            throw new IllegalArgumentException("Number of zones should be >= 1 but was " + zoneCount + " for " + this);
+        }
+        this.zoneCount = zoneCount;
+    }
+
     public void setNumberOfNodes(int numberOfNodes) {
         checkFrozen();
 
@@ -124,12 +134,31 @@ public void setNumberOfNodes(int numberOfNodes) {
             );
         }
 
-        for (int i = nodes.size(); i < numberOfNodes; i++) {
-            addNode(clusterName + "-" + i);
+        if (numberOfNodes < zoneCount) {
+            throw new IllegalArgumentException(
+                "Number of nodes should be >= zoneCount but was " + numberOfNodes + " for " + this.zoneCount
+            );
         }
+
+        if (hasZoneProperty()) {
+            int currentZone;
+            for (int i = nodes.size(); i < numberOfNodes; i++) {
+                currentZone = i % zoneCount + 1;
+                String zoneName = "zone-" + currentZone;
+                addNode(clusterName + "-" + i, zoneName);
+            }
+        } else {
+            for (int i = nodes.size(); i < numberOfNodes; i++) {
+                addNode(clusterName + "-" + i, "");
+            }
+        }
+    }
+
+    private boolean hasZoneProperty() {
+        return this.project.findProperty("numZones") != null;
     }
 
-    private void addNode(String nodeName) {
+    private void addNode(String nodeName, String zoneName) {
         OpenSearchNode newNode = new OpenSearchNode(
             path,
             nodeName,
@@ -138,7 +167,8 @@ private void addNode(String nodeName) {
             fileSystemOperations,
             archiveOperations,
             workingDirBase,
-            bwcJdk
+            bwcJdk,
+            zoneName
         );
         // configure the cluster name eagerly
         newNode.defaultConfig.put("cluster.name", safeName(clusterName));

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java

@@ -32,6 +32,7 @@
 package org.opensearch.gradle.testclusters;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
 import org.opensearch.gradle.Architecture;
 import org.opensearch.gradle.DistributionDownloadPlugin;
 import org.opensearch.gradle.OpenSearchDistribution;
@@ -175,6 +176,8 @@ public class OpenSearchNode implements TestClusterConfiguration {
     private final Config legacyESConfig;
     private Config currentConfig;
 
+    private String zone;
+
     OpenSearchNode(
         String path,
         String name,
@@ -183,7 +186,8 @@ public class OpenSearchNode implements TestClusterConfiguration {
         FileSystemOperations fileSystemOperations,
         ArchiveOperations archiveOperations,
         File workingDirBase,
-        Jdk bwcJdk
+        Jdk bwcJdk,
+        String zone
     ) {
         this.path = path;
         this.name = name;
@@ -205,6 +209,7 @@ public class OpenSearchNode implements TestClusterConfiguration {
         opensearchConfig = Config.getOpenSearchConfig(workingDir);
         legacyESConfig = Config.getLegacyESConfig(workingDir);
         currentConfig = opensearchConfig;
+        this.zone = zone;
     }
 
     /*
@@ -1239,6 +1244,10 @@ private void createConfiguration() {
         baseConfig.put("path.logs", confPathLogs.toAbsolutePath().toString());
         baseConfig.put("path.shared_data", workingDir.resolve("sharedData").toString());
         baseConfig.put("node.attr.testattr", "test");
+        if (StringUtils.isNotBlank(zone)) {
+            baseConfig.put("cluster.routing.allocation.awareness.attributes", "zone");
+            baseConfig.put("node.attr.zone", zone);
+        }
         baseConfig.put("node.portsfile", "true");
         baseConfig.put("http.port", httpPort);
         if (getVersion().onOrAfter(Version.fromString("6.7.0"))) {

buildSrc/version.properties

@@ -1,8 +1,8 @@
 opensearch        = 3.0.0
-lucene            = 9.3.0
+lucene            = 9.4.0-snapshot-ddf0d0a
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 17.0.3+7
+bundled_jdk = 17.0.4+8
 
 
 

client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java

@@ -157,7 +157,6 @@
 import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder;
-import org.opensearch.search.aggregations.metrics.GeoBoundsAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.InternalHDRPercentileRanks;
 import org.opensearch.search.aggregations.metrics.InternalHDRPercentiles;
@@ -169,7 +168,6 @@
 import org.opensearch.search.aggregations.metrics.ParsedAvg;
 import org.opensearch.search.aggregations.metrics.ParsedCardinality;
 import org.opensearch.search.aggregations.metrics.ParsedExtendedStats;
-import org.opensearch.search.aggregations.metrics.ParsedGeoBounds;
 import org.opensearch.search.aggregations.metrics.ParsedGeoCentroid;
 import org.opensearch.search.aggregations.metrics.ParsedHDRPercentileRanks;
 import org.opensearch.search.aggregations.metrics.ParsedHDRPercentiles;
@@ -2116,7 +2114,6 @@ static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
         map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c));
         map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c));
         map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c));
-        map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c));
         map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c));
         map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c));
         map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c));

client/rest-high-level/src/test/java/org/opensearch/client/RestHighLevelClientTests.java

@@ -885,7 +885,8 @@ public void testApiNamingConventions() throws Exception {
             "nodes.hot_threads",
             "nodes.usage",
             "nodes.reload_secure_settings",
-            "search_shards", };
+            "search_shards",
+            "remote_store.restore", };
         List<String> booleanReturnMethods = Arrays.asList("security.enable_user", "security.disable_user", "security.change_password");
         Set<String> deprecatedMethods = new HashSet<>();
         deprecatedMethods.add("indices.force_merge");

distribution/docker/src/docker/config/log4j2.properties

@@ -53,3 +53,13 @@ logger.index_indexing_slowlog.name = index.indexing.slowlog.index
 logger.index_indexing_slowlog.level = trace
 logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
 logger.index_indexing_slowlog.additivity = false
+
+appender.task_detailslog_rolling.type = Console
+appender.task_detailslog_rolling.name = task_detailslog_rolling
+appender.task_detailslog_rolling.layout.type = OpenSearchJsonLayout
+appender.task_detailslog_rolling.layout.type_name = task_detailslog
+
+logger.task_detailslog_rolling.name = task.detailslog
+logger.task_detailslog_rolling.level = trace
+logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling.ref = task_detailslog_rolling
+logger.task_detailslog_rolling.additivity = false

distribution/src/config/log4j2.properties

@@ -195,3 +195,40 @@ logger.index_indexing_slowlog.level = trace
 logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
 logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old
 logger.index_indexing_slowlog.additivity = false
+
+######## Task details log JSON ####################
+appender.task_detailslog_rolling.type = RollingFile
+appender.task_detailslog_rolling.name = task_detailslog_rolling
+appender.task_detailslog_rolling.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog.json
+appender.task_detailslog_rolling.filePermissions = rw-r-----
+appender.task_detailslog_rolling.layout.type = OpenSearchJsonLayout
+appender.task_detailslog_rolling.layout.type_name = task_detailslog
+appender.task_detailslog_rolling.layout.opensearchmessagefields=taskId,type,action,description,start_time_millis,resource_stats,metadata
+
+appender.task_detailslog_rolling.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog-%i.json.gz
+appender.task_detailslog_rolling.policies.type = Policies
+appender.task_detailslog_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.task_detailslog_rolling.policies.size.size = 1GB
+appender.task_detailslog_rolling.strategy.type = DefaultRolloverStrategy
+appender.task_detailslog_rolling.strategy.max = 4
+#################################################
+######## Task details log -  old style pattern ####
+appender.task_detailslog_rolling_old.type = RollingFile
+appender.task_detailslog_rolling_old.name = task_detailslog_rolling_old
+appender.task_detailslog_rolling_old.fileName = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog.log
+appender.task_detailslog_rolling_old.filePermissions = rw-r-----
+appender.task_detailslog_rolling_old.layout.type = PatternLayout
+appender.task_detailslog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
+
+appender.task_detailslog_rolling_old.filePattern = ${sys:opensearch.logs.base_path}${sys:file.separator}${sys:opensearch.logs.cluster_name}_task_detailslog-%i.log.gz
+appender.task_detailslog_rolling_old.policies.type = Policies
+appender.task_detailslog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy
+appender.task_detailslog_rolling_old.policies.size.size = 1GB
+appender.task_detailslog_rolling_old.strategy.type = DefaultRolloverStrategy
+appender.task_detailslog_rolling_old.strategy.max = 4
+#################################################
+logger.task_detailslog_rolling.name = task.detailslog
+logger.task_detailslog_rolling.level = trace
+logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling.ref = task_detailslog_rolling
+logger.task_detailslog_rolling.appenderRef.task_detailslog_rolling_old.ref = task_detailslog_rolling_old
+logger.task_detailslog_rolling.additivity = false

distribution/tools/upgrade-cli/src/test/java/org/opensearch/upgrade/ImportLog4jPropertiesTaskTests.java

@@ -67,7 +67,7 @@ public void testImportLog4jPropertiesTask() throws IOException {
         Properties properties = new Properties();
         properties.load(Files.newInputStream(taskInput.getOpenSearchConfig().resolve(ImportLog4jPropertiesTask.LOG4J_PROPERTIES)));
         assertThat(properties, is(notNullValue()));
-        assertThat(properties.entrySet(), hasSize(137));
+        assertThat(properties.entrySet(), hasSize(165));
         assertThat(properties.get("appender.rolling.layout.type"), equalTo("OpenSearchJsonLayout"));
         assertThat(
             properties.get("appender.deprecation_rolling.fileName"),