[pull] master from elastic:master

.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots.yml

@@ -25,8 +25,6 @@
           black-list-labels:
             - '>test-mute'
             - 'test-full-bwc'
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     axes:
       - axis:
           type: slave

.ci/jobs.t/elastic+elasticsearch+pull-request+docs-check.yml

@@ -22,8 +22,6 @@
             - ^docs/.*
           black-list-labels:
             - '>test-mute'
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+pull-request+eql-correctness.yml

@@ -20,7 +20,6 @@
           cancel-builds-on-update: true
           black-list-target-branches:
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+example-plugins.yml

@@ -23,8 +23,6 @@
             - build-tools/.*
             - build-tools-internal/.*
             - plugins/examples/.*
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-unix-sample.yml

@@ -21,7 +21,6 @@
           cancel-builds-on-update: true
           black-list-target-branches:
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml

@@ -24,7 +24,6 @@
           black-list-target-branches:
             - 7.17
             - 7.16
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-upgrade-tests.yml

@@ -22,7 +22,6 @@
           cancel-builds-on-update: true
           black-list-target-branches:
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           white-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+part-3.yml

@@ -1,7 +1,7 @@
 ---
 - job:
-    name: "elastic+elasticsearch+pull-request+par-3"
-    display-name: "elastic / elasticsearch - pull request part 3"
+    name: "elastic+elasticsearch+pull-request+part-3"
+    display-name: "elastic / elasticsearch - pull request part-3"
     description: "Testing of Elasticsearch pull requests - part 3"
     workspace: "/dev/shm/elastic+elasticsearch+pull-request+part-3"
     scm:

.ci/jobs.t/elastic+elasticsearch+pull-request+precommit.yml

@@ -20,8 +20,6 @@
           cancel-builds-on-update: true
           white-list-labels:
             - '>test-mute'
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml

@@ -23,7 +23,6 @@
             - 7.16
             - 7.15
             - 6.8
-            - feature/desired-balance-allocator
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/templates.t/pull-request-gradle-unix.yml

@@ -22,8 +22,6 @@
             - ^docs/.*
           black-list-labels:
             - '>test-mute'
-          black-list-target-branches:
-            - feature/desired-balance-allocator
     builders:
       - inject:
           properties-file: '.ci/java-versions.properties'

.github/CODEOWNERS

@@ -0,0 +1,17 @@
+# GitHub CODEOWNERS definition
+# Identify which groups will be pinged by changes to different parts of the codebase.
+# For more info, see https://help.github.com/articles/about-codeowners/
+
+# Stack Monitoring index templates
+x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-beats-mb.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-beats.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-ent-search-mb.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-es-mb.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-es.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-kibana-mb.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-kibana.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-logstash-mb.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-logstash.json @elastic/infra-monitoring-ui
+x-pack/plugin/core/src/main/resources/monitoring-mb-ilm-policy.json @elastic/infra-monitoring-ui
+x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/MonitoringTemplateRegistry.java @elastic/infra-monitoring-ui

CONTRIBUTING.md

@@ -604,7 +604,7 @@ threshold has been breached:
 
     logger.warn(
         "flood stage disk watermark [{}] exceeded on {}, all indices on this node will be marked read-only",
-        diskThresholdSettings.describeFloodStageThreshold(),
+        diskThresholdSettings.describeFloodStageThreshold(total, false),
         usage
     );
 

build-tools-internal/src/main/resources/changelog-schema.json

@@ -83,6 +83,7 @@
             "TLS",
             "Transform",
             "TSDB",
+            "Vector Search",
             "Watcher"
           ]
         },

build-tools-internal/version.properties

@@ -1,8 +1,8 @@
 elasticsearch     = 8.4.0
-lucene            = 9.3.0-snapshot-2d05f5c623e
+lucene            = 9.3.0-snapshot-b8d1fcfd0ec
 
 bundled_jdk_vendor = openjdk
-bundled_jdk = 18.0.1.1+2@65ae32619e2f40f3a9af3af1851d6e19
+bundled_jdk = 18.0.2+9@f6ad4b4450fd4d298113270ec84f30ee
 
 # optional dependencies
 spatial4j         = 0.7

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java

@@ -90,7 +90,9 @@ public List<ResourceLock> getSharedResources() {
 
         int nodeCount = clusters.stream().mapToInt(cluster -> cluster.getNodes().size()).sum();
         if (nodeCount > 0) {
-            locks.add(resource.getResourceLock());
+            for (int i = 0; i < Math.min(nodeCount, resource.getMaxUsages()); i++) {
+                locks.add(resource.getResourceLock());
+            }
         }
         return Collections.unmodifiableList(locks);
     }

docs/changelog/86838.yaml

@@ -0,0 +1,5 @@
+pr: 86838
+summary: Adding cardinality support for `random_sampler` agg
+area: Aggregations
+type: enhancement
+issues: []

docs/changelog/88023.yaml

@@ -0,0 +1,5 @@
+pr: 88023
+summary: "If signature validation fails, reload JWKs and retry if new JWKs are found"
+area: Authentication
+type: enhancement
+issues: []

docs/changelog/88385.yaml

@@ -1,5 +1,6 @@
 pr: 88385
-summary: Test balance improves after rebalancing
+summary: Prevent re-balancing using outdated node weights in some cases
 area: Allocation
 type: bug
-issues: []
+issues:
+ - 88384

docs/changelog/88551.yaml

@@ -0,0 +1,6 @@
+pr: 88551
+summary: "Fix: use status code 500 for aggregation reduce phase errors if no shard failed"
+area: Search
+type: bug
+issues:
+ - 20004

docs/changelog/88622.yaml

@@ -0,0 +1,6 @@
+pr: 88622
+summary: Use origin for the client when running _features/_reset
+area: Infra/Core
+type: bug
+issues:
+ - 88617

docs/changelog/88626.yaml

@@ -0,0 +1,5 @@
+pr: 88626
+summary: Adding the ability to register a `PeerFinderListener` to Coordinator
+area: Distributed
+type: enhancement
+issues: []

docs/changelog/88641.yaml

@@ -0,0 +1,5 @@
+pr: 88641
+summary: Replace health request with a state observer
+area: Allocation
+type: bug
+issues: []

docs/changelog/88642.yaml

@@ -0,0 +1,5 @@
+pr: 88642
+summary: Include API key metadata in audit log when an API key is created, granted, or updated
+area: Audit
+type: enhancement
+issues: []

docs/changelog/88655.yaml

@@ -0,0 +1,5 @@
+pr: 88655
+summary: Make `bucket_correlation` aggregation generally available
+area: Machine Learning
+type: feature
+issues: []

docs/changelog/88675.yaml

@@ -0,0 +1,6 @@
+pr: 88675
+summary: Upgrade to OpenJDK 18.0.2+9
+area: Packaging
+type: upgrade
+issues:
+ - 88673

docs/changelog/88694.yaml

@@ -0,0 +1,14 @@
+pr: 88694
+summary: Integrate ANN into `_search` endpoint
+area: Vector Search
+type: feature
+issues:
+ - 87625
+highlight:
+  title: Integrate ANN into `_search` endpoint
+  body: |-
+    This change adds a `knn` option to the `_search` API to support ANN
+    search. It's powered by the same Lucene ANN capabilities as the old
+    `_knn_search` endpoint. The `knn` option can be combined with other
+    search features like queries and aggregations.
+  notable: true

docs/changelog/88702.yaml

@@ -0,0 +1,5 @@
+pr: 88702
+summary: Fix validation of `close_pit` request
+area: Search
+type: bug
+issues: []

docs/changelog/88707.yaml

@@ -0,0 +1,5 @@
+pr: 88707
+summary: Avoid capturing `SnapshotsInProgress$Entry` in queue
+area: Snapshot/Restore
+type: bug
+issues: []

docs/changelog/88719.yaml

@@ -0,0 +1,5 @@
+pr: 88719
+summary: Convert disk watermarks to RelativeByteSizeValues
+area: Infra/Settings
+type: enhancement
+issues: []

docs/reference/aggregations/bucket/random-sampler-aggregation.asciidoc

@@ -94,3 +94,17 @@ higher sampling rates, the relative error is still low.
 
 NOTE: This represents the result of aggregations against a typical positively skewed APM data set which also has outliers in the upper tail. The linear dependence of the relative error on the sample size is found to hold widely, but the slope depends on the variation in the quantity being aggregated. As such, the variance in your own data may
       cause relative error rates to increase or decrease at a different rate.
+
+[[random-sampler-special-cases]]
+==== Random sampling special cases
+
+All counts returned by the random sampler aggregation are scaled to ease visualizations and calculations. For example,
+when randomly sampling a <<search-aggregations-bucket-datehistogram-aggregation, date histogram aggregation>> every
+`doc_count` value for every bucket is scaled by the inverse of the random_sampler `probability` value. So, if `doc_count`
+for a bucket is `10,000` with `probability: 0.1`, the actual number of documents aggregated is `1,000`.
+
+An exception to this is <<search-aggregations-metrics-cardinality-aggregation, cardinality aggregation>>. Unique item
+counts are not suitable for automatic scaling. When interpreting the cardinality count, compare it
+to the number of sampled docs provided in the top level `doc_count` within the random_sampler aggregation. It gives 
+you an idea of unique values as a percentage of total values. It may not reflect, however, the exact number of unique values
+for the given field.

docs/reference/aggregations/pipeline/bucket-correlation-aggregation.asciidoc

@@ -5,8 +5,6 @@
 <titleabbrev>Bucket correlation</titleabbrev>
 ++++
 
-experimental::[]
-
 A sibling pipeline aggregation which executes a correlation function on the
 configured sibling multi-bucket aggregation.
 

docs/reference/cluster/tasks.asciidoc

@@ -183,8 +183,8 @@ The API returns the following result:
 The new `description` field contains human readable text that identifies the
 particular request that the task is performing such as identifying the search
 request being performed by a search task like the example above. Other kinds of
-task have different descriptions, like <<docs-reindex,`_reindex`>> which
-has the search and the destination, or <<docs-bulk,`_bulk`>> which just has the
+tasks have different descriptions, like <<docs-reindex,`_reindex`>> which
+has the source and the destination, or <<docs-bulk,`_bulk`>> which just has the
 number of requests and the destination indices. Many requests will only have an
 empty description because more detailed information about the request is not
 easily available or particularly helpful in identifying the request.

docs/reference/mapping/types/dense-vector.asciidoc

@@ -152,6 +152,7 @@ NOTE: Although they are conceptually related, the `similarity` parameter is
 different from <<text,`text`>> field <<similarity,`similarity`>> and accepts
 a distinct set of options.
 
+[[dense-vector-index-options]]
 `index_options`::
 (Optional, object)
 An optional section that configures the kNN indexing algorithm. The HNSW

docs/reference/modules/cluster/disk_allocator.asciidoc

@@ -72,14 +72,14 @@ Defaults to `true`. Set to `false` to disable the disk allocation decider.
 // tag::cluster-routing-watermark-low-tag[]
 `cluster.routing.allocation.disk.watermark.low` {ess-icon}::
 (<<dynamic-cluster-setting,Dynamic>>)
-Controls the low watermark for disk usage. It defaults to `85%`, meaning that {es} will not allocate shards to nodes that have more than 85% disk used. It can also be set to an absolute byte value (like `500mb`) to prevent {es} from allocating shards if less than the specified amount of space is available. This setting has no effect on the primary shards of newly-created indices but will prevent their replicas from being allocated.
+Controls the low watermark for disk usage. It defaults to `85%`, meaning that {es} will not allocate shards to nodes that have more than 85% disk used. It can alternatively be set to a ratio value, e.g., `0.85`. It can also be set to an absolute byte value (like `500mb`) to prevent {es} from allocating shards if less than the specified amount of space is available. This setting has no effect on the primary shards of newly-created indices but will prevent their replicas from being allocated.
 // end::cluster-routing-watermark-low-tag[]
 
 [[cluster-routing-watermark-high]]
 // tag::cluster-routing-watermark-high-tag[]
 `cluster.routing.allocation.disk.watermark.high` {ess-icon}::
 (<<dynamic-cluster-setting,Dynamic>>)
-Controls the high watermark. It defaults to `90%`, meaning that {es} will attempt to relocate shards away from a node whose disk usage is above 90%. It can also be set to an absolute byte value (similarly to the low watermark) to relocate shards away from a node if it has less than the specified amount of free space. This setting affects the allocation of all shards, whether previously allocated or not.
+Controls the high watermark. It defaults to `90%`, meaning that {es} will attempt to relocate shards away from a node whose disk usage is above 90%. It can alternatively be set to a ratio value, e.g., `0.9`. It can also be set to an absolute byte value (similarly to the low watermark) to relocate shards away from a node if it has less than the specified amount of free space. This setting affects the allocation of all shards, whether previously allocated or not.
 // end::cluster-routing-watermark-high-tag[]
 
 `cluster.routing.allocation.disk.watermark.enable_for_single_data_node`::
@@ -95,10 +95,10 @@ is now `true`. The setting will be removed in a future release.
 +
 --
 (<<dynamic-cluster-setting,Dynamic>>)
-Controls the flood stage watermark, which defaults to 95%. {es} enforces a read-only index block (`index.blocks.read_only_allow_delete`) on every index that has one or more shards allocated on the node, and that has at least one disk exceeding the flood stage. This setting is a last resort to prevent nodes from running out of disk space. The index block is automatically released when the disk utilization falls below the high watermark.
+Controls the flood stage watermark, which defaults to 95%. {es} enforces a read-only index block (`index.blocks.read_only_allow_delete`) on every index that has one or more shards allocated on the node, and that has at least one disk exceeding the flood stage. This setting is a last resort to prevent nodes from running out of disk space. The index block is automatically released when the disk utilization falls below the high watermark. Similarly to the low and high watermark values, it can alternatively be set to a ratio value, e.g., `0.95`, or an absolute byte value.
 
-NOTE: You cannot mix the usage of percentage values and byte values within
-these settings. Either all values are set to percentage values, or all are set to byte values. This enforcement is so that {es} can validate that the settings are internally consistent, ensuring that the low disk threshold is less than the high disk threshold, and the high disk threshold is less than the flood stage threshold.
+NOTE: You cannot mix the usage of percentage/ratio values and byte values within
+the watermark settings. Either all values are set to percentage/ratio values, or all are set to byte values. This enforcement is so that {es} can validate that the settings are internally consistent, ensuring that the low disk threshold is less than the high disk threshold, and the high disk threshold is less than the flood stage threshold.
 
 An example of resetting the read-only index block on the `my-index-000001` index: