[Filebeat][httpjson] httpjson chain calls

CHANGELOG.next.asciidoc

@@ -106,6 +106,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif
 - Add extraction of `related.hosts` to Microsoft 365 Defender ingest pipeline {issue}29859[29859] {pull}29863[29863]
 - threatintel module: Add new Recorded Future integration. {pull}30030[30030]
 - Add pipeline in FB's supported hints. {pull}30212[30212]
+- Add support in httpjson input for chain calls. {pull}29816[29816]
 
 *Auditbeat*
 

x-pack/filebeat/docs/inputs/input-httpjson.asciidoc

@@ -370,6 +370,7 @@ NOTE: Only one of the credentials settings can be set at once. If none is provid
 default credentials from the environment will be attempted via ADC. For more information about
 how to provide Google credentials, please refer to https://cloud.google.com/docs/authentication.
 
+[[request-parameters]]
 [float]
 ==== `request.url`
 
@@ -931,6 +932,149 @@ This will output:
 ]
 ----
 
+[[chain]]
+[float]
+=== `chain`
+
+Chain is a list of calls to be made after the first call.
+
+chain list option: [`step`].
+
+[float]
+==== `chain[].step`
+
+chain[].step will contain basic request and response configurations for chain calls.
+
+[float]
+==== `chain[].step.request`
+
+Please refer <<request-parameters,request parameters>>. also place same replace string in url where collected values from previous call should be place. Required. 
+
+Example:
+
+first call: https://example.com/services/data/v1.0/
+
+second call: https://example.com/services/data/v1.0/1/export_ids
+
+third call: https://example.com/services/data/v1.0/export_ids/file_1/info
+
+[float]
+==== `chain[].step.respose.split`
+
+Please refer <<response-split,response split parameter>>. Required.
+
+[float]
+==== `chain[].step.replace`
+
+replace is JSONPath string to parse values from response JSONs, collected from previous chain calls. Please look at the official doc of https://goessner.net/articles/JsonPath/index.html#e2[JSONPath]. Required.
+
+Example:
+
+- first call: https://example.com/services/data/v1.0/
++
+<<response-json1,response>>
+
+- second call: https://example.com/services/data/v1.0/`$.records[:].id`/export_ids
++
+<<response-json2,response>>
+
+third call: https://example.com/services/data/v1.0/export_ids`/`$.file_name`/info
+
+["source","yaml",subs="attributes"]
+----
+filebeat.inputs:
+- type: httpjson
+  enabled: true
+  # first call
+  request.url: https://example.com/services/data/v1.0/records
+  interval: 1h
+  chain:
+    # second call
+    - step:
+      request.url: https://example.com/services/data/v1.0/$.records[:].id/export_ids
+      request.method: GET
+      replace: $.records[:].id
+    # third call
+    - step:
+      request.url: https://example.com/services/data/v1.0/export_ids/$.file_name/info
+      request.method: GET
+      replace: $.file_name
+----
+
+Example:
+
+- First call to collect record ids
+
++
+request_url: https://example.com/services/data/v1.0/records
+
++
+response_json:
+
++
+[[response-json1]]
+["source","json",subs="attributes"]
+----
+{
+    "records": [
+        {
+            "id": 1,
+        },
+        {
+            "id": 2,
+        },
+        {
+            "id": 3,
+        },
+    ]
+}
+----
+
+- Second call to collect `file_name` using collected ids from first call.
+
++
+request_url using id as '1': https://example.com/services/data/v1.0/1/export_ids
+
++
+response_json using id as '1':
+
++
+[[response-json1]]
+["source","json",subs="attributes"]
+----
+{
+    "file_name": "file_1"
+}
+----
+
++
+request_url using id as '2': https://example.com/services/data/v1.0/2/export_ids
+
++
+response_json using id as '2':
+
++
+["source","json",subs="attributes"]
+----
+{
+    "file_name": "file_2"
+}
+----
+
+- Third call to collect `files` using collected `file_name` from second call.
+
++
+request_url using file_name as 'file_1': https://example.com/services/data/v1.0/export_ids/file_1/info
+
++
+request_url using file_name as 'file_2': https://example.com/services/data/v1.0/export_ids/file_2/info
+
++
+Collect and make events from response in any format supported by httpjson for all calls.
+
+NOTE: httpjson chain will only create and ingest events from last call on chain configurations.
+httpjson chain will also 
+
 [[cursor]]
 [float]
 ==== `cursor`

x-pack/filebeat/input/httpjson/chain.go

@@ -0,0 +1,74 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// Example:
+// 1. First call to collect record ids
+//   request_url: https://some_url.com/services/data/v1.0/records
+//   response_json:
+//     {
+//         "records": [
+//             {
+//                 "id": 1,
+//             },
+//             {
+//                 "id": 2,
+//             },
+//             {
+//                 "id": 3,
+//             },
+//         ]
+//     }
+//
+// 2. Second call to collect file name using collected ids from first call.
+//   request_url using id as '1': https://some_url.com/services/data/v1.0/1/export_ids
+//   response_json using id as '1':
+//     {
+//         "file_name": "file_1"
+//     }
+//   request_url using id as '2': https://some_url.com/services/data/v1.0/2/export_ids
+//   response_json using id as '2':
+//     {
+//         "file_name": "file_2"
+//     }
+//
+// 3. Third call to collect files using collected file names from second call.
+//   request_url using file_name as 'file_1': https://some_url.com/services/data/v1.0/export_ids/file_1/info
+//   request_url using file_name as 'file_2': https://some_url.com/services/data/v1.0/export_ids/file_2/info
+//
+//   Collect and make events from response in any format[csv, json, etc.] for all calls.
+//
+// Example configuration:
+//
+// - type: httpjson
+//   enabled: true
+//   request.url: https://some_url.com/services/data/v1.0/records (first call)
+//   interval: 1h
+//   chain:
+//     - step:
+//         request.url: https://some_url.com/services/data/v1.0/$.records[:].id/export_ids (second call)
+//         request.method: GET
+//         replace: $.records[:].id
+//     - step:
+//         request.url: https://some_url.com/services/data/v1.0/export_ids/$.file_name/info (third call)
+//         request.method: GET
+//         replace: $.file_name
+
+package httpjson
+
+// chainConfig for chain request.
+// Following contains basic call structure for each call after normal httpjson
+// call.
+type chainConfig struct {
+	Step stepConfig `config:"step" validate:"required"`
+}
+
+// stepConfig will contain basic properties like, request.url,
+// request.method and replace parameter. Each step: request.url
+// will contain replace string with original URL to make a skeleton for the
+// call request.
+type stepConfig struct {
+	Request  requestConfig  `config:"request"`
+	Response responseConfig `config:"response,omitempty"`
+	Replace  string         `config:"replace,omitempty"`
+}

x-pack/filebeat/input/httpjson/config.go

@@ -17,6 +17,7 @@ type config struct {
 	Request  *requestConfig  `config:"request" validate:"required"`
 	Response *responseConfig `config:"response"`
 	Cursor   cursorConfig    `config:"cursor"`
+	Chain    []chainConfig   `config:"chain"`
 }
 
 type cursorConfig map[string]cursorEntry

x-pack/filebeat/input/httpjson/input.go

@@ -111,7 +111,7 @@ func run(
 		return err
 	}
 
-	requestFactory := newRequestFactory(config.Request, config.Auth, log)
+	requestFactory := newRequestFactory(config, log)
 	pagination := newPagination(config, httpClient, log)
 	responseProcessor := newResponseProcessor(config.Response, pagination, log)
 	requester := newRequester(httpClient, requestFactory, responseProcessor, log)