[GCP] Add pubsub ingest pipeline

packages/gcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.11.10-beta.4"
+  changes:
+    - description: Add GCP PubSub ingest pipeline
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/4349
 - version: "2.11.10-beta.3"
   changes:
     - description: Add GCP Storage ingest pipeline

packages/gcp/data_stream/pubsub/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,207 @@
+---
+description: Pipeline for parsing GCP PubSub metrics.
+processors:
+  - rename:
+      field: gcp.metrics.snapshot.backlog.bytes
+      target_field: gcp.pubsub.snapshot.backlog.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.snapshot.backlog_bytes_by_region.bytes
+      target_field: gcp.pubsub.snapshot.backlog_bytes_by_region.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.snapshot.config_updates.count
+      target_field: gcp.pubsub.snapshot.config_updates.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.snapshot.num_messages.value
+      target_field: gcp.pubsub.snapshot.num_messages.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.snapshot.num_messages_by_region.value
+      target_field: gcp.pubsub.snapshot.num_messages_by_region.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.snapshot.oldest_message_age.sec
+      target_field: gcp.pubsub.snapshot.oldest_message_age.sec
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.snapshot.oldest_message_age_by_region.sec
+      target_field: gcp.pubsub.snapshot.oldest_message_age_by_region.sec
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.ack_message.count
+      target_field: gcp.pubsub.subscription.ack_message.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.backlog.bytes
+      target_field: gcp.pubsub.subscription.backlog.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.byte_cost.bytes
+      target_field: gcp.pubsub.subscription.byte_cost.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.config_updates.count
+      target_field: gcp.pubsub.subscription.config_updates.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.dead_letter_message.count
+      target_field: gcp.pubsub.subscription.dead_letter_message.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.mod_ack_deadline_message.count
+      target_field: gcp.pubsub.subscription.mod_ack_deadline_message.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.mod_ack_deadline_message_operation.count
+      target_field: gcp.pubsub.subscription.mod_ack_deadline_message_operation.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.mod_ack_deadline_request.count
+      target_field: gcp.pubsub.subscription.mod_ack_deadline_request.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.num_outstanding_messages.value
+      target_field: gcp.pubsub.subscription.num_outstanding_messages.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.num_undelivered_messages.value
+      target_field: gcp.pubsub.subscription.num_undelivered_messages.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.oldest_retained_acked_message_age.sec
+      target_field: gcp.pubsub.subscription.oldest_retained_acked_message_age.sec
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.oldest_retained_acked_message_age_by_region.value
+      target_field: gcp.pubsub.subscription.oldest_retained_acked_message_age_by_region.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.oldest_unacked_message_age.sec
+      target_field: gcp.pubsub.subscription.oldest_unacked_message_age.sec
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.oldest_unacked_message_age_by_region.value
+      target_field: gcp.pubsub.subscription.oldest_unacked_message_age_by_region.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.pull_ack_message_operation.count
+      target_field: gcp.pubsub.subscription.pull_ack_message_operation.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.pull_ack_request.count
+      target_field: gcp.pubsub.subscription.pull_ack_request.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.pull_message_operation.count
+      target_field: gcp.pubsub.subscription.pull_message_operation.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.pull_request.count
+      target_field: gcp.pubsub.subscription.pull_request.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.push_request.count
+      target_field: gcp.pubsub.subscription.push_request.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.retained_acked.bytes
+      target_field: gcp.pubsub.subscription.retained_acked.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.retained_acked_bytes_by_region.bytes
+      target_field: gcp.pubsub.subscription.retained_acked_bytes_by_region.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.seek_request.count
+      target_field: gcp.pubsub.subscription.seek_request.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.sent_message.count
+      target_field: gcp.pubsub.subscription.sent_message.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.streaming_pull_ack_message_operation.count
+      target_field: gcp.pubsub.subscription.streaming_pull_ack_message_operation.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.streaming_pull_ack_request.count
+      target_field: gcp.pubsub.subscription.streaming_pull_ack_request.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.streaming_pull_message_operation.count
+      target_field: gcp.pubsub.subscription.streaming_pull_message_operation.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.streaming_pull_mod_ack_deadline_message_operation.count
+      target_field: gcp.pubsub.subscription.streaming_pull_mod_ack_deadline_message_operation.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.streaming_pull_mod_ack_deadline_request.count
+      target_field: gcp.pubsub.subscription.streaming_pull_mod_ack_deadline_request.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.streaming_pull_response.count
+      target_field: gcp.pubsub.subscription.streaming_pull_response.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.unacked_bytes_by_region.bytes
+      target_field: gcp.pubsub.subscription.unacked_bytes_by_region.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.topic.byte_cost.bytes
+      target_field: gcp.pubsub.topic.byte_cost.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.topic.config_updates.count
+      target_field: gcp.pubsub.topic.config_updates.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.topic.message_sizes.bytes
+      target_field: gcp.pubsub.topic.message_sizes.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.topic.oldest_retained_acked_message_age_by_region.value
+      target_field: gcp.pubsub.topic.oldest_retained_acked_message_age_by_region.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.topic.oldest_unacked_message_age_by_region.value
+      target_field: gcp.pubsub.topic.oldest_unacked_message_age_by_region.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.topic.retained_acked_bytes_by_region.bytes
+      target_field: gcp.pubsub.topic.retained_acked_bytes_by_region.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.topic.send_message_operation.count
+      target_field: gcp.pubsub.topic.send_message_operation.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.topic.send_request.count
+      target_field: gcp.pubsub.topic.send_request.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.topic.streaming_pull_response.count
+      target_field: gcp.pubsub.topic.streaming_pull_response.count
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.topic.unacked_bytes_by_region.bytes
+      target_field: gcp.pubsub.topic.unacked_bytes_by_region.bytes
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.ack_latencies.value
+      target_field: gcp.pubsub.subscription.ack_latencies.value
+      ignore_missing: true
+  - rename:
+      field: gcp.metrics.subscription.push_request_latencies.value
+      target_field: gcp.pubsub.subscription.push_request_latencies.value
+      ignore_missing: true
+  - remove:
+      field:
+        - gcp.metrics
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'

packages/gcp/manifest.yml

@@ -1,6 +1,6 @@
 name: gcp
 title: Google Cloud Platform
-version: "2.11.10-beta.3"
+version: "2.11.10-beta.4"
 release: ga
 description: Collect logs from Google Cloud Platform with Elastic Agent.
 type: integration