From da5ba4c0cae4f13ebb22e8040da4ed2b2f926a28 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 25 Feb 2019 15:57:46 +0100 Subject: [PATCH] Migrate metricbeat docker module to ECS (#10927) Fix #10900 (cherry picked from commit 3b1b0cf7a3dea9ff75a5a6b2220e38df3c156064) --- CHANGELOG.next.asciidoc | 4 + .../module/docker/container/_meta/data.json | 54 +++++++++---- .../module/docker/container/container.go | 14 +++- .../container/container_integration_test.go | 13 +++- metricbeat/module/docker/container/data.go | 44 ++++++----- metricbeat/module/docker/cpu/_meta/data.json | 75 ++++++++++++------- metricbeat/module/docker/cpu/cpu.go | 14 +++- .../module/docker/cpu/cpu_integration_test.go | 13 +++- metricbeat/module/docker/cpu/data.go | 18 ++--- .../module/docker/diskio/_meta/data.json | 59 +++++++++------ metricbeat/module/docker/diskio/data.go | 18 ++--- metricbeat/module/docker/diskio/diskio.go | 14 +++- .../docker/diskio/diskio_integration_test.go | 13 +++- metricbeat/module/docker/fields.go | 2 +- .../module/docker/healthcheck/_meta/data.json | 56 +++++++++----- metricbeat/module/docker/healthcheck/data.go | 32 ++++---- .../module/docker/healthcheck/healthcheck.go | 14 +++- .../healthcheck_integration_test.go | 13 +++- metricbeat/module/docker/helper.go | 13 +++- .../module/docker/memory/_meta/data.json | 59 +++++++++------ metricbeat/module/docker/memory/data.go | 19 +++-- metricbeat/module/docker/memory/memory.go | 14 +++- .../docker/memory/memory_integration_test.go | 22 +++--- .../module/docker/memory/memory_test.go | 33 +++++--- .../module/docker/network/_meta/data.json | 54 ++++++++----- .../module/docker/network/_meta/fields.yml | 2 - metricbeat/module/docker/network/data.go | 7 +- metricbeat/module/docker/network/network.go | 6 ++ metricbeat/tests/system/test_docker.py | 2 +- 29 files changed, 447 insertions(+), 254 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e9c674dd870f..ba4e00cdbcbb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -20,6 +20,10 @@ https://github.com/elastic/beats/compare/v7.0.0-beta1...master[Check the HEAD di *Metricbeat* +- Migrate docker module to ECS. {pull}10927[10927] + +*Packetbeat* + *Winlogbeat* *Functionbeat* diff --git a/metricbeat/module/docker/container/_meta/data.json b/metricbeat/module/docker/container/_meta/data.json index fb7c93711ccc..881232869394 100644 --- a/metricbeat/module/docker/container/_meta/data.json +++ b/metricbeat/module/docker/container/_meta/data.json @@ -1,37 +1,59 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, + "container": { + "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a", + "image": { + "name": "metricbeat_elasticsearch" + }, + "name": "metricbeat_elasticsearch_1_df866b3a7b3d", + "runtime": "docker" + }, "docker": { "container": { - "command": "go test -tags=integration github.com/elastic/beats/metricbeat/module/... -data", - "created": "2017-12-07T07:20:57.000Z", - "id": "d88e67bb6961a5bb70c1c1c48094c6030e43768eed91e827f437111888f9967e", - "image": "metricbeatsnapshotnoxpack700alpha1d71419298b58ed8b0a5b60a6d1e4a476ffaf80a8_beat", + "command": "/usr/local/bin/docker-entrypoint.sh eswrapper", + "created": "2019-02-25T10:18:10.000Z", "ip_addresses": [ - "172.18.0.27" + "172.23.0.2" ], "labels": { + "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78", "com_docker_compose_container-number": "1", - "com_docker_compose_oneoff": "True", - "com_docker_compose_project": "metricbeatsnapshotnoxpack700alpha1d71419298b58ed8b0a5b60a6d1e4a476ffaf80a8", - "com_docker_compose_service": "beat", - "com_docker_compose_version": "1.16.1" + "com_docker_compose_oneoff": "False", + "com_docker_compose_project": "metricbeat", + "com_docker_compose_service": "elasticsearch", + "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d", + "com_docker_compose_version": "1.23.1", + "license": "Elastic License", + "org_label-schema_build-date": "20181006", + "org_label-schema_license": "GPLv2", + "org_label-schema_name": "elasticsearch", + "org_label-schema_schema-version": "1.0", + "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", + "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker", + "org_label-schema_vendor": "Elastic", + "org_label-schema_version": "6.5.1" }, - "name": "metricbeatsnapshotnoxpack700alpha1d71419298b58ed8b0a5b60a6d1e4a476ffaf80a8_beat_run_1", "size": { "root_fs": 0, "rw": 0 }, - "status": "Up 6 minutes (healthy)" + "status": "Up 7 minutes (healthy)" } }, + "event": { + "dataset": "docker.container", + "duration": 115000, + "module": "docker" + }, "metricset": { - "host": "/var/run/docker.sock", - "module": "docker", - "name": "container", - "rtt": 115 + "name": "container" + }, + "service": { + "address": "/var/run/docker.sock", + "type": "docker" } } \ No newline at end of file diff --git a/metricbeat/module/docker/container/container.go b/metricbeat/module/docker/container/container.go index 0247c179ea75..6032493e755a 100644 --- a/metricbeat/module/docker/container/container.go +++ b/metricbeat/module/docker/container/container.go @@ -22,12 +22,15 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/client" + "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/docker" ) +var logger = logp.NewLogger("docker.container") + func init() { mb.Registry.MustAddMetricSet("docker", "container", New, mb.WithHostParser(docker.HostParser), @@ -62,11 +65,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch returns a list of all containers as events. // This is based on https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/list-containers. -func (m *MetricSet) Fetch() ([]common.MapStr, error) { +func (m *MetricSet) Fetch(r mb.ReporterV2) { // Fetch a list of all containers. containers, err := m.dockerClient.ContainerList(context.Background(), types.ContainerListOptions{}) if err != nil { - return nil, err + err = errors.Wrap(err, "failed to get docker containers list") + logger.Error(err) + r.Error(err) + return } - return eventsMapping(containers, m.dedot), nil + eventsMapping(r, containers, m.dedot) } diff --git a/metricbeat/module/docker/container/container_integration_test.go b/metricbeat/module/docker/container/container_integration_test.go index 48cf0e2a6648..321a727fa433 100644 --- a/metricbeat/module/docker/container/container_integration_test.go +++ b/metricbeat/module/docker/container/container_integration_test.go @@ -22,13 +22,20 @@ package container import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) func TestData(t *testing.T) { - f := mbtest.NewEventsFetcher(t, getConfig()) - err := mbtest.WriteEvents(f, t) - if err != nil { + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } diff --git a/metricbeat/module/docker/container/data.go b/metricbeat/module/docker/container/data.go index 80e4179fa150..c0bb07518faf 100644 --- a/metricbeat/module/docker/container/data.go +++ b/metricbeat/module/docker/container/data.go @@ -23,39 +23,49 @@ import ( "github.com/docker/docker/api/types" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/docker" ) -func eventsMapping(containersList []types.Container, dedot bool) []common.MapStr { - myEvents := []common.MapStr{} +func eventsMapping(r mb.ReporterV2, containersList []types.Container, dedot bool) { for _, container := range containersList { - myEvents = append(myEvents, eventMapping(&container, dedot)) + eventMapping(r, &container, dedot) } - return myEvents } -func eventMapping(cont *types.Container, dedot bool) common.MapStr { +func eventMapping(r mb.ReporterV2, cont *types.Container, dedot bool) { event := common.MapStr{ - "created": common.Time(time.Unix(cont.Created, 0)), - "id": cont.ID, - "name": docker.ExtractContainerName(cont.Names), - "command": cont.Command, - "image": cont.Image, - "ip_addresses": extractIPAddresses(cont.NetworkSettings), - "size": common.MapStr{ - "root_fs": cont.SizeRootFs, - "rw": cont.SizeRw, + "container": common.MapStr{ + "id": cont.ID, + "image": common.MapStr{ + "name": cont.Image, + }, + "name": docker.ExtractContainerName(cont.Names), + "runtime": "docker", + }, + "docker": common.MapStr{ + "container": common.MapStr{ + "created": common.Time(time.Unix(cont.Created, 0)), + "command": cont.Command, + "ip_addresses": extractIPAddresses(cont.NetworkSettings), + "size": common.MapStr{ + "root_fs": cont.SizeRootFs, + "rw": cont.SizeRw, + }, + "status": cont.Status, + }, }, - "status": cont.Status, } labels := docker.DeDotLabels(cont.Labels, dedot) if len(labels) > 0 { - event["labels"] = labels + event.Put("docker.container.labels", labels) } - return event + r.Event(mb.Event{ + RootFields: event, + }) } func extractIPAddresses(networks *types.SummaryNetworkSettings) []string { diff --git a/metricbeat/module/docker/cpu/_meta/data.json b/metricbeat/module/docker/cpu/_meta/data.json index 5589d9858013..370558eed754 100644 --- a/metricbeat/module/docker/cpu/_meta/data.json +++ b/metricbeat/module/docker/cpu/_meta/data.json @@ -1,67 +1,84 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, + "container": { + "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a", + "image": { + "name": "metricbeat_elasticsearch" + }, + "name": "metricbeat_elasticsearch_1_df866b3a7b3d", + "runtime": "docker" + }, "docker": { "container": { - "id": "bbdcbc751e7eda7a50e773b4a5d8c2800af664f835ef9c0ad6bbb9c160c50d83", "labels": { - "build-date": "20170911", - "com_docker_compose_config-hash": "371e477ae73fd44b19bcbcf0d4feaf4de9adfb69137a8f16d09cff749724ca99", + "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", "com_docker_compose_project": "metricbeat", "com_docker_compose_service": "elasticsearch", - "com_docker_compose_version": "1.21.0", - "license": "GPLv2", - "maintainer": "Elastic Docker Team \u003cdocker@elastic.co\u003e", - "name": "CentOS Base Image", - "vendor": "CentOS" - }, - "name": "metricbeat_elasticsearch_1" + "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d", + "com_docker_compose_version": "1.23.1", + "license": "Elastic License", + "org_label-schema_build-date": "20181006", + "org_label-schema_license": "GPLv2", + "org_label-schema_name": "elasticsearch", + "org_label-schema_schema-version": "1.0", + "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", + "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker", + "org_label-schema_vendor": "Elastic", + "org_label-schema_version": "6.5.1" + } }, "cpu": { "core": { "0": { - "pct": 0.01012583677581864, - "ticks": 9528454911 + "pct": 0.03263313721518987, + "ticks": 38346196894 }, "1": { - "pct": 0.0069975889168765746, - "ticks": 11916812270 + "pct": 0.014317838987341772, + "ticks": 37143007802 }, "2": { - "pct": 0.001329603022670025, - "ticks": 10894346015 + "pct": 0.0028625296202531647, + "ticks": 37194678570 }, "3": { - "pct": 0.0018390015113350126, - "ticks": 10847487614 + "pct": 0.005687502784810126, + "ticks": 39335551141 } }, "kernel": { - "pct": 0.010075566750629723, - "ticks": 1960000000 + "pct": 0.010126582278481013, + "ticks": 10560000000 }, "system": { "pct": 4, - "ticks": 1092479570000000 + "ticks": 5566563680000000 }, "total": { - "pct": 0.02029203022670025 + "pct": 0.05550100860759494 }, "user": { - "pct": 0.010075566750629723, - "ticks": 40960000000 + "pct": 0.05063291139240506, + "ticks": 139520000000 } } }, + "event": { + "dataset": "docker.cpu", + "duration": 115000, + "module": "docker" + }, "metricset": { - "host": "/var/run/docker.sock", - "module": "docker", - "name": "cpu", - "rtt": 115 + "name": "cpu" + }, + "service": { + "address": "/var/run/docker.sock", + "type": "docker" } } \ No newline at end of file diff --git a/metricbeat/module/docker/cpu/cpu.go b/metricbeat/module/docker/cpu/cpu.go index 954eb37d6d21..a7dd10cf8926 100644 --- a/metricbeat/module/docker/cpu/cpu.go +++ b/metricbeat/module/docker/cpu/cpu.go @@ -19,12 +19,15 @@ package cpu import ( "github.com/docker/docker/client" + "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/docker" ) +var logger = logp.NewLogger("docker.cpu") + func init() { mb.Registry.MustAddMetricSet("docker", "cpu", New, mb.WithHostParser(docker.HostParser), @@ -69,12 +72,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } // Fetch returns a list of docker CPU stats. -func (m *MetricSet) Fetch() ([]common.MapStr, error) { +func (m *MetricSet) Fetch(r mb.ReporterV2) { stats, err := docker.FetchStats(m.dockerClient, m.Module().Config().Timeout) if err != nil { - return nil, err + err = errors.Wrap(err, "failed to get docker stats") + logger.Error(err) + r.Error(err) + return } formattedStats := m.cpuService.getCPUStatsList(stats, m.dedot) - return eventsMapping(formattedStats), nil + eventsMapping(r, formattedStats) } diff --git a/metricbeat/module/docker/cpu/cpu_integration_test.go b/metricbeat/module/docker/cpu/cpu_integration_test.go index 209f66335586..1dad5cb1dd8f 100644 --- a/metricbeat/module/docker/cpu/cpu_integration_test.go +++ b/metricbeat/module/docker/cpu/cpu_integration_test.go @@ -22,13 +22,20 @@ package cpu import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) func TestData(t *testing.T) { - f := mbtest.NewEventsFetcher(t, getConfig()) - err := mbtest.WriteEvents(f, t) - if err != nil { + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } diff --git a/metricbeat/module/docker/cpu/data.go b/metricbeat/module/docker/cpu/data.go index 3cf93fe93d68..a8c3042ed4d4 100644 --- a/metricbeat/module/docker/cpu/data.go +++ b/metricbeat/module/docker/cpu/data.go @@ -22,19 +22,14 @@ import ( "github.com/elastic/beats/metricbeat/mb" ) -func eventsMapping(cpuStatsList []CPUStats) []common.MapStr { - events := []common.MapStr{} +func eventsMapping(r mb.ReporterV2, cpuStatsList []CPUStats) { for _, cpuStats := range cpuStatsList { - events = append(events, eventMapping(&cpuStats)) + eventMapping(r, &cpuStats) } - return events } -func eventMapping(stats *CPUStats) common.MapStr { - event := common.MapStr{ - mb.ModuleDataKey: common.MapStr{ - "container": stats.Container.ToMapStr(), - }, +func eventMapping(r mb.ReporterV2, stats *CPUStats) { + fields := common.MapStr{ "core": stats.PerCpuUsage, "total": common.MapStr{ "pct": stats.TotalUsage, @@ -53,5 +48,8 @@ func eventMapping(stats *CPUStats) common.MapStr { }, } - return event + r.Event(mb.Event{ + RootFields: stats.Container.ToMapStr(), + MetricSetFields: fields, + }) } diff --git a/metricbeat/module/docker/diskio/_meta/data.json b/metricbeat/module/docker/diskio/_meta/data.json index 3adb0cc16750..2fba47412c8b 100644 --- a/metricbeat/module/docker/diskio/_meta/data.json +++ b/metricbeat/module/docker/diskio/_meta/data.json @@ -1,52 +1,69 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, + "container": { + "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a", + "image": { + "name": "metricbeat_elasticsearch" + }, + "name": "metricbeat_elasticsearch_1_df866b3a7b3d", + "runtime": "docker" + }, "docker": { "container": { - "id": "59c5d4838454f38c7d67fdacec7a32ca4476a062ef00edf69ba6be9117cf2e7b", "labels": { - "build-date": "20170911", - "com_docker_compose_config-hash": "a2bcfc1f8c99a4be6920deda8da8d4d06fe0d10d51623b8e1dbcc8228e96926c", + "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", "com_docker_compose_project": "metricbeat", "com_docker_compose_service": "elasticsearch", - "com_docker_compose_version": "1.20.1", - "license": "GPLv2", - "maintainer": "Elastic Docker Team \u003cdocker@elastic.co\u003e", - "name": "CentOS Base Image", - "vendor": "CentOS" - }, - "name": "metricbeat_elasticsearch_1" + "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d", + "com_docker_compose_version": "1.23.1", + "license": "Elastic License", + "org_label-schema_build-date": "20181006", + "org_label-schema_license": "GPLv2", + "org_label-schema_name": "elasticsearch", + "org_label-schema_schema-version": "1.0", + "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", + "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker", + "org_label-schema_vendor": "Elastic", + "org_label-schema_version": "6.5.1" + } }, "diskio": { "read": { - "bytes": 61964288, - "ops": 3284, + "bytes": 998932480, + "ops": 8473, "rate": 0 }, "reads": 0, "summary": { - "bytes": 63479808, - "ops": 3500, + "bytes": 1090650112, + "ops": 9585, "rate": 0 }, "total": 0, "write": { - "bytes": 1515520, - "ops": 216, + "bytes": 91717632, + "ops": 1112, "rate": 0 }, "writes": 0 } }, + "event": { + "dataset": "docker.diskio", + "duration": 115000, + "module": "docker" + }, "metricset": { - "host": "/var/run/docker.sock", - "module": "docker", - "name": "diskio", - "rtt": 115 + "name": "diskio" + }, + "service": { + "address": "/var/run/docker.sock", + "type": "docker" } } \ No newline at end of file diff --git a/metricbeat/module/docker/diskio/data.go b/metricbeat/module/docker/diskio/data.go index 5613c4f713b5..2a7d4cb9569a 100644 --- a/metricbeat/module/docker/diskio/data.go +++ b/metricbeat/module/docker/diskio/data.go @@ -22,19 +22,14 @@ import ( "github.com/elastic/beats/metricbeat/mb" ) -func eventsMapping(blkioStatsList []BlkioStats) []common.MapStr { - myEvents := []common.MapStr{} +func eventsMapping(r mb.ReporterV2, blkioStatsList []BlkioStats) { for _, blkioStats := range blkioStatsList { - myEvents = append(myEvents, eventMapping(&blkioStats)) + eventMapping(r, &blkioStats) } - return myEvents } -func eventMapping(stats *BlkioStats) common.MapStr { - event := common.MapStr{ - mb.ModuleDataKey: common.MapStr{ - "container": stats.Container.ToMapStr(), - }, +func eventMapping(r mb.ReporterV2, stats *BlkioStats) { + fields := common.MapStr{ "reads": stats.reads, "writes": stats.writes, "total": stats.totals, @@ -55,5 +50,8 @@ func eventMapping(stats *BlkioStats) common.MapStr { }, } - return event + r.Event(mb.Event{ + RootFields: stats.Container.ToMapStr(), + MetricSetFields: fields, + }) } diff --git a/metricbeat/module/docker/diskio/diskio.go b/metricbeat/module/docker/diskio/diskio.go index 994823bba57d..d300e92d3e46 100644 --- a/metricbeat/module/docker/diskio/diskio.go +++ b/metricbeat/module/docker/diskio/diskio.go @@ -19,12 +19,15 @@ package diskio import ( "github.com/docker/docker/client" + "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/docker" ) +var logger = logp.NewLogger("docker.diskio") + func init() { mb.Registry.MustAddMetricSet("docker", "diskio", New, mb.WithHostParser(docker.HostParser), @@ -60,12 +63,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } // Fetch creates list of events with diskio stats for all containers. -func (m *MetricSet) Fetch() ([]common.MapStr, error) { +func (m *MetricSet) Fetch(r mb.ReporterV2) { stats, err := docker.FetchStats(m.dockerClient, m.Module().Config().Timeout) if err != nil { - return nil, err + err = errors.Wrap(err, "failed to get docker stats") + logger.Error(err) + r.Error(err) + return } formattedStats := m.blkioService.getBlkioStatsList(stats, m.dedot) - return eventsMapping(formattedStats), nil + eventsMapping(r, formattedStats) } diff --git a/metricbeat/module/docker/diskio/diskio_integration_test.go b/metricbeat/module/docker/diskio/diskio_integration_test.go index 1f3a7d1ceed8..21e3b20c20d6 100644 --- a/metricbeat/module/docker/diskio/diskio_integration_test.go +++ b/metricbeat/module/docker/diskio/diskio_integration_test.go @@ -22,13 +22,20 @@ package diskio import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) func TestData(t *testing.T) { - f := mbtest.NewEventsFetcher(t, getConfig()) - err := mbtest.WriteEvents(f, t) - if err != nil { + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } diff --git a/metricbeat/module/docker/fields.go b/metricbeat/module/docker/fields.go index f2e8dfeab15b..ec367b3a5a70 100644 --- a/metricbeat/module/docker/fields.go +++ b/metricbeat/module/docker/fields.go @@ -32,5 +32,5 @@ func init() { // AssetDocker returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/docker. func AssetDocker() string { - return "eJzsm82O2zgSgO/9FIXsYYFF2sYugj34sMBOMoPxIUkjSc/VoKmSzbFEavhjt/P0A5KSJUuUZMuyuzuIjpZd9bFYPySLvocN7mcQCbpBeQegmU5wBm8+uA/e3AFEqKhkmWaCz+B/dwAA/iUoTbQCKpIEqcYIYinS/N3kDkBigkThDFbkDkCthdQLKnjMVjOISaLwDiBmmERq5qTeAycpVljso/eZlSCFyfJPAjz2mfNYyJTYj4HwyMExpRlVQJbC6FzsPxVIwznjK6CCa8I4SjXJpVRpqkSHbx7ehMA64CpGO8iCFLVk9KDcPscmK5461jFamhIeHb0r4Da43wlZf9eBaJ/3XiDoNdGwIwrwCamx08s46DU2xjEJc0kkGsNcEdF4HtQHohF2a/QEpQktX64pjGG9wKgxrVOo9pLDWlm2IFEkUSkM62bZULXzBziIbhky+163bthXzxsu+44hj4UW/6wSSSH0Iq5bogRLBF8FXvawueeb0CTxcCIGkiTOQWKWoCr8tcVRjwB312D7mlOVRC6m1mSLsETkheeCkEDXhK8wAsU4Rf+CCR6eYE1WI3r0PCUrdDInzbyXmUsy3hfDNUsR3j88jpPsNig5JpOM6uD4FSUJRos4EaT+BV8bZpChpMg1WZ2ZgB4Ov3MzakfFeM4DKiMUw3OVE2tGN+E5C/hXX0w+PIKTdxqB2iuN6QuwmYtUD++tZ+Mip+siH9t2Xqw3YVixUShfgMFyM1margl2tFdzsD7tblqfw1jfDv5kFFm10FEhcfKvVjyx/BMbr/yHixvP9nFgMOXRuwbVPuX9wzrfKT6ZdImyJM3dI4B6WMYztWHiohUzUxuYTz+PUzwkkvBydMC66P+UmtQkrnpbuQoiI+2ewua0hMWHuh/aObSBVmFFdo3lUjmJg6BLvOVeN1a2vYBFwLT9+IQB/GJ/6uCHs8vm7qMX/SzbUiMlcp3bOLP5E6mo7dGqXhmO4o7UE2EmkVrvm8F/J++GRvJZoDvJGnYbJX6c4FcXQMOoX0oEWXqN/BUEUW7nU5zzucPoNFRl0pTI/fUqEeGvNaZsqRcZSrf3fbWx5apTMQmvI8gqRu/xXrfcf6Y4a7h3gLXgxC1yfdHa059y1uWcv+wc+wTyV0sUkno4fBzxMNgrY5E/eyVbwhKyTDCoN5YiHX2YwkgaVmdlj6fu2xrdz62f+Z0SYMq0LgK37gclB6FW5nVIOrWKeuK4oIY0hfXVgoaX9Q37BI5i+PMPRY48bSqODKO1ZEvTVQCC22Oob5EvGsUfRDJhlBUy3ZLEYIXreGxvbXZEHtnRCQ5Mq2PPLsa1RpLoNV0j3YyQ1kZuQsWEJYyvlJZINkGvZFzjqlHqeixJBafGFSirACOoD/96Wfb30ty5bKAiCmeikEMOjkUnLDf2uX0W5NEi0FuDrsbbCUh1eyCPwpIqkyH1LUicom4WYXRmQsE+QpRXUVr0HKbmielFw4OqJOEIGWaUhrsesnZ6fEw5NH84OePkjpblyoDQsaXDgyVkj1IBi5BrFrNm668vkvKF8XXcBh45+8sUrCUkrNgWOZgsrwPhLmAVMyNXpJyXYHnFcsBvgcXAtPVopdVbf+ljt2Z07Xc1+ZbCDy5iEqlO9k4h8npKu+JlAXeFgaWVWwOeqP/GwIjtc99bdd1p75Ln+uGWSW0aGy4YuzvtTNPRHseVSUgoNY3cv7csJEmAErrGyGMpIEoJytwJhxZNJyu5g/OZkCUmQ/skF3TUvd4euNu18hmPL+rFzHksioQPS6IwssvVtdaZmk2nkaBq4i9YTahIp8hXjONUYowSOcUpydjUv19ITIXGBcnYYvvvyX/eTf8xjZjKErK/963Z+x2L8J6V97kuvSFVXPMaK6w/b1E6Nz26DHR2cGfEqEbSgxGiygcVPxyceEWB+25Npvxu3A2g2m/hNamUFll2E1Plmk6iCp2FXYPJVdouU13j5Cdfo+TrvbVQOricCnO4vB1kOb/x3GoNr6WZ6VJMxdH5+tm57qOTMHx527pHnlBhWraKncennQb6jTCbigzXbXcxE5aysNbAdHSdg/eQ5HZz6sIkUo2WhL98/ZpP9bDsOyh6R2gSeH/OySUqF1WgULtVUMdaP3hk1es8cMpNlRPRP7ZAVwS3XVFq6hw88Y/K7z2HT31Knp5h4j+Sp4I6cGMJXuRUO9DW6YWXFk41uxZsHPVOyIvOTT95ESPXBGZzdkzoiI2MAvQg2qlqWS+E2xbhmLyghTfnVKS2ZOcTkf9xpGzfnRvAz9Xgra9FWDEwJ7M9OCLZvX7tCeoBZLnGkjAjdIPNVFk5oZRSNLZIMNpy1ot3LY6TkfIv3GCJ3c1UOUu+TcB8NnolfsSAEcXAXmzAHAhfTsCcjnS7gOlmKgvMUpiWP6fdqsr4f9Qc/23M9Y3qB8CvJ4rGKjvjucPPcnOdcnPD8GmpOT9g+IxVhMYPn5/FZ2Dx+TsAAP//9lmlhw==" + return "eJzsm82O2zgSgO/9FIXsYYFF2sYugj34sMBOMoPxIUkjSc/VoKmSzbFEavhjt/P0A5KSJUuUZMuyuzuIjpZd9bFYPySLvocN7mcQCbpBeQegmU5wBm8+uA/e3AFEqKhkmWaCz+B/dwAA/iUoTbQCKpIEqcYIYinS/N3kDkBigkThDFbkDkCthdQLKnjMVjOISaLwDiBmmERq5qTeAycpVljso/eZlSCFyfJPAjz2mfNYyJTYj4HwyMExpRlVQJbC6FzsPxVIwznjK6CCa8I4SjXJpVRpqkSHbx7ehMA64CpGO8iCFLVk9KDcPscmK5461jFamhIeHb0r4Da43wlZf9eBaJ/3XiDoNdGwIwrwCamx08s46DU2xjEJc0kkGsNcEdF4HtQHohF2a/QEpQktX64pjGG9wKgxrVOo9pLDWlm2IFEkUSkM62bZULXzBziIbhky+163bthXzxsu+44hj4UW/6wSSSH0Iq5bogRLBF8FXvawueeb0CTxcCIGkiTOQWKWoCr8tcVRjwB312D7mlOVRC6m1mSLsETkheeCkEDXhK8wAsU4Rf+CCR6eYE1WI3r0PCUrdDInzbyXmUsy3hfDNUsR3j88jpPsNig5JpOM6uD4FSUJRos4EaT+BV8bZpChpMg1WZ2ZgB4Ov3MzakfFeM4DKiMUw3OVE2tGN+E5C/hXX0w+PIKTdxqB2iuN6QuwmYtUD++tZ+Mip+siH9t2Xqw3YVixUShfgMFyM1margl2tFdzsD7tblqfw1jfDv5kFFm10FEhcfKvVjyx/BMbr/yHixvP9nFgMOXRuwbVPuX9wzrfKT6ZdImyJM3dI4B6WMYztWHiohUzUxuYTz+PUzwkkvBydMC66P+UmtQkrnpbuQoiI+2ewua0hMWHuh/aObSBVmFFdo3lUjmJg6BLvOVeN1a2vYBFwLT9+IQB/GJ/6uCHs8vm7qMX/SzbUiMlcp3bOLP5E6mo7dGqXhmO4o7UE2EmkVrvm8F/J++GRvJZoDvJGnYbJX6c4FcXQMOoX0oEWXqN/BUEUW7nU5zzucPoNFRl0pTI/fUqEeGvNaZsqRcZSrf3fbWx5apTMQmvI8gqRu/xXrfcf6Y4a7h3gLXgxC1yfdHa059y1uWcv+wc+wTyV0sUkno4fBzxMNgrY5E/eyVbwhKyTDCoN5YiHX2YwkgaVmdlj6fu2xrdz62f+Z0SYMq0LgK37gclB6FW5nVIOrWKeuK4oIY0hfXVgoaX9Q37BI5i+PMPRY48bSqODKO1ZEvTVQCC22Oob5EvGsUfRDJhlBUy3ZLEYIXreGxvbXZEHtnRCQ5Mq2PPLsa1RpLoNV0j3YyQ1kZuQsWEJYyvlJZINkGvZFzjqlHqeixJBafGFSirACOoD/96Wfb30ty5bKAiCmeikEMOjkUnLDf2uX0W5NEi0FuDrsbbCUh1eyCPwpIqkyH1LUicom4WYXRmQsE+QpRXUVr0HKbmielFw4OqJOEIGWaUhrsesnZ6fEw5NH84OePkjpblyoDQsaXDgyVkj1IBi5BrFrNm668vkvKF8XXcBh45+8sUrCUkrNgWOZgsrwPhLmAVMyNXpJyXYHnFcsBvgcXAtPVopdVbf+ljt2Z07Xc1+ZbCDy5iEqlO9k4h8npKu+JlAXeFgaWVWwOeqP/GwIjtc99bdd1p75Ln+uGWSW0aGy4YuzvtTNPRHseVSUgoNY3cv7csJEmAErrGyGMpIEoJytwJhxZNJyu5g/OZkCUmQ/skF3TUvd4euNu18hmPL+rFzHksioQPS6IwssvVtdaZmk2nkaBq4i9YTahIp8hXjONUYowSOcUpydjUv19ITIXGBcnYYvvvyX/eTf8xjZjKErK/963Z+x2L8J6V97kuvSFVXPMaK6w/b1E6Nz26DHR2cGfEqEbSgxGiygcVPxyceEWB+25Npvxu3A2g2m/hNamUFll2E1Plmk6iCp2FXYPJVdouU13j5Cdfo+TrvbVQOricCnO4vB1kOb/x3GoNr6WZ6VJMxdH5+tm57qOTMHx527pHnlBhWraKncennQb6jTCbigzXbXcxE5aysNbAdHSdg/eQ5HZz6sIkUo2WhL98/ZpP9bDsOyh6R2gSeH/OySUqF1WgULtVUMdaP3hk1es8cMpNlRPRP7ZAVwS3XVFq6hw88Y/K7z2HT31Knp5h4j+Sp4I6cGMJXuRUO9DW6YWXFk41uxZsHPVOyIvOTT95ESPXBGZzdkzoiI2MAvQg2qlqWS+E2xbhmLyghTfnVKS2ZOcTkf9xpGzfnRvAz9Xgra9FWDEwJ7M9OCLZvX7tCeoBZLnGkjAjdIPNVFk5oZRSNLZIMNpy1ot3LY6TkfIv3GCJ3c1UOUu+TcB8NnolfsSAEcXAXmzAHAhfTsCcjnS7gOlmKgvMUpiWP6cNOU4N1xH/n5njP4a5zlD9iPf1xMlYhWW8Cf9ZUK5TUEYNkJa68QMGyFiFZPwA+VlABhaQvwMAAP//ay+X7Q==" } diff --git a/metricbeat/module/docker/healthcheck/_meta/data.json b/metricbeat/module/docker/healthcheck/_meta/data.json index ce47fb6706e6..ace6af48c147 100644 --- a/metricbeat/module/docker/healthcheck/_meta/data.json +++ b/metricbeat/module/docker/healthcheck/_meta/data.json @@ -1,37 +1,59 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, + "container": { + "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a", + "image": { + "name": "metricbeat_elasticsearch" + }, + "name": "metricbeat_elasticsearch_1_df866b3a7b3d", + "runtime": "docker" + }, "docker": { "container": { - "id": "228848ff8ea238430c4e9cc21deb3175ea5c77483a7f9f4b7c42f7cd6d18c0fb", "labels": { - "com_docker_compose_config-hash": "13347b7d96bf1b3f1ddc089bdef099e42dde235e", + "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", "com_docker_compose_project": "metricbeat", - "com_docker_compose_service": "aerospike", - "com_docker_compose_version": "1.5.0" - }, - "name": "metricbeat_aerospike_1" + "com_docker_compose_service": "elasticsearch", + "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d", + "com_docker_compose_version": "1.23.1", + "license": "Elastic License", + "org_label-schema_build-date": "20181006", + "org_label-schema_license": "GPLv2", + "org_label-schema_name": "elasticsearch", + "org_label-schema_schema-version": "1.0", + "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", + "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker", + "org_label-schema_vendor": "Elastic", + "org_label-schema_version": "6.5.1" + } }, "healthcheck": { "event": { - "end_date": "2017-12-07T07:27:18.738Z", - "exit_code": 1, - "output": "", - "start_date": "2017-12-07T07:27:17.416Z" + "end_date": "2019-02-25T10:59:07.472Z", + "exit_code": 0, + "output": " % Total % Received % Xferd Average Speed Time Time Time Current\n Dload Upload Total Spent Left Speed\n\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r100 338 100 338 0 0 13188 0 --:--:-- --:--:-- --:--:-- 13520\n{\n \"license\" : {\n \"status\" : \"active\",\n \"uid\" : \"ea5a516e-d9ee-4131-8eec-b39741e80869\",\n \"type\" : \"basic\",\n \"issue_date\" : \"2019-02-25T10:18:24.885Z\",\n \"issue_date_in_millis\" : 1551089904885,\n \"max_nodes\" : 1000,\n \"issued_to\" : \"docker-cluster\",\n \"issuer\" : \"elasticsearch\",\n \"start_date_in_millis\" : -1\n }\n}\n", + "start_date": "2019-02-25T10:59:07.342Z" }, - "failingstreak": 1, - "status": "starting" + "failingstreak": 0, + "status": "healthy" } }, + "event": { + "dataset": "docker.healthcheck", + "duration": 115000, + "module": "docker" + }, "metricset": { - "host": "/var/run/docker.sock", - "module": "docker", - "name": "healthcheck", - "rtt": 115 + "name": "healthcheck" + }, + "service": { + "address": "/var/run/docker.sock", + "type": "docker" } } \ No newline at end of file diff --git a/metricbeat/module/docker/healthcheck/data.go b/metricbeat/module/docker/healthcheck/data.go index dc91093c9186..673e15a73230 100644 --- a/metricbeat/module/docker/healthcheck/data.go +++ b/metricbeat/module/docker/healthcheck/data.go @@ -24,49 +24,40 @@ import ( "github.com/docker/docker/api/types" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/docker" ) -func eventsMapping(containers []types.Container, m *MetricSet) []common.MapStr { - var events []common.MapStr +func eventsMapping(r mb.ReporterV2, containers []types.Container, m *MetricSet) { for _, container := range containers { - event := eventMapping(&container, m) - if event != nil { - events = append(events, event) - } + eventMapping(r, &container, m) } - return events } -func eventMapping(cont *types.Container, m *MetricSet) common.MapStr { +func eventMapping(r mb.ReporterV2, cont *types.Container, m *MetricSet) { if !hasHealthCheck(cont.Status) { - return nil + return } container, err := m.dockerClient.ContainerInspect(context.TODO(), cont.ID) if err != nil { - logp.Err("Error inspecting container %v: %v", cont.ID, err) - return nil + logger.Error("Error inspecting container %v: %v", cont.ID, err) + return } // Check if the container has any health check if container.State.Health == nil { - return nil + return } lastEvent := len(container.State.Health.Log) - 1 // Checks if a healthcheck already happened if lastEvent < 0 { - return nil + return } - return common.MapStr{ - mb.ModuleDataKey: common.MapStr{ - "container": docker.NewContainer(cont, m.dedot).ToMapStr(), - }, + fields := common.MapStr{ "status": container.State.Health.Status, "failingstreak": container.State.Health.FailingStreak, "event": common.MapStr{ @@ -76,6 +67,11 @@ func eventMapping(cont *types.Container, m *MetricSet) common.MapStr { "output": container.State.Health.Log[lastEvent].Output, }, } + + r.Event(mb.Event{ + RootFields: docker.NewContainer(cont, m.dedot).ToMapStr(), + MetricSetFields: fields, + }) } // hasHealthCheck detects if healthcheck is available for container diff --git a/metricbeat/module/docker/healthcheck/healthcheck.go b/metricbeat/module/docker/healthcheck/healthcheck.go index 781871587cc1..64c710140e8e 100644 --- a/metricbeat/module/docker/healthcheck/healthcheck.go +++ b/metricbeat/module/docker/healthcheck/healthcheck.go @@ -22,12 +22,15 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/client" + "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/docker" ) +var logger = logp.NewLogger("docker.healthcheck") + func init() { mb.Registry.MustAddMetricSet("docker", "healthcheck", New, mb.WithHostParser(docker.HostParser), @@ -62,11 +65,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch returns a list of all containers as events. // This is based on https://docs.docker.com/engine/reference/api/docker_remote_api_v1.24/#/list-containers. -func (m *MetricSet) Fetch() ([]common.MapStr, error) { +func (m *MetricSet) Fetch(r mb.ReporterV2) { // Fetch a list of all containers. containers, err := m.dockerClient.ContainerList(context.TODO(), types.ContainerListOptions{}) if err != nil { - return nil, err + err = errors.Wrap(err, "failed to get docker containers list") + logger.Error(err) + r.Error(err) + return } - return eventsMapping(containers, m), nil + eventsMapping(r, containers, m) } diff --git a/metricbeat/module/docker/healthcheck/healthcheck_integration_test.go b/metricbeat/module/docker/healthcheck/healthcheck_integration_test.go index 0448a1e3313e..79dfeb00ab77 100644 --- a/metricbeat/module/docker/healthcheck/healthcheck_integration_test.go +++ b/metricbeat/module/docker/healthcheck/healthcheck_integration_test.go @@ -22,13 +22,20 @@ package healthcheck import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) func TestData(t *testing.T) { - f := mbtest.NewEventsFetcher(t, getConfig()) - err := mbtest.WriteEvents(f, t) - if err != nil { + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } diff --git a/metricbeat/module/docker/helper.go b/metricbeat/module/docker/helper.go index 1306b89e8c7f..a4284db25d32 100644 --- a/metricbeat/module/docker/helper.go +++ b/metricbeat/module/docker/helper.go @@ -35,13 +35,18 @@ type Container struct { func (c *Container) ToMapStr() common.MapStr { m := common.MapStr{ - "id": c.ID, - "name": c.Name, - "image": c.Image, + "container": common.MapStr{ + "id": c.ID, + "name": c.Name, + "image": common.MapStr{ + "name": c.Image, + }, + "runtime": "docker", + }, } if len(c.Labels) > 0 { - m["labels"] = c.Labels + m.Put("docker.container.labels", c.Labels) } return m } diff --git a/metricbeat/module/docker/memory/_meta/data.json b/metricbeat/module/docker/memory/_meta/data.json index 10c6587f43c9..8537971aacac 100644 --- a/metricbeat/module/docker/memory/_meta/data.json +++ b/metricbeat/module/docker/memory/_meta/data.json @@ -1,47 +1,64 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, + "container": { + "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a", + "image": { + "name": "metricbeat_elasticsearch" + }, + "name": "metricbeat_elasticsearch_1_df866b3a7b3d", + "runtime": "docker" + }, "docker": { "container": { - "id": "da57ef738524e5a4f6ae17b477f134d30719603db7b96d2c01b6f34010412e66", "labels": { - "build-date": "20170911", - "com_docker_compose_config-hash": "13a74b89a90a6fdcb6bbbc7eb37b7cb0615bdaf8", + "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", "com_docker_compose_project": "metricbeat", "com_docker_compose_service": "elasticsearch", - "com_docker_compose_version": "1.5.0", - "license": "GPLv2", - "maintainer": "Elastic Docker Team \u003cdocker@elastic.co\u003e", - "name": "CentOS Base Image", - "vendor": "CentOS" - }, - "name": "metricbeat_elasticsearch_1" + "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d", + "com_docker_compose_version": "1.23.1", + "license": "Elastic License", + "org_label-schema_build-date": "20181006", + "org_label-schema_license": "GPLv2", + "org_label-schema_name": "elasticsearch", + "org_label-schema_schema-version": "1.0", + "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", + "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker", + "org_label-schema_vendor": "Elastic", + "org_label-schema_version": "6.5.1" + } }, "memory": { "fail": { "count": 0 }, - "limit": 4139122688, + "limit": 16571719680, "rss": { - "pct": 0.08521683713860477, - "total": 352722944 + "pct": 0.02170185104168984, + "total": 359636992 }, "usage": { - "max": 370184192, - "pct": 0.08716433389277685, - "total": 360783872 + "max": 521326592, + "pct": 0.029796111057558028, + "total": 493772800 } } }, + "event": { + "dataset": "docker.memory", + "duration": 115000, + "module": "docker" + }, "metricset": { - "host": "/var/run/docker.sock", - "module": "docker", - "name": "memory", - "rtt": 115 + "name": "memory" + }, + "service": { + "address": "/var/run/docker.sock", + "type": "docker" } } \ No newline at end of file diff --git a/metricbeat/module/docker/memory/data.go b/metricbeat/module/docker/memory/data.go index 6504a64b36b6..95346ac83d37 100644 --- a/metricbeat/module/docker/memory/data.go +++ b/metricbeat/module/docker/memory/data.go @@ -22,19 +22,14 @@ import ( "github.com/elastic/beats/metricbeat/mb" ) -func eventsMapping(memoryDataList []MemoryData) []common.MapStr { - events := []common.MapStr{} +func eventsMapping(r mb.ReporterV2, memoryDataList []MemoryData) { for _, memoryData := range memoryDataList { - events = append(events, eventMapping(&memoryData)) + eventMapping(r, &memoryData) } - return events } -func eventMapping(memoryData *MemoryData) common.MapStr { - event := common.MapStr{ - mb.ModuleDataKey: common.MapStr{ - "container": memoryData.Container.ToMapStr(), - }, +func eventMapping(r mb.ReporterV2, memoryData *MemoryData) { + fields := common.MapStr{ "fail": common.MapStr{ "count": memoryData.Failcnt, }, @@ -49,5 +44,9 @@ func eventMapping(memoryData *MemoryData) common.MapStr { "max": memoryData.MaxUsage, }, } - return event + + r.Event(mb.Event{ + RootFields: memoryData.Container.ToMapStr(), + MetricSetFields: fields, + }) } diff --git a/metricbeat/module/docker/memory/memory.go b/metricbeat/module/docker/memory/memory.go index 466b6ca38d06..240ac448f0b2 100644 --- a/metricbeat/module/docker/memory/memory.go +++ b/metricbeat/module/docker/memory/memory.go @@ -19,12 +19,15 @@ package memory import ( "github.com/docker/docker/client" + "github.com/pkg/errors" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/docker" ) +var logger = logp.NewLogger("docker.memory") + func init() { mb.Registry.MustAddMetricSet("docker", "memory", New, mb.WithHostParser(docker.HostParser), @@ -60,12 +63,15 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } // Fetch creates a list of memory events for each container. -func (m *MetricSet) Fetch() ([]common.MapStr, error) { +func (m *MetricSet) Fetch(r mb.ReporterV2) { stats, err := docker.FetchStats(m.dockerClient, m.Module().Config().Timeout) if err != nil { - return nil, err + err = errors.Wrap(err, "failed to get docker stats") + logger.Error(err) + r.Error(err) + return } memoryStats := m.memoryService.getMemoryStatsList(stats, m.dedot) - return eventsMapping(memoryStats), nil + eventsMapping(r, memoryStats) } diff --git a/metricbeat/module/docker/memory/memory_integration_test.go b/metricbeat/module/docker/memory/memory_integration_test.go index cf29fc4faa16..e77284614f04 100644 --- a/metricbeat/module/docker/memory/memory_integration_test.go +++ b/metricbeat/module/docker/memory/memory_integration_test.go @@ -22,24 +22,20 @@ package memory import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" ) -/* -// TODO: Enable -func TestFetch(t *testing.T) { - f := mbtest.NewEventsFetcher(t, getConfig()) - event, err := f.Fetch() - if err != nil { - t.Fatal(err) +func TestData(t *testing.T) { + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) } - t.Logf(" module : %s metricset : %s event: %+v", f.Module().Name(), f.Name(), event) -}*/ + assert.NotEmpty(t, events) -func TestData(t *testing.T) { - f := mbtest.NewEventsFetcher(t, getConfig()) - err := mbtest.WriteEvents(f, t) - if err != nil { + if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { t.Fatal("write", err) } } diff --git a/metricbeat/module/docker/memory/memory_test.go b/metricbeat/module/docker/memory/memory_test.go index 594665e87a8f..77918dbe2c2e 100644 --- a/metricbeat/module/docker/memory/memory_test.go +++ b/metricbeat/module/docker/memory/memory_test.go @@ -23,11 +23,11 @@ import ( "time" "github.com/docker/docker/api/types" + "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/docker" - - "github.com/stretchr/testify/assert" ) func TestMemoryService_GetMemoryStats(t *testing.T) { @@ -57,12 +57,17 @@ func TestMemoryService_GetMemoryStats(t *testing.T) { memoryRawStats.Stats = memorystats totalRSS := memorystats.MemoryStats.Stats["total_rss"] - expectedEvent := common.MapStr{ - "_module": common.MapStr{ + expectedRootFields := common.MapStr{ + "container": common.MapStr{ + "id": containerID, + "name": "name1", + "image": common.MapStr{ + "name": "image", + }, + "runtime": "docker", + }, + "docker": common.MapStr{ "container": common.MapStr{ - "id": containerID, - "name": "name1", - "image": "image", "labels": common.MapStr{ "label1": "val1", "label2": common.MapStr{ @@ -72,6 +77,8 @@ func TestMemoryService_GetMemoryStats(t *testing.T) { }, }, }, + } + expectedFields := common.MapStr{ "fail": common.MapStr{ "count": memorystats.MemoryStats.Failcnt, }, @@ -88,11 +95,15 @@ func TestMemoryService_GetMemoryStats(t *testing.T) { } //WHEN rawStats := memoryService.getMemoryStats(memoryRawStats, false) - event := eventMapping(&rawStats) + r := &mbtest.CapturingReporterV2{} + eventMapping(r, &rawStats) + events := r.GetEvents() //THEN - assert.True(t, equalEvent(expectedEvent, event)) - t.Logf(" expected : %v", expectedEvent) - t.Logf(" returned : %v", event) + assert.Empty(t, r.GetErrors()) + assert.NotEmpty(t, events) + event := events[0] + assert.Equal(t, expectedRootFields, event.RootFields) + assert.Equal(t, expectedFields, event.MetricSetFields) } func getMemoryStats(read time.Time, number uint64) types.StatsJSON { diff --git a/metricbeat/module/docker/network/_meta/data.json b/metricbeat/module/docker/network/_meta/data.json index ddb3c827a8aa..fba2fae388c3 100644 --- a/metricbeat/module/docker/network/_meta/data.json +++ b/metricbeat/module/docker/network/_meta/data.json @@ -1,23 +1,37 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, + "container": { + "id": "cc78e58acfda4501105dc4de8e3ae218f2da616213e6e3af168c40103829302a", + "image": { + "name": "metricbeat_elasticsearch" + }, + "name": "metricbeat_elasticsearch_1_df866b3a7b3d", + "runtime": "docker" + }, "docker": { "container": { - "id": "452523bf833fd9fd1a8425135b720de4cb9b5a32096deac5b52a97e97bb6d16d", "labels": { - "com_docker_compose_config-hash": "68a840a9e1c606ca1026492e50620e139ca342c585f330025a90f39a5fd32538", + "com_docker_compose_config-hash": "e3e0a2c6e5d1afb741bc8b1ecb09cda0395886b7a3e5084a9fd110be46d70f78", "com_docker_compose_container-number": "1", "com_docker_compose_oneoff": "False", "com_docker_compose_project": "metricbeat", "com_docker_compose_service": "elasticsearch", - "com_docker_compose_version": "1.21.0", - "maintainer": "Elastic Docker Team \u003cdocker@elastic.co\u003e", - "org_label-schema_schema-version": "= 1.0 org.label-schema.name=CentOS Base Image org.label-schema.vendor=CentOS org.label-schema.license=GPLv2 org.label-schema.build-date=20180402" - }, - "name": "metricbeat_elasticsearch_1" + "com_docker_compose_slug": "df866b3a7b3d50c0802350cbe58ee5b34fa32b7f6ba7fe9e48cde2c12dd0201d", + "com_docker_compose_version": "1.23.1", + "license": "Elastic License", + "org_label-schema_build-date": "20181006", + "org_label-schema_license": "GPLv2", + "org_label-schema_name": "elasticsearch", + "org_label-schema_schema-version": "1.0", + "org_label-schema_url": "https://www.elastic.co/products/elasticsearch", + "org_label-schema_vcs-url": "https://github.com/elastic/elasticsearch-docker", + "org_label-schema_vendor": "Elastic", + "org_label-schema_version": "6.5.1" + } }, "network": { "in": { @@ -27,10 +41,10 @@ "packets": 0 }, "inbound": { - "bytes": 61694097, + "bytes": 23047, "dropped": 0, "errors": 0, - "packets": 714036 + "packets": 241 }, "interface": "eth0", "out": { @@ -40,17 +54,23 @@ "packets": 0 }, "outbound": { - "bytes": 69114459, + "bytes": 0, "dropped": 0, "errors": 0, - "packets": 713985 + "packets": 0 } } }, + "event": { + "dataset": "docker.network", + "duration": 115000, + "module": "docker" + }, "metricset": { - "host": "/var/run/docker.sock", - "module": "docker", - "name": "network", - "rtt": 115 + "name": "network" + }, + "service": { + "address": "/var/run/docker.sock", + "type": "docker" } -} +} \ No newline at end of file diff --git a/metricbeat/module/docker/network/_meta/fields.yml b/metricbeat/module/docker/network/_meta/fields.yml index 63d06ce7fb8e..035047eb0918 100644 --- a/metricbeat/module/docker/network/_meta/fields.yml +++ b/metricbeat/module/docker/network/_meta/fields.yml @@ -57,7 +57,6 @@ Total number of outgoing packets. - name: inbound type: group - deprecated: 6.4 description: > Incoming network stats since the container started. fields: @@ -80,7 +79,6 @@ Total number of incoming packets. - name: outbound type: group - deprecated: 6.4 description: > Outgoing network stats since the container started. fields: diff --git a/metricbeat/module/docker/network/data.go b/metricbeat/module/docker/network/data.go index 80273317613e..30f6bd34b937 100644 --- a/metricbeat/module/docker/network/data.go +++ b/metricbeat/module/docker/network/data.go @@ -29,19 +29,18 @@ func eventsMapping(r mb.ReporterV2, netsStatsList []NetStats) { } func eventMapping(r mb.ReporterV2, stats *NetStats) { - // Deprecated fields r.Event(mb.Event{ - ModuleFields: common.MapStr{ - "container": stats.Container.ToMapStr(), - }, + RootFields: stats.Container.ToMapStr(), MetricSetFields: common.MapStr{ "interface": stats.NameInterface, + // Deprecated "in": common.MapStr{ "bytes": stats.RxBytes, "dropped": stats.RxDropped, "errors": stats.RxErrors, "packets": stats.RxPackets, }, + // Deprecated "out": common.MapStr{ "bytes": stats.TxBytes, "dropped": stats.TxDropped, diff --git a/metricbeat/module/docker/network/network.go b/metricbeat/module/docker/network/network.go index 67dd2423286f..78b908dd47b4 100644 --- a/metricbeat/module/docker/network/network.go +++ b/metricbeat/module/docker/network/network.go @@ -19,11 +19,15 @@ package network import ( "github.com/docker/docker/client" + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/logp" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/docker" ) +var logger = logp.NewLogger("docker.network") + func init() { mb.Registry.MustAddMetricSet("docker", "network", New, mb.WithHostParser(docker.HostParser), @@ -64,6 +68,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(r mb.ReporterV2) { stats, err := docker.FetchStats(m.dockerClient, m.Module().Config().Timeout) if err != nil { + err = errors.Wrap(err, "failed to get docker stats") + logger.Error(err) r.Error(err) return } diff --git a/metricbeat/tests/system/test_docker.py b/metricbeat/tests/system/test_docker.py index c757eea06f64..294031744315 100644 --- a/metricbeat/tests/system/test_docker.py +++ b/metricbeat/tests/system/test_docker.py @@ -136,7 +136,7 @@ def test_memory_fields(self): @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_network_fields(self): """ - test info fields + test network fields """ self.render_config_template(modules=[{ "name": "docker",