From e311d5ef8858e0df7d2b2a361053c581851534ba Mon Sep 17 00:00:00 2001 From: Saggi Mizrahi Date: Mon, 30 Oct 2023 14:56:10 +0000 Subject: [PATCH] Use protobuf reflection for serialization This is used instead of regular Go relection because it is harder to map it to the table schema. The protobuf is the canonical representation making converison easier. --- tools/mc2bq/pkg/export/export.go | 11 +- tools/mc2bq/pkg/export/v1.go | 6 +- .../migrationcenter_v1_latest.schema.json | 2674 ++++++++--------- tools/mc2bq/pkg/schema/schema.go | 285 +- tools/mc2bq/pkg/schema/schema_test.go | 122 +- .../pkg/schema/testdata/exporter.json.golden | 66 +- 6 files changed, 1538 insertions(+), 1626 deletions(-) diff --git a/tools/mc2bq/pkg/export/export.go b/tools/mc2bq/pkg/export/export.go index 46777d1..c3de311 100644 --- a/tools/mc2bq/pkg/export/export.go +++ b/tools/mc2bq/pkg/export/export.go @@ -30,6 +30,7 @@ import ( "golang.org/x/sync/errgroup" "google.golang.org/api/iterator" "google.golang.org/api/option" + "google.golang.org/protobuf/reflect/protoreflect" "github.com/GoogleCloudPlatform/migrationcenter-utils/tools/mc2bq/pkg/gapiutil" "github.com/GoogleCloudPlatform/migrationcenter-utils/tools/mc2bq/pkg/mcutil" @@ -187,7 +188,7 @@ type iterable[T any] interface { Next() (T, error) } -type objectReader[T any] struct { +type objectReader[T protoreflect.ProtoMessage] struct { schema bigquery.Schema it iterable[T] serializer func(obj T) ([]byte, error) @@ -197,9 +198,9 @@ type objectReader[T any] struct { bytesRead uint64 } -func newObjectReader[T any](it iterable[*T], root string, schema bigquery.Schema) *objectReader[*T] { - return &objectReader[*T]{ - serializer: exporterschema.NewSerializer[*T](root, schema), +func newObjectReader[T protoreflect.ProtoMessage](it iterable[T], root string, schema bigquery.Schema) *objectReader[T] { + return &objectReader[T]{ + serializer: exporterschema.NewSerializer[T](root, schema), it: it, schema: schema, } @@ -247,7 +248,7 @@ func (r *objectReader[T]) Read(buf []byte) (int, error) { return n, nil } -func newMigrationCenterLoadSource[T any](r *objectReader[T]) bigquery.LoadSource { +func newMigrationCenterLoadSource[T protoreflect.ProtoMessage](r *objectReader[T]) bigquery.LoadSource { // Creating a full blown bigquery.LoadSource requires a lot of low level big query operations. // To save on time we create a ReaderSource and feed it the assets as a json stream. src := bigquery.NewReaderSource(r) diff --git a/tools/mc2bq/pkg/export/v1.go b/tools/mc2bq/pkg/export/v1.go index d30d439..e8b5d87 100644 --- a/tools/mc2bq/pkg/export/v1.go +++ b/tools/mc2bq/pkg/export/v1.go @@ -36,7 +36,7 @@ func (mc *MCv1) AssetSource(ctx context.Context, pal mcutil.ProjectAndLocation) Parent: pal.String(), PageSize: 1000, }) - r := newObjectReader[migrationcenterpb.Asset](it, "asset", mc.schema.AssetTable) + r := newObjectReader[*migrationcenterpb.Asset](it, "asset", mc.schema.AssetTable) src := newMigrationCenterLoadSource(r) return &struct { bigquery.LoadSource @@ -49,7 +49,7 @@ func (mc *MCv1) GroupSource(ctx context.Context, pal mcutil.ProjectAndLocation) Parent: pal.String(), PageSize: 1000, }) - r := newObjectReader[migrationcenterpb.Group](it, "asset", mc.schema.GroupTable) + r := newObjectReader[*migrationcenterpb.Group](it, "group", mc.schema.GroupTable) src := newMigrationCenterLoadSource(r) return &struct { bigquery.LoadSource @@ -62,7 +62,7 @@ func (mc *MCv1) PreferenceSetSource(ctx context.Context, pal mcutil.ProjectAndLo Parent: pal.String(), PageSize: 1000, }) - r := newObjectReader[migrationcenterpb.PreferenceSet](it, "asset", mc.schema.PreferenceSetTable) + r := newObjectReader[*migrationcenterpb.PreferenceSet](it, "preference_set", mc.schema.PreferenceSetTable) src := newMigrationCenterLoadSource(r) return &struct { bigquery.LoadSource diff --git a/tools/mc2bq/pkg/schema/migrationcenter_v1_latest.schema.json b/tools/mc2bq/pkg/schema/migrationcenter_v1_latest.schema.json index 0cd1066..38b66ff 100644 --- a/tools/mc2bq/pkg/schema/migrationcenter_v1_latest.schema.json +++ b/tools/mc2bq/pkg/schema/migrationcenter_v1_latest.schema.json @@ -1,1449 +1,1419 @@ -{"asset_table":[ - { - "name": "name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "create_time", - "type": "TIMESTAMP", - "mode": "NULLABLE" - }, - { - "name": "update_time", - "type": "TIMESTAMP", - "mode": "NULLABLE" - }, - { - "name": "labels", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "key", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "value", - "type": "STRING", - "mode": "NULLABLE" - } - ] - }, - { - "name": "attributes", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "key", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "value", - "type": "STRING", - "mode": "NULLABLE" - } - ] - }, - { - "name": "machine_details", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "uuid", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "machine_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "create_time", - "type": "TIMESTAMP", - "mode": "NULLABLE" - }, - { - "name": "core_count", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "memory_mb", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "power_state", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "architecture", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "cpu_architecture", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "cpu_name", +{ + "asset_table": [ + { + "name": "name", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "vendor", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "cpu_thread_count", - "type": "INTEGER", + }, + { + "name": "create_time", + "type": "TIMESTAMP", "mode": "NULLABLE" - }, - { - "name": "cpu_socket_count", - "type": "INTEGER", + }, + { + "name": "update_time", + "type": "TIMESTAMP", "mode": "NULLABLE" - }, - { - "name": "bios", + }, + { + "name": "labels", "type": "RECORD", - "mode": "NULLABLE", + "mode": "REPEATED", "fields": [ - { - "name": "bios_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "id", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "manufacturer", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "version", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "release_date", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "year", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "month", - "type": "INTEGER", + { + "name": "key", + "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "day", - "type": "INTEGER", + }, + { + "name": "value", + "type": "STRING", "mode": "NULLABLE" - } - ] - }, - { - "name": "smbios_uuid", - "type": "STRING", - "mode": "NULLABLE" - } + } ] - }, - { - "name": "firmware_type", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "hyperthreading", - "type": "STRING", - "mode": "NULLABLE" - } - ] - }, - { - "name": "guest_os", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "os_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "family", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "version", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "config", + }, + { + "name": "attributes", "type": "RECORD", - "mode": "NULLABLE", + "mode": "REPEATED", "fields": [ - { - "name": "issue", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "fstab", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "spec", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "file", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "vfstype", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "mntops", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "freq", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "passno", - "type": "INTEGER", - "mode": "NULLABLE" - } - ] - } - ] - }, - { - "name": "hosts", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "ip", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "host_names", - "type": "STRING", - "mode": "REPEATED" - } - ] - } - ] - }, - { - "name": "nfs_exports", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "export_directory", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "hosts", - "type": "STRING", - "mode": "REPEATED" - } - ] - } - ] - }, - { - "name": "selinux_mode", - "type": "STRING", - "mode": "NULLABLE" - } + { + "name": "key", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "value", + "type": "STRING", + "mode": "NULLABLE" + } ] - }, - { - "name": "runtime", + }, + { + "name": "machine_details", "type": "RECORD", "mode": "NULLABLE", "fields": [ - { - "name": "services", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "service_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "state", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "start_mode", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "exe_path", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "cmdline", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "pid", - "type": "INTEGER", - "mode": "NULLABLE" - } - ] - } - ] - }, - { - "name": "processes", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "pid", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "exe_path", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "cmdline", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "user", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "attributes", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "key", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "value", - "type": "STRING", - "mode": "NULLABLE" - } - ] - } - ] - } - ] - }, - { - "name": "network", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "scan_time", + { + "name": "uuid", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "machine_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "create_time", "type": "TIMESTAMP", "mode": "NULLABLE" - }, - { - "name": "connections", + }, + { + "name": "core_count", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "memory_mb", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "power_state", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "architecture", "type": "RECORD", "mode": "NULLABLE", "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "protocol", + { + "name": "cpu_architecture", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "local_ip_address", + }, + { + "name": "cpu_name", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "local_port", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "remote_ip_address", + }, + { + "name": "vendor", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "remote_port", + }, + { + "name": "cpu_thread_count", "type": "INTEGER", "mode": "NULLABLE" - }, - { - "name": "state", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "pid", + }, + { + "name": "cpu_socket_count", "type": "INTEGER", "mode": "NULLABLE" - }, - { - "name": "process_name", + }, + { + "name": "bios", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "bios_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "id", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "manufacturer", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "version", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "release_date", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "year", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "month", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "day", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + }, + { + "name": "smbios_uuid", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "firmware_type", "type": "STRING", "mode": "NULLABLE" - } - ] - } - ] - } - ] - }, - { - "name": "last_boot_time", - "type": "TIMESTAMP", - "mode": "NULLABLE" - }, - { - "name": "domain", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "machine_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "installed_apps", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "application_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "vendor", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "install_time", - "type": "TIMESTAMP", - "mode": "NULLABLE" - }, - { - "name": "path", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "version", - "type": "STRING", - "mode": "NULLABLE" - } - ] - } - ] - }, - { - "name": "open_file_list", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "command", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "user", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "file_type", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "file_path", - "type": "STRING", - "mode": "NULLABLE" - } + }, + { + "name": "hyperthreading", + "type": "STRING", + "mode": "NULLABLE" + } ] - } - ] - } - ] - } - ] - }, - { - "name": "network", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "primary_ip_address", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "public_ip_address", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "primary_mac_address", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "adapters", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "adapter_type", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "mac_address", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "addresses", + }, + { + "name": "guest_os", "type": "RECORD", "mode": "NULLABLE", "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "ip_address", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "subnet_mask", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "bcast", + { + "name": "os_name", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "fqdn", + }, + { + "name": "family", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "assignment", + }, + { + "name": "version", "type": "STRING", "mode": "NULLABLE" - } - ] - } + }, + { + "name": "config", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "issue", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "fstab", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "spec", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "file", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "vfstype", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "mntops", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "freq", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "passno", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "hosts", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "ip", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "host_names", + "type": "STRING", + "mode": "REPEATED" + } + ] + } + ] + }, + { + "name": "nfs_exports", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "export_directory", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "hosts", + "type": "STRING", + "mode": "REPEATED" + } + ] + } + ] + }, + { + "name": "selinux_mode", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "runtime", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "services", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "service_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "state", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "start_mode", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "exe_path", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "cmdline", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "pid", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "processes", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "pid", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "exe_path", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "cmdline", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "user", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "attributes", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "key", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "value", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } + ] + } + ] + }, + { + "name": "network", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "scan_time", + "type": "TIMESTAMP", + "mode": "NULLABLE" + }, + { + "name": "connections", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "protocol", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "local_ip_address", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "local_port", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "remote_ip_address", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "remote_port", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "state", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "pid", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "process_name", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } + ] + } + ] + }, + { + "name": "last_boot_time", + "type": "TIMESTAMP", + "mode": "NULLABLE" + }, + { + "name": "domain", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "machine_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "installed_apps", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "application_name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "vendor", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "install_time", + "type": "TIMESTAMP", + "mode": "NULLABLE" + }, + { + "name": "path", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "version", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "open_file_list", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "command", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "user", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "file_type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "file_path", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } + ] + } + ] + } ] - } - ] - } - ] - } - ] - }, - { - "name": "disks", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "total_capacity_bytes", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "total_free_bytes", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "disks", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "capacity_bytes", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "free_bytes", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "disk_label", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "disk_label_type", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "interface_type", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "partitions", + }, + { + "name": "network", "type": "RECORD", "mode": "NULLABLE", "fields": [ - { - "name": "entries", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "type", + { + "name": "primary_ip_address", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "file_system", + }, + { + "name": "public_ip_address", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "mount_point", + }, + { + "name": "primary_mac_address", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "capacity_bytes", + }, + { + "name": "adapters", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "adapter_type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "mac_address", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "addresses", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "ip_address", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "subnet_mask", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "bcast", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "fqdn", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "assignment", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } + ] + } + ] + } + ] + } + ] + }, + { + "name": "disks", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "total_capacity_bytes", "type": "INTEGER", "mode": "NULLABLE" - }, - { - "name": "free_bytes", + }, + { + "name": "total_free_bytes", "type": "INTEGER", "mode": "NULLABLE" - }, - { - "name": "uuid", - "type": "STRING", - "mode": "NULLABLE" - } - ] - } + }, + { + "name": "disks", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "capacity_bytes", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "free_bytes", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "disk_label", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "disk_label_type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "interface_type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "partitions", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "entries", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "file_system", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "mount_point", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "capacity_bytes", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "free_bytes", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "uuid", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "hw_address", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "vmware", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "backing_type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "shared", + "type": "BOOLEAN", + "mode": "NULLABLE" + }, + { + "name": "vmdk_mode", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "rdm_compatibility", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } + ] + } + ] + } ] - }, - { - "name": "hw_address", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "vmware", + }, + { + "name": "platform", "type": "RECORD", "mode": "NULLABLE", "fields": [ - { - "name": "backing_type", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "shared", - "type": "BOOLEAN", - "mode": "NULLABLE" - }, - { - "name": "vmdk_mode", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "rdm_compatibility", - "type": "STRING", - "mode": "NULLABLE" - } + { + "name": "vmware_details", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "vcenter_version", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "esx_version", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "osid", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "vcenter_folder", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "vcenter_uri", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "vcenter_vm_id", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "aws_ec2_details", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "machine_type_label", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "location", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "azure_vm_details", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "machine_type_label", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "location", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "provisioning_state", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "generic_details", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "location", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "physical_details", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "location", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } ] - } - ] - } + } ] - } - ] - }, - { - "name": "platform", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "vmware_details", + }, + { + "name": "insight_list", "type": "RECORD", "mode": "NULLABLE", "fields": [ - { - "name": "vcenter_version", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "esx_version", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "osid", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "vcenter_folder", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "vcenter_uri", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "vcenter_vm_id", - "type": "STRING", - "mode": "NULLABLE" - } - ] - }, - { - "name": "aws_ec2_details", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "machine_type_label", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "location", - "type": "STRING", - "mode": "NULLABLE" - } - ] - }, - { - "name": "azure_vm_details", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "machine_type_label", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "location", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "provisioning_state", - "type": "STRING", - "mode": "NULLABLE" - } - ] - }, - { - "name": "generic_details", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "location", - "type": "STRING", - "mode": "NULLABLE" - } - ] - }, - { - "name": "physical_details", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "location", - "type": "STRING", - "mode": "NULLABLE" - } - ] - } - ] - } - ] - }, - { - "name": "insight_list", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "insights", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "migration_insight", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "fit", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "fit_level", - "type": "STRING", - "mode": "NULLABLE" - } - ] - }, - { - "name": "compute_engine_target", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "shape", + { + "name": "insights", "type": "RECORD", - "mode": "NULLABLE", + "mode": "REPEATED", "fields": [ - { - "name": "memory_mb", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "physical_core_count", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "logical_core_count", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "series", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "machine_type", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "storage", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "type", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "size_gb", - "type": "INTEGER", - "mode": "NULLABLE" - } - ] - } + { + "name": "migration_insight", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "fit", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "fit_level", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "compute_engine_target", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "shape", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "memory_mb", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "physical_core_count", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "logical_core_count", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "series", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "machine_type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "storage", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "size_gb", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + } + ] + } + ] + } + ] + }, + { + "name": "generic_insight", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "message_id", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "default_message", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "additional_information", + "type": "STRING", + "mode": "REPEATED" + } + ] + } ] - } - ] - } - ] - }, - { - "name": "generic_insight", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "message_id", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "default_message", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "additional_information", - "type": "STRING", - "mode": "REPEATED" - } - ] - }, - { - "name": "software_insight", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "detected_software", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "software_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "software_family", - "type": "STRING", - "mode": "NULLABLE" - } - ] - } - ] - } - ] - }, - { - "name": "update_time", - "type": "TIMESTAMP", - "mode": "NULLABLE" - } - ] - }, - { - "name": "performance_data", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "daily_resource_usage_aggregations", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "date", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "year", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "month", - "type": "INTEGER", - "mode": "NULLABLE" - }, - { - "name": "day", - "type": "INTEGER", - "mode": "NULLABLE" - } - ] - }, - { - "name": "cpu", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "utilization_percentage", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "average", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "median", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "nintey_fifth_percentile", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "peak", - "type": "FLOAT", - "mode": "NULLABLE" - } - ] - } - ] - }, - { - "name": "memory", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "utilization_percentage", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "average", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "median", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "nintey_fifth_percentile", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "peak", - "type": "FLOAT", - "mode": "NULLABLE" - } - ] - } - ] - }, - { - "name": "network", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "ingress_bps", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "average", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "median", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "nintey_fifth_percentile", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "peak", - "type": "FLOAT", - "mode": "NULLABLE" - } - ] - }, - { - "name": "egress_bps", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "average", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "median", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "nintey_fifth_percentile", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "peak", - "type": "FLOAT", + }, + { + "name": "update_time", + "type": "TIMESTAMP", "mode": "NULLABLE" - } - ] - } + } ] - }, - { - "name": "disk", + }, + { + "name": "performance_data", "type": "RECORD", "mode": "NULLABLE", "fields": [ - { - "name": "iops", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "average", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "median", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "nintey_fifth_percentile", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "peak", - "type": "FLOAT", - "mode": "NULLABLE" - } - ] - } + { + "name": "daily_resource_usage_aggregations", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "date", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "year", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "month", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "day", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + }, + { + "name": "cpu", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "utilization_percentage", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "average", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "median", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "nintey_fifth_percentile", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "peak", + "type": "FLOAT", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "memory", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "utilization_percentage", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "average", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "median", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "nintey_fifth_percentile", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "peak", + "type": "FLOAT", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "network", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "ingress_bps", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "average", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "median", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "nintey_fifth_percentile", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "peak", + "type": "FLOAT", + "mode": "NULLABLE" + } + ] + }, + { + "name": "egress_bps", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "average", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "median", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "nintey_fifth_percentile", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "peak", + "type": "FLOAT", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "disk", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "iops", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "average", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "median", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "nintey_fifth_percentile", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "peak", + "type": "FLOAT", + "mode": "NULLABLE" + } + ] + } + ] + } + ] + } ] - } - ] - } - ] - }, - { - "name": "sources", - "type": "STRING", - "mode": "REPEATED" - }, - { - "name": "assigned_groups", - "type": "STRING", - "mode": "REPEATED" - } -],"group_table":[ - { - "name": "name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "create_time", - "type": "TIMESTAMP", - "mode": "NULLABLE" - }, - { - "name": "update_time", - "type": "TIMESTAMP", - "mode": "NULLABLE" - }, - { - "name": "labels", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "key", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "value", - "type": "STRING", - "mode": "NULLABLE" - } - ] - }, - { - "name": "display_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "description", - "type": "STRING", - "mode": "NULLABLE" - } -],"preference_set_table":[ - { - "name": "name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "create_time", - "type": "TIMESTAMP", - "mode": "NULLABLE" - }, - { - "name": "update_time", - "type": "TIMESTAMP", - "mode": "NULLABLE" - }, - { - "name": "display_name", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "description", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "virtual_machine_preferences", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "business_goal", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "target_product", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "region_preferences", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "preferred_regions", + }, + { + "name": "sources", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "assigned_groups", "type": "STRING", "mode": "REPEATED" - } - ] - }, - { - "name": "commitment_plan", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "sizing_optimization_strategy", - "type": "STRING", - "mode": "NULLABLE" - }, - { - "name": "compute_engine_preferences", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "persistent_disk_type", + } + ], + "group_table": [ + { + "name": "name", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "machine_preferences", + }, + { + "name": "create_time", + "type": "TIMESTAMP", + "mode": "NULLABLE" + }, + { + "name": "update_time", + "type": "TIMESTAMP", + "mode": "NULLABLE" + }, + { + "name": "labels", "type": "RECORD", - "mode": "NULLABLE", + "mode": "REPEATED", "fields": [ - { - "name": "allowed_machine_series", - "type": "RECORD", - "mode": "REPEATED", - "fields": [ - { - "name": "code", + { + "name": "key", "type": "STRING", "mode": "NULLABLE" - } - ] - } + }, + { + "name": "value", + "type": "STRING", + "mode": "NULLABLE" + } ] - }, - { - "name": "license_type", + }, + { + "name": "display_name", "type": "STRING", "mode": "NULLABLE" - } - ] - }, - { - "name": "vmware_engine_preferences", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "cpu_overcommit_ratio", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "memory_overcommit_ratio", - "type": "FLOAT", - "mode": "NULLABLE" - }, - { - "name": "storage_deduplication_compression_ratio", - "type": "FLOAT", + }, + { + "name": "description", + "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "commitment_plan", + } + ], + "preference_set_table": [ + { + "name": "name", "type": "STRING", "mode": "NULLABLE" - } - ] - }, - { - "name": "sole_tenancy_preferences", - "type": "RECORD", - "mode": "NULLABLE", - "fields": [ - { - "name": "cpu_overcommit_ratio", - "type": "FLOAT", + }, + { + "name": "create_time", + "type": "TIMESTAMP", + "mode": "NULLABLE" + }, + { + "name": "update_time", + "type": "TIMESTAMP", "mode": "NULLABLE" - }, - { - "name": "host_maintenance_policy", + }, + { + "name": "display_name", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "commitment_plan", + }, + { + "name": "description", "type": "STRING", "mode": "NULLABLE" - }, - { - "name": "node_types", + }, + { + "name": "virtual_machine_preferences", "type": "RECORD", - "mode": "REPEATED", + "mode": "NULLABLE", "fields": [ - { - "name": "node_name", - "type": "STRING", - "mode": "NULLABLE" - } + { + "name": "target_product", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "region_preferences", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "preferred_regions", + "type": "STRING", + "mode": "REPEATED" + } + ] + }, + { + "name": "commitment_plan", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "sizing_optimization_strategy", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "compute_engine_preferences", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "machine_preferences", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "allowed_machine_series", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "code", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "license_type", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "vmware_engine_preferences", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "cpu_overcommit_ratio", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "memory_overcommit_ratio", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "storage_deduplication_compression_ratio", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "commitment_plan", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "sole_tenancy_preferences", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "cpu_overcommit_ratio", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "host_maintenance_policy", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "commitment_plan", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "node_types", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "node_name", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } + ] + } ] - } - ] - } + } ] - } -]} +} diff --git a/tools/mc2bq/pkg/schema/schema.go b/tools/mc2bq/pkg/schema/schema.go index dfd5709..13a09eb 100644 --- a/tools/mc2bq/pkg/schema/schema.go +++ b/tools/mc2bq/pkg/schema/schema.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "reflect" - "regexp" "sort" "strings" "time" @@ -116,9 +115,9 @@ func init() { // NewSerializer creates a type safe serializer for type T. // It's the callers responsibility to make sure that the schema and type T match. // root describes the root node string that will appear in errors. -func NewSerializer[T any](root string, schema bigquery.Schema) func(obj T) ([]byte, error) { +func NewSerializer[T protoreflect.ProtoMessage](root string, schema bigquery.Schema) func(obj T) ([]byte, error) { return func(obj T) ([]byte, error) { - return SerializeObjectToBigQuery(obj, root, schema) + return SerializeObjectToBigQuery(obj.ProtoReflect(), root, schema) } } @@ -127,7 +126,7 @@ func NewSerializer[T any](root string, schema bigquery.Schema) func(obj T) ([]by // The function should never return an error in production, if it fails it's a bug // resulting from a mismatch between the API object and the BigQuery schema and both are generated // from the same protobuf. -func SerializeObjectToBigQuery(obj any, root string, schema bigquery.Schema) ([]byte, error) { +func SerializeObjectToBigQuery(obj protoreflect.Message, root string, schema bigquery.Schema) ([]byte, error) { serializedObj, err := normalizeToSchema( obj, &bigquery.FieldSchema{ @@ -148,157 +147,177 @@ func SerializeObjectToBigQuery(obj any, root string, schema bigquery.Schema) ([] return append(res, '\n'), err } -// normalizeToSchema normalizes obj to according to the provided schema. -// Specifically converts structs to maps, and maps to key-value lists. -func normalizeToSchema(obj any, schema *bigquery.FieldSchema) (any, error) { - // short circuit for nil values - if obj == nil { - return nil, nil - } +func fieldConversionError(kind protoreflect.Kind, bqtype bigquery.FieldType) error { + return fmt.Errorf("convert proto kind %q to bigquery type %q", kind.String(), bqtype) +} - objValue := reflect.ValueOf(obj) - if objValue.Type().Kind() == reflect.Ptr { - if objValue.IsNil() { - return nil, nil +func convertProtoValueToBQType(value protoreflect.Value, fd protoreflect.FieldDescriptor, schema *bigquery.FieldSchema) (any, error) { + bqtype := schema.Type + kind := fd.Kind() + switch kind { + case protoreflect.BoolKind: + switch bqtype { + case bigquery.BooleanFieldType: + return value.Bool(), nil + } + case protoreflect.Int32Kind, + protoreflect.Sint32Kind, + protoreflect.Sfixed32Kind, + protoreflect.Int64Kind, + protoreflect.Sint64Kind, + protoreflect.Sfixed64Kind: + switch bqtype { + case bigquery.IntegerFieldType: + return value.Int(), nil + } + case protoreflect.Uint32Kind, + protoreflect.Fixed32Kind, + protoreflect.Uint64Kind, + protoreflect.Fixed64Kind: + switch bqtype { + case bigquery.IntegerFieldType: + return value.Uint(), nil + } + case protoreflect.StringKind: + switch bqtype { + case bigquery.StringFieldType: + return value.String(), nil + } + case protoreflect.FloatKind, + protoreflect.DoubleKind: + return value.Float(), nil + case protoreflect.MessageKind: + return normalizeToSchema(value.Message(), schema) + case protoreflect.EnumKind: + switch bqtype { + case bigquery.StringFieldType: + return protoimpl.X.EnumStringOf(fd.Enum(), value.Enum()), nil } - - objValue = objValue.Elem() } - objType := objValue.Type() - if schema.Repeated { - switch objType.Kind() { - case reflect.Slice: - // This is an array, we need to just normalize each item in the array. - itemSchema := *schema - itemSchema.Repeated = false // we are serializing the item so it's not repeated - result := make([]any, objValue.Len()) - for i := 0; i < objValue.Len(); i++ { - var err error - result[i], err = normalizeToSchema(objValue.Index(i).Interface(), &itemSchema) - if err != nil { - return nil, wrapWithSerializeError(fmt.Sprintf("%s[%d]", schema.Name, i), err) - } - } - return result, nil - case reflect.Map: - // This is a dynamic map, those are not supported by BigQuery so we need to convert it to an - // array in the form of [struct{key: string, value: T}, ...] - if len(schema.Schema) != 2 { - return nil, wrapWithSerializeError(schema.Name, errors.New("schema for dynamic map is invalid")) - } + return nil, fieldConversionError(kind, bqtype) +} - if schema.Schema[1].Name != "value" { - return nil, wrapWithSerializeError(schema.Name, errors.New("schema for dynamic map is invalid")) - } - keySchema := schema.Schema[1] - - result := []map[string]any{} - iter := objValue.MapRange() - for iter.Next() { - var err error - item := map[string]any{} - key := iter.Key() - value := iter.Value() - - item["key"] = key.String() - item["value"], err = normalizeToSchema(value.Interface(), keySchema) - if err != nil { - return nil, wrapWithSerializeError(fmt.Sprintf("%s[%q]", schema.Name, key), err) - } - - result = append(result, item) - } +func normalizeMessageField(obj protoreflect.Message, col *bigquery.FieldSchema) (any, error) { + fieldDesc := obj.Descriptor().Fields().ByName(protoreflect.Name(col.Name)) + if fieldDesc == nil { + return nil, fmt.Errorf("field %q not found", col.Name) + } + value := obj.Get(fieldDesc) - // Sort by key to make the output list stable, otherwise each export - // might reorder the items making diffing harder - sort.SliceStable(result, func(i, j int) bool { - return strings.Compare(result[i]["key"].(string), result[j]["key"].(string)) < 0 - }) - return result, nil - default: - return nil, wrapWithSerializeError(schema.Name, fmt.Errorf("schema does not match object: %d", objType.Kind())) + if fieldDesc.Cardinality() != protoreflect.Repeated { + res, err := convertProtoValueToBQType(value, fieldDesc, col) + if err != nil { + return nil, wrapWithSerializeError(col.Name, err) } + + return res, nil } - switch schema.Type { - case bigquery.TimestampFieldType: - switch obj := objValue.Interface().(type) { - case timestamppb.Timestamp: - return time.Unix(obj.Seconds, int64(obj.Nanos)).Format(time.RFC3339), nil - default: - return nil, wrapWithSerializeError(schema.Name, fmt.Errorf("convert field of type %q to timestamp", reflect.TypeOf(obj).String())) + if fieldDesc.IsList() { + lst := value.List() + if lst.Len() == 0 { + return nil, nil } - case bigquery.StringFieldType: - switch obj := objValue.Interface().(type) { - case string: - return obj, nil - case protoreflect.Enum: - return protoimpl.X.EnumStringOf(obj.Descriptor(), protoreflect.EnumNumber(obj.Number())), nil - default: - return nil, wrapWithSerializeError(schema.Name, fmt.Errorf("convert field of type %q to string", reflect.TypeOf(obj).String())) + + tmpList := make([]any, lst.Len()) + itemSchema := *col + itemSchema.Repeated = false // we are serializing the item so it's not repeated + for i := 0; i < len(tmpList); i++ { + var err error + item := lst.Get(i) + tmpList[i], err = convertProtoValueToBQType(item, fieldDesc, &itemSchema) + if err != nil { + return nil, wrapWithSerializeError(fmt.Sprintf("%s[%d]", col.Name, i), err) + } } - case bigquery.IntegerFieldType: - switch obj := objValue.Interface().(type) { - case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: - return obj, nil - default: - return nil, wrapWithSerializeError(schema.Name, fmt.Errorf("convert field of type %q to integer", reflect.TypeOf(obj).String())) + + return tmpList, nil + } + + if fieldDesc.IsMap() { + // Create a dynamic map. Because maps are not supported by BigQuery. + // We need to convert it to an array in the form of [struct{key: string, value: T}, ...] + + if len(col.Schema) != 2 { + return nil, wrapWithSerializeError(col.Name, errors.New("schema for dynamic map is invalid")) } - case bigquery.FloatFieldType: - switch obj := objValue.Interface().(type) { - case float32, float64: - return obj, nil - default: - return nil, wrapWithSerializeError(schema.Name, fmt.Errorf("convert field of type %q to float", reflect.TypeOf(obj).String())) + + if col.Schema[1].Name != "value" { + return nil, wrapWithSerializeError(col.Name, errors.New("schema for dynamic map is invalid")) } - case bigquery.BooleanFieldType: - switch obj := objValue.Interface().(type) { - case bool: - return obj, nil - default: - return nil, wrapWithSerializeError(schema.Name, fmt.Errorf("convert field of type %q to bool", reflect.TypeOf(obj).String())) + + if fieldDesc.MapKey().Kind() != protoreflect.StringKind { + return nil, wrapWithSerializeError(col.Name, errors.New("schema for dynamic map is invalid")) } - case bigquery.RecordFieldType: - // This is a static map - result := map[string]any{} - for _, col := range schema.Schema { - var err error - // The protobuf and table fields are in snake case but the struct fields are in camel case - fieldName := snakeCaseToCamelCase(col.Name) - value := objValue.FieldByName(fieldName) - if !value.IsValid() { - // The schema might be newer than the exporter, just ignore - continue - } - result[col.Name], err = normalizeToSchema(value.Interface(), col) + + dict := value.Map() + keySchema := col.Schema[1] + result := []map[string]any{} + + var err error + dict.Range(func(mk protoreflect.MapKey, v protoreflect.Value) bool { + item := map[string]any{} + key := mk.String() + item["key"] = key + item["value"], err = convertProtoValueToBQType(v, fieldDesc.MapValue(), keySchema) if err != nil { - return nil, wrapWithSerializeError(schema.Name, err) + err = wrapWithSerializeError(fmt.Sprintf("%s[%q]", col.Name, key), err) + return false } + + result = append(result, item) + return true + }) + + if err != nil { + return nil, err } + + // Sort by key to make the output list stable, otherwise each export + // might reorder the items making diffing harder + sort.SliceStable(result, func(i, j int) bool { + return strings.Compare(result[i]["key"].(string), result[j]["key"].(string)) < 0 + }) + + if len(result) == 0 { + return nil, nil + } + return result, nil - default: - return nil, wrapWithSerializeError(schema.Name, fmt.Errorf("unsupported bigquery field type %q", schema.Type)) } + + // Fields are either scalars, lists or maps + panic("unreachable code") } -var snakeCaseWordStartRE = regexp.MustCompile(`_[\w]`) - -// We keep converting the same strings in a tight loop so -// we memoize the results -var snakeCaseToCamelCaseMemoize = map[string]string{} - -func snakeCaseToCamelCase(name string) string { - result, ok := snakeCaseToCamelCaseMemoize[name] - if !ok { - result = strings.ToUpper(name[:1]) + snakeCaseWordStartRE.ReplaceAllStringFunc( - name[1:], - func(s string) string { - return strings.ToUpper(s[1:]) - }) - snakeCaseToCamelCaseMemoize[name] = result +// normalizeToSchema normalizes obj to according to the provided schema. +// Specifically converts structs to maps, and maps to key-value lists. +func normalizeToSchema(obj protoreflect.Message, schema *bigquery.FieldSchema) (any, error) { + // short circuit for nil values + if !obj.IsValid() { + return nil, nil + } + + if obj, ok := obj.Interface().(*timestamppb.Timestamp); ok { + return time.Unix(obj.Seconds, int64(obj.Nanos)).Format(time.RFC3339), nil + } + + result := map[string]any{} + + for _, col := range schema.Schema { + res, err := normalizeMessageField(obj, col) + if err != nil { + return nil, err + } + + if res == nil { + continue + } + + result[col.Name] = res } - return result + return result, nil } type serializeError struct { diff --git a/tools/mc2bq/pkg/schema/schema_test.go b/tools/mc2bq/pkg/schema/schema_test.go index 03fafc8..f0f0ba7 100644 --- a/tools/mc2bq/pkg/schema/schema_test.go +++ b/tools/mc2bq/pkg/schema/schema_test.go @@ -32,107 +32,49 @@ import ( // The tests uses a golden output file. // To update it run: go test -test.generate-golden-files $PWD func TestObjectSerializer(t *testing.T) { - type TestInnerStruct struct { - FieldA string - FieldB int - } - innerStructSchema := bigquery.Schema{ - {Name: "field_a", Type: bigquery.StringFieldType}, - {Name: "field_b", Type: bigquery.IntegerFieldType}, - } - type TestStruct struct { - IntScalar int - StringScalar string - BoolScalar bool - Float32Scalar float32 - Float64Scalar float64 - IntArray []int - Timestamp timestamppb.Timestamp - TimestampPtr *timestamppb.Timestamp - DynamicMap map[string]string - Record TestInnerStruct - RecordPtr *TestInnerStruct - NullPtr *TestInnerStruct - RecordArray []*TestInnerStruct - RecordMap map[string]*TestInnerStruct - DeprecatedField string - Enum migrationcenterpb.CommitmentPlan - } - schema := bigquery.Schema{ - {Name: "int_scalar", Type: bigquery.IntegerFieldType}, - {Name: "string_scalar", Type: bigquery.StringFieldType}, - {Name: "bool_scalar", Type: bigquery.BooleanFieldType}, - {Name: "float32_scalar", Type: bigquery.FloatFieldType}, - {Name: "float64_scalar", Type: bigquery.FloatFieldType}, - {Name: "int_array", Type: bigquery.IntegerFieldType, Repeated: true}, - {Name: "timestamp", Type: bigquery.TimestampFieldType}, - {Name: "timestamp_ptr", Type: bigquery.TimestampFieldType}, - {Name: "dynamic_map", Type: bigquery.RecordFieldType, Repeated: true, Schema: bigquery.Schema{ - {Name: "key", Type: bigquery.StringFieldType}, - {Name: "value", Type: bigquery.StringFieldType}, - }}, - {Name: "record", Type: bigquery.RecordFieldType, Schema: innerStructSchema}, - {Name: "record_ptr", Type: bigquery.RecordFieldType, Schema: innerStructSchema}, - {Name: "record_array", Type: bigquery.RecordFieldType, Schema: innerStructSchema, Repeated: true}, - {Name: "record_map", Type: bigquery.RecordFieldType, Repeated: true, Schema: bigquery.Schema{ - {Name: "key", Type: bigquery.StringFieldType}, - {Name: "value", Type: bigquery.RecordFieldType, Schema: innerStructSchema}, - }}, - {Name: "enum", Type: bigquery.StringFieldType}, - // DeprecatedField doesn't appear in the schema as it has been deprecated - } + schema := EmbeddedSchema.AssetTable // This ensures that we are testing all the field types that exist in the schema ensureTypeSetCoverage(t, schema) - obj := &TestStruct{ - IntScalar: 3, - StringScalar: "foo", - BoolScalar: false, - IntArray: []int{1, 2, 3}, - Float32Scalar: 0.32, - Float64Scalar: 0.64, - DynamicMap: map[string]string{"foo": "bar", "fizz": "buzz"}, - Timestamp: *timestamppb.New(time.Unix(1337, 1773)), - TimestampPtr: timestamppb.New(time.Unix(1337, 1773)), - Record: TestInnerStruct{FieldA: "test", FieldB: 10}, - RecordPtr: &TestInnerStruct{FieldA: "test", FieldB: 10}, - NullPtr: nil, - RecordArray: []*TestInnerStruct{{FieldA: "1", FieldB: 1}, {FieldA: "2", FieldB: 2}}, - RecordMap: map[string]*TestInnerStruct{"key": {FieldA: "value"}}, - DeprecatedField: "I should not be in the result JSON", - Enum: 1, + asset := migrationcenterpb.Asset{ + Name: "foo", // string + CreateTime: timestamppb.New(time.Unix(10, 10)), // timestamp + Labels: map[string]string{ // map + "key": "value", + "key2": "value2", + }, + InsightList: &migrationcenterpb.InsightList{ + Insights: []*migrationcenterpb.Insight{ + { + Insight: &migrationcenterpb.Insight_MigrationInsight{ // one_of + MigrationInsight: &migrationcenterpb.MigrationInsight{ + Fit: &migrationcenterpb.FitDescriptor{ + FitLevel: migrationcenterpb.FitDescriptor_FIT, // enum + }, + }, + }, + }, + }, + }, + AssetDetails: &migrationcenterpb.Asset_MachineDetails{ + MachineDetails: &migrationcenterpb.MachineDetails{ + MachineName: "foo", + MemoryMb: 10, // integer + }, + }, } - got, err := SerializeObjectToBigQuery(obj, "object", schema) + serializer := NewSerializer[*migrationcenterpb.Asset]("asset", schema) + + got, err := serializer(&asset) if err != nil { - t.Fatalf("SerializeObjectToBigQuery(%+v, ...): unexpected error: %v", obj, err) + t.Fatalf("SerializeObjectToBigQuery(%+v, ...): unexpected error: %v", &asset, err) } // We do this so that the diff is more human readable got = prettyPrintJSON(got) if diff := golden.Compare(t, "exporter.json", string(got)); diff != "" { - t.Fatalf("SerializeObjectToBigQuery(%+v, ...): mismatch (-want, +got):\n%s", obj, diff) - } -} - -func TestInvalidTimestamp(t *testing.T) { - type TestStruct struct { - Timestamp int - } - schema := bigquery.Schema{ - {Name: "timestamp", Type: bigquery.TimestampFieldType}, - } - obj := &TestStruct{ - Timestamp: 10, - } - - got, err := SerializeObjectToBigQuery(obj, "object", schema) - if err == nil { - t.Fatalf("SerializeObjectToBigQuery(%+v, ...): succeeded unexpectedly", obj) - } - - if diff := cmp.Diff([]uint8(nil), got); diff != "" { - t.Fatalf("SerializeObjectToBigQuery(%+v, ...): mismatch (-want, +got):\n%s", obj, diff) + t.Fatalf("SerializeObjectToBigQuery(%+v, ...): mismatch (-want, +got):\n%s", &asset, diff) } } diff --git a/tools/mc2bq/pkg/schema/testdata/exporter.json.golden b/tools/mc2bq/pkg/schema/testdata/exporter.json.golden index e4f2120..ee0262d 100644 --- a/tools/mc2bq/pkg/schema/testdata/exporter.json.golden +++ b/tools/mc2bq/pkg/schema/testdata/exporter.json.golden @@ -1,52 +1,32 @@ { - "bool_scalar": false, - "dynamic_map": [ - { - "key": "fizz", - "value": "buzz" - }, - { - "key": "foo", - "value": "bar" - } - ], - "enum": "COMMITMENT_PLAN_NONE", - "float32_scalar": 0.32, - "float64_scalar": 0.64, - "int_array": [ - 1, - 2, - 3 - ], - "int_scalar": 3, - "record": { - "field_a": "test", - "field_b": 10 + "create_time": "1970-01-01T00:00:10Z", + "insight_list": { + "insights": [ + { + "migration_insight": { + "fit": { + "fit_level": "FIT" + } + } + } + ] }, - "record_array": [ + "labels": [ { - "field_a": "1", - "field_b": 1 + "key": "key", + "value": "value" }, { - "field_a": "2", - "field_b": 2 - } - ], - "record_map": [ - { - "key": "key", - "value": { - "field_a": "value", - "field_b": 0 - } + "key": "key2", + "value": "value2" } ], - "record_ptr": { - "field_a": "test", - "field_b": 10 + "machine_details": { + "core_count": 0, + "machine_name": "foo", + "memory_mb": 10, + "power_state": "POWER_STATE_UNSPECIFIED", + "uuid": "" }, - "string_scalar": "foo", - "timestamp": "1970-01-01T00:22:17Z", - "timestamp_ptr": "1970-01-01T00:22:17Z" + "name": "foo" }