From 5d714ddce32c9da031602998953ec2bf33608e06 Mon Sep 17 00:00:00 2001
From: chowbao <simon.chow@stellar.org>
Date: Fri, 30 Aug 2024 16:54:58 -0400
Subject: [PATCH] Fix logging issues in stellar-etl writing to airflow (#281)

* Fix logging issues in stellar-etl writing to airflow

* fix typo
---
 cmd/command_utils.go               | 6 +++---
 cmd/export_assets.go               | 2 +-
 cmd/export_effects.go              | 4 +++-
 cmd/export_ledger_entry_changes.go | 4 +++-
 internal/input/changes.go          | 5 ++++-
 5 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/cmd/command_utils.go b/cmd/command_utils.go
index cc0a3791..30c4ebc7 100644
--- a/cmd/command_utils.go
+++ b/cmd/command_utils.go
@@ -127,7 +127,7 @@ func maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path strin
 	}
 
 	if len(cloudStorageBucket) == 0 {
-		cmdLogger.Error("No bucket specified")
+		cmdLogger.Fatal("No bucket specified")
 		return
 	}
 
@@ -137,11 +137,11 @@ func maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path strin
 		cloudStorage = newGCS(cloudCredentials, cloudStorageBucket)
 		err := cloudStorage.UploadTo(cloudCredentials, cloudStorageBucket, path)
 		if err != nil {
-			cmdLogger.Errorf("Unable to upload output to GCS: %s", err)
+			cmdLogger.Fatalf("Unable to upload output to GCS: %s", err)
 			return
 		}
 	default:
-		cmdLogger.Error("Unknown cloud provider")
+		cmdLogger.Fatal("Unknown cloud provider")
 	}
 }
 
diff --git a/cmd/export_assets.go b/cmd/export_assets.go
index 02f1b214..b24d8b66 100644
--- a/cmd/export_assets.go
+++ b/cmd/export_assets.go
@@ -58,7 +58,7 @@ var assetsCmd = &cobra.Command{
 			seenIDs[transformed.AssetID] = true
 			numBytes, err := exportEntry(transformed, outFile, commonArgs.Extra)
 			if err != nil {
-				cmdLogger.Error(err)
+				cmdLogger.LogError(err)
 				numFailures += 1
 				continue
 			}
diff --git a/cmd/export_effects.go b/cmd/export_effects.go
index 00abc6cf..be372248 100644
--- a/cmd/export_effects.go
+++ b/cmd/export_effects.go
@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"fmt"
+
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/stellar/stellar-etl/internal/input"
@@ -34,7 +36,7 @@ var effectsCmd = &cobra.Command{
 			effects, err := transform.TransformEffect(transformInput.Transaction, LedgerSeq, transformInput.LedgerCloseMeta, env.NetworkPassphrase)
 			if err != nil {
 				txIndex := transformInput.Transaction.Index
-				cmdLogger.Errorf("could not transform transaction %d in ledger %d: %v", txIndex, LedgerSeq, err)
+				cmdLogger.LogError(fmt.Errorf("could not transform transaction %d in ledger %d: %v", txIndex, LedgerSeq, err))
 				numFailures += 1
 				continue
 			}
diff --git a/cmd/export_ledger_entry_changes.go b/cmd/export_ledger_entry_changes.go
index 4697cbee..6779801f 100644
--- a/cmd/export_ledger_entry_changes.go
+++ b/cmd/export_ledger_entry_changes.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/stellar/go/ingest/ledgerbackend"
 	"github.com/stellar/go/xdr"
@@ -28,6 +29,7 @@ confirmed by the Stellar network.
 If no data type flags are set, then by default all of them are exported. If any are set, it is assumed that the others should not
 be exported.`,
 	Run: func(cmd *cobra.Command, args []string) {
+		cmdLogger.SetLevel(logrus.InfoLevel)
 		commonArgs := utils.MustCommonFlags(cmd.Flags(), cmdLogger)
 		cmdLogger.StrictExport = commonArgs.StrictExport
 		env := utils.GetEnvironmentDetails(commonArgs)
@@ -316,7 +318,7 @@ func exportTransformedData(
 				case transform.ClaimableBalanceOutput:
 					// Skipping ClaimableBalanceOutputParquet because it is not needed in the current scope of work
 					// Note that ClaimableBalanceOutputParquet uses nested structs that will need to be handled
-					// for parquet conversio
+					// for parquet conversion
 					skip = true
 				case transform.ConfigSettingOutput:
 					transformedResource = append(transformedResource, v)
diff --git a/internal/input/changes.go b/internal/input/changes.go
index d3011e29..eb2d5eb4 100644
--- a/internal/input/changes.go
+++ b/internal/input/changes.go
@@ -128,7 +128,10 @@ func extractBatch(
 				if !ok {
 					// TODO: once LedgerEntryTypeData is tracked as well, all types should be addressed,
 					// so this info log should be a warning.
-					logger.Infof("change type: %v not tracked", change.Type)
+					// Skip LedgerEntryTypeData as we are intentionally not processing it
+					if change.Type != xdr.LedgerEntryTypeData {
+						logger.Infof("change type: %v not tracked", change.Type)
+					}
 				} else {
 					cache.AddChange(change)
 				}