Update

cmd/export_external_data.go

@@ -0,0 +1,72 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/stellar/stellar-etl/internal/input"
+	"github.com/stellar/stellar-etl/internal/transform"
+	"github.com/stellar/stellar-etl/internal/utils"
+)
+
+var externalDataCmd = &cobra.Command{
+	Use:   "export_external_data",
+	Short: "Exports external data updated over a specified timestamp range",
+	Long:  "Exports external data updated over a specified timestamp range to an output file.",
+	Run: func(cmd *cobra.Command, args []string) {
+		cmdLogger.SetLevel(logrus.InfoLevel)
+		timestampArgs := utils.MustTimestampRangeFlags(cmd.Flags(), cmdLogger)
+		path := utils.MustBucketFlags(cmd.Flags(), cmdLogger)
+		provider := utils.MustProviderFlags(cmd.Flags(), cmdLogger)
+		cloudStorageBucket, cloudCredentials, cloudProvider := utils.MustCloudStorageFlags(cmd.Flags(), cmdLogger)
+
+		outFile := mustOutFile(path)
+		numFailures := 0
+		totalNumBytes := 0
+
+		switch provider {
+		case "retool":
+			entities, err := input.GetEntityData[utils.RetoolEntityDataTransformInput](nil, provider, timestampArgs.StartTime, timestampArgs.EndTime)
+			if err != nil {
+				cmdLogger.Fatal("could not read entity data: ", err)
+			}
+
+			for _, transformInput := range entities {
+				transformed, err := transform.TransformRetoolEntityData(transformInput)
+				if err != nil {
+					numFailures += 1
+					continue
+				}
+
+				numBytes, err := exportEntry(transformed, outFile, nil)
+				if err != nil {
+					cmdLogger.LogError(fmt.Errorf("could not export entity data: %v", err))
+					numFailures += 1
+					continue
+				}
+				totalNumBytes += numBytes
+			}
+			outFile.Close()
+			cmdLogger.Info("Number of bytes written: ", totalNumBytes)
+
+			printTransformStats(len(entities), numFailures)
+
+		default:
+			panic("unsupported provider: " + provider)
+		}
+
+		maybeUpload(cloudCredentials, cloudStorageBucket, cloudProvider, path)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(externalDataCmd)
+	utils.AddArchiveFlags("entity", externalDataCmd.Flags())
+	utils.AddCloudStorageFlags(externalDataCmd.Flags())
+	utils.AddTimestampRangeFlags(externalDataCmd.Flags())
+	utils.AddProviderFlags(externalDataCmd.Flags())
+	externalDataCmd.MarkFlagRequired("provider")
+	externalDataCmd.MarkFlagRequired("start-time")
+	externalDataCmd.MarkFlagRequired("end-time")
+}

cmd/export_external_data_test.go

@@ -0,0 +1,20 @@
+package cmd
+
+import (
+	"testing"
+)
+
+func TestExportExternalData(t *testing.T) {
+	tests := []cliTest{
+		{
+			name:    "external data from retool",
+			args:    []string{"export_external_data", "--provider", "retool", "--start-time", "", "--end-time", "", "-o", gotTestDir(t, "external_data_retool.txt")},
+			golden:  "external_data_retool.golden",
+			wantErr: nil,
+		},
+	}
+
+	for _, test := range tests {
+		runCLITest(t, test, "testdata/external_data/")
+	}
+}

internal/input/external_data.go

@@ -0,0 +1,82 @@
+package input
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+
+	"github.com/stellar/go/utils/apiclient"
+	"github.com/stellar/stellar-etl/internal/utils"
+)
+
+type ProviderConfig struct {
+	BaseURL     string
+	AuthType    string
+	AuthKeyEnv  string
+	AuthHeaders map[string]interface{}
+	Endpoint    string
+	QueryParams url.Values
+	RequestType string
+}
+
+func GetProviderConfig(provider string) ProviderConfig {
+	var providerConfig ProviderConfig
+	switch provider {
+	case "retool":
+		providerConfig = ProviderConfig{
+			BaseURL:     "https://xrri-vvsg-obfa.n7c.xano.io/api:glgSAjxV",
+			AuthType:    "api_key",
+			AuthHeaders: map[string]interface{}{"api_key": utils.GetEnv("RETOOL_API_KEY", "test-api-key")},
+			RequestType: "GET",
+			Endpoint:    "apps_details",
+			QueryParams: url.Values{},
+		}
+	default:
+		panic("unsupported provider: " + provider)
+	}
+	return providerConfig
+}
+
+func GetEntityData[T any](client *apiclient.APIClient, provider string, startTime string, endTime string) ([]T, error) {
+	providerConfig := GetProviderConfig(provider)
+
+	if client == nil {
+		client = &apiclient.APIClient{
+			BaseURL:     providerConfig.BaseURL,
+			AuthType:    providerConfig.AuthType,
+			AuthHeaders: providerConfig.AuthHeaders,
+		}
+	}
+	reqParams := apiclient.RequestParams{
+		RequestType: providerConfig.RequestType,
+		Endpoint:    providerConfig.Endpoint,
+		QueryParams: providerConfig.QueryParams,
+	}
+	result, err := client.CallAPI(reqParams)
+	if err != nil {
+		return nil, fmt.Errorf("failed to call API: %w", err)
+	}
+
+	// Assert that the result is a slice of interfaces
+	resultSlice, ok := result.([]interface{})
+	if !ok {
+		return nil, fmt.Errorf("Result is not a slice of interface")
+	}
+	dataSlice := []T{}
+
+	for i, item := range resultSlice {
+		if itemMap, ok := item.(map[string]interface{}); ok {
+			var resp T
+			err := utils.MapToStruct(itemMap, &resp)
+			if err != nil {
+				log.Printf("Error converting map to struct: %v", err)
+				continue
+			}
+			dataSlice = append(dataSlice, resp)
+		} else {
+			fmt.Printf("Item %d is not a map\n", i)
+		}
+	}
+
+	return dataSlice, nil
+}

internal/input/retool_test.go → internal/input/external_data_test.go

@@ -11,9 +11,31 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestGetRetoolData(t *testing.T) {
+func getMockClient(provider string, mockResponses []httptest.ResponseData) *apiclient.APIClient {
 	hmock := httptest.NewClient()
+	providerConfig := GetProviderConfig(provider)
 
+	hmock.On("GET", fmt.Sprintf("%s/%s", providerConfig.BaseURL, providerConfig.Endpoint)).
+		ReturnMultipleResults(mockResponses)
+
+	mockClient := &apiclient.APIClient{
+		BaseURL: providerConfig.BaseURL,
+		HTTP:    hmock,
+	}
+	return mockClient
+}
+
+func getEntityDataHelper[T any](t *testing.T, provider string, mockResponses []httptest.ResponseData, expected []T) {
+	mockClient := getMockClient(provider, mockResponses)
+	result, err := GetEntityData[T](mockClient, provider, "", "")
+	if err != nil {
+		t.Fatalf("Error calling GetEntityData: %v", err)
+	}
+
+	assert.Equal(t, expected, result)
+}
+
+func TestGetEntityDataForRetool(t *testing.T) {
 	mockResponses := []httptest.ResponseData{
 		{
 			Status: http.StatusOK,
@@ -87,20 +109,6 @@ func TestGetRetoolData(t *testing.T) {
 			Header: nil,
 		},
 	}
-
-	hmock.On("GET", fmt.Sprintf("%s/apps_details", baseUrl)).
-		ReturnMultipleResults(mockResponses)
-
-	mockClient := &apiclient.APIClient{
-		BaseURL: baseUrl,
-		HTTP:    hmock,
-	}
-
-	result, err := GetRetoolData(mockClient)
-	if err != nil {
-		t.Fatalf("Error calling GetRetoolData: %v", err)
-	}
-
 	expected := []utils.RetoolEntityDataTransformInput{
 		{
 			ID:             16,
@@ -167,6 +175,5 @@ func TestGetRetoolData(t *testing.T) {
 			},
 		},
 	}
-
-	assert.Equal(t, expected, result)
+	getEntityDataHelper[utils.RetoolEntityDataTransformInput](t, "retool", mockResponses, expected)
 }

internal/utils/main.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"os"
 	"time"
 
 	"github.com/spf13/pflag"
@@ -288,6 +289,15 @@ func AddExportTypeFlags(flags *pflag.FlagSet) {
 	flags.BoolP("export-ttl", "", false, "set in order to export ttl changes")
 }
 
+func AddTimestampRangeFlags(flags *pflag.FlagSet) {
+	flags.String("start-time", "", "Start timestamp of the range")
+	flags.String("end-time", "", "End timestamp of the range")
+}
+
+func AddProviderFlags(flags *pflag.FlagSet) {
+	flags.StringP("provider", "p", "", "Third party provider name. Example: retool, github")
+}
+
 // TODO: https://stellarorg.atlassian.net/browse/HUBBLE-386 better flags/params
 // Some flags should be named better
 type FlagValues struct {
@@ -453,6 +463,11 @@ type CommonFlagValues struct {
 	WriteParquet   bool
 }
 
+type TimestampRangeFlagValues struct {
+	StartTime string
+	EndTime   string
+}
+
 // MustCommonFlags gets the values of the the flags common to all commands: end-ledger and strict-export.
 // If any do not exist, it stops the program fatally using the logger
 func MustCommonFlags(flags *pflag.FlagSet, logger *EtlLogger) CommonFlagValues {
@@ -651,6 +666,31 @@ func MustExportTypeFlags(flags *pflag.FlagSet, logger *EtlLogger) map[string]boo
 	return exports
 }
 
+func MustTimestampRangeFlags(flags *pflag.FlagSet, logger *EtlLogger) TimestampRangeFlagValues {
+	startTime, err := flags.GetString("start-time")
+	if err != nil {
+		logger.Fatal("could not get start time of the range: ", err)
+	}
+
+	endTime, err := flags.GetString("end-time")
+	if err != nil {
+		logger.Fatal("could not get end time of the range: ", err)
+	}
+
+	return TimestampRangeFlagValues{
+		StartTime: startTime,
+		EndTime:   endTime,
+	}
+}
+
+func MustProviderFlags(flags *pflag.FlagSet, logger *EtlLogger) string {
+	provider, err := flags.GetString("provider")
+	if err != nil {
+		logger.Fatal("could not get provider: ", err)
+	}
+	return provider
+}
+
 type historyArchiveBackend struct {
 	client  historyarchive.ArchiveInterface
 	ledgers map[uint32]*historyarchive.Ledger
@@ -1110,3 +1150,10 @@ func MapToStruct(data map[string]interface{}, result interface{}) error {
 	}
 	return json.Unmarshal(jsonData, result)
 }
+
+func GetEnv(key, defaultValue string) string {
+	if value := os.Getenv(key); value != "" {
+		return value
+	}
+	return defaultValue
+}