From 776e9adb021a05ccf4da64bc7560bd65f007b3bb Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 23 Jan 2025 10:53:05 -0500 Subject: [PATCH] Fix scvals to print in json string --- internal/transform/contract_data.go | 10 ++- internal/transform/contract_events.go | 101 +++++++++++++++++++++++--- 2 files changed, 99 insertions(+), 12 deletions(-) diff --git a/internal/transform/contract_data.go b/internal/transform/contract_data.go index a1b8de1e..c66b78a2 100644 --- a/internal/transform/contract_data.go +++ b/internal/transform/contract_data.go @@ -104,8 +104,14 @@ func (t *TransformContractDataStruct) TransformContractData(ledgerChange ingest. ledgerSequence := header.Header.LedgerSeq - outputKey, outputKeyDecoded := serializeScVal(contractData.Key) - outputVal, outputValDecoded := serializeScVal(contractData.Val) + outputKey, outputKeyDecoded, err := serializeScVal(contractData.Key) + if err != nil { + return ContractDataOutput{}, err, false + } + outputVal, outputValDecoded, err := serializeScVal(contractData.Val) + if err != nil { + return ContractDataOutput{}, err, false + } outputContractDataXDR, err := xdr.MarshalBase64(contractData) if err != nil { diff --git a/internal/transform/contract_events.go b/internal/transform/contract_events.go index f9ce1a8a..feafc855 100644 --- a/internal/transform/contract_events.go +++ b/internal/transform/contract_events.go @@ -2,6 +2,7 @@ package transform import ( "encoding/base64" + "encoding/json" "fmt" "github.com/stellar/stellar-etl/internal/toid" @@ -48,12 +49,18 @@ func TransformContractEvent(transaction ingest.LedgerTransaction, lhe xdr.Ledger outputTypeString := event.Type.String() eventTopics := getEventTopics(event.Body) - outputTopics, outputTopicsDecoded := serializeScValArray(eventTopics) + outputTopics, outputTopicsDecoded, err := serializeScValArray(eventTopics) + if err != nil { + return []ContractEventOutput{}, err + } outputTopicsJson["topics"] = outputTopics outputTopicsDecodedJson["topics_decoded"] = outputTopicsDecoded eventData := getEventData(event.Body) - outputData, outputDataDecoded := serializeScVal(eventData) + outputData, outputDataDecoded, err := serializeScVal(eventData) + if err != nil { + return []ContractEventOutput{}, err + } // Convert the xdrContactId to string // TODO: https://stellarorg.atlassian.net/browse/HUBBLE-386 this should be a stellar/go/xdr function @@ -117,7 +124,7 @@ func getEventData(eventBody xdr.ContractEventBody) xdr.ScVal { } // TODO this should also be used in the operations processor -func serializeScVal(scVal xdr.ScVal) (map[string]string, map[string]string) { +func serializeScVal(scVal xdr.ScVal) (map[string]string, map[string]string, error) { serializedData := map[string]string{} serializedData["value"] = "n/a" serializedData["type"] = "n/a" @@ -129,25 +136,99 @@ func serializeScVal(scVal xdr.ScVal) (map[string]string, map[string]string) { if scValTypeName, ok := scVal.ArmForSwitch(int32(scVal.Type)); ok { serializedData["type"] = scValTypeName serializedDataDecoded["type"] = scValTypeName - if raw, err := scVal.MarshalBinary(); err == nil { - serializedData["value"] = base64.StdEncoding.EncodeToString(raw) - serializedDataDecoded["value"] = scVal.String() + raw, err := scVal.MarshalBinary() + if err != nil { + return nil, nil, err + } + + serializedData["value"] = base64.StdEncoding.EncodeToString(raw) + serializedDataDecoded["value"], err = printScValJSON(scVal) + if err != nil { + return nil, nil, err } } - return serializedData, serializedDataDecoded + return serializedData, serializedDataDecoded, nil } // TODO this should also be used in the operations processor -func serializeScValArray(scVals []xdr.ScVal) ([]map[string]string, []map[string]string) { +func serializeScValArray(scVals []xdr.ScVal) ([]map[string]string, []map[string]string, error) { data := make([]map[string]string, 0, len(scVals)) dataDecoded := make([]map[string]string, 0, len(scVals)) for _, scVal := range scVals { - serializedData, serializedDataDecoded := serializeScVal(scVal) + serializedData, serializedDataDecoded, err := serializeScVal(scVal) + if err != nil { + return nil, nil, err + } data = append(data, serializedData) dataDecoded = append(dataDecoded, serializedDataDecoded) } - return data, dataDecoded + return data, dataDecoded, nil +} + +// printScValJSON is used to print json parsable ScVal output instead of +// the ScVal.String() output which prints out %v values that aren't parsable +// Example: {"type":"Map","value":"[{collateral [{1 2386457777}]} {liabilities []} {supply []}]"} +// +// This function traverses through the ScVal structure by calling removeNulls which recursively removes +// the null pointers from the given ScVal. +// +// The output is then: {"type":"Map","value":"[{\"collateral\": \"[{\"1\": \"2386457777\"}]\"} {\"liabilities\": \"[]\"} {\"supply\": \"[]\"}]\"} +// where "value" is a valid JSON string +func printScValJSON(scVal xdr.ScVal) (string, error) { + var data map[string]interface{} + + rawData, err := json.Marshal(scVal) + if err != nil { + return "", err + } + + json.Unmarshal(rawData, &data) + cleaned := removeNulls(data) + cleanedData, err := json.Marshal(cleaned) + if err != nil { + return "", err + } + + return string(cleanedData), nil +} + +// removeNulls will recursively traverse through the data map and remove any null values +// In theory ScVals can be nested infinitely but the depth of recursion is unlikely to be very high +// given the resource limits of smart contracts +func removeNulls(data map[string]interface{}) map[string]interface{} { + cleaned := make(map[string]interface{}) + + for k, v := range data { + switch value := v.(type) { + // There don't seem to be other data types that need traversing aside from maps and arrays + case map[string]interface{}: // recurse through maps + nested := removeNulls(value) + if len(nested) > 0 { + cleaned[k] = nested + } + case []interface{}: // recurse through arrays + var newArr []interface{} + for _, item := range value { + if itemMap, ok := item.(map[string]interface{}); ok { + filteredItem := removeNulls(itemMap) + if len(filteredItem) > 0 { + newArr = append(newArr, filteredItem) + } + } else if item != nil { + newArr = append(newArr, item) + } + } + if len(newArr) > 0 { + cleaned[k] = newArr + } + default: + if v != nil { + cleaned[k] = v + } + } + } + return cleaned }