Add list incoming and outgoing payments calls (#53)

- added `GET /payments/outgoing` endpoint: returns a list of 
outgoing payments. Only Lightning outgoing payments are listed. 
Other types of outgoing payments (splice, channel close, ...)
are ignored.

- updated `GET /payments/incoming` endpoint to accept new
parameters for paging and date filtering.

src/commonMain/kotlin/fr/acinq/lightning/bin/Api.kt

@@ -12,6 +12,8 @@ import fr.acinq.lightning.NodeParams
 import fr.acinq.lightning.bin.db.SqlitePaymentsDb
 import fr.acinq.lightning.bin.db.WalletPaymentId
 import fr.acinq.lightning.bin.json.ApiType.*
+import fr.acinq.lightning.bin.json.ApiType.IncomingPayment
+import fr.acinq.lightning.bin.json.ApiType.OutgoingPayment
 import fr.acinq.lightning.blockchain.fee.FeeratePerByte
 import fr.acinq.lightning.blockchain.fee.FeeratePerKw
 import fr.acinq.lightning.channel.ChannelCommand
@@ -128,26 +130,40 @@ class Api(private val nodeParams: NodeParams, private val peer: Peer, private va
                     }
                     call.respond(GeneratedInvoice(invoice.amount?.truncateToSatoshi(), invoice.paymentHash, serialized = invoice.write()))
                 }
+                get("payments/incoming") {
+                    val listAll = call.parameters["all"]?.toBoolean() ?: false // by default, only list incoming payments that have been received
+                    val externalId = call.parameters["externalId"] // may filter incoming payments by an external id
+                    val from = call.parameters.getOptionalLong("from") ?: 0L
+                    val to = call.parameters.getOptionalLong("to") ?: currentTimestampMillis()
+                    val limit = call.parameters.getOptionalLong("limit") ?: 20
+                    val offset = call.parameters.getOptionalLong("offset") ?: 0
+
+                    val payments = if (externalId.isNullOrBlank()) {
+                        paymentDb.listIncomingPayments(from, to, limit, offset, listAll)
+                    } else {
+                        paymentDb.listIncomingPaymentsForExternalId(externalId, from, to, limit, offset, listAll)
+                    }.map { (payment, externalId) ->
+                        IncomingPayment(payment, externalId)
+                    }
+                    call.respond(payments)
+                }
                 get("payments/incoming/{paymentHash}") {
                     val paymentHash = call.parameters.getByteVector32("paymentHash")
                     paymentDb.getIncomingPayment(paymentHash)?.let {
                         val metadata = paymentDb.metadataQueries.get(WalletPaymentId.IncomingPaymentId(paymentHash))
-                        call.respond(IncomingPayment(it, metadata))
+                        call.respond(IncomingPayment(it, metadata?.externalId))
                     } ?: call.respond(HttpStatusCode.NotFound)
                 }
-                get("payments/incoming") {
-                    val externalId = call.parameters.getString("externalId")
-                    val metadataList = paymentDb.metadataQueries.getByExternalId(externalId)
-                    metadataList.mapNotNull { (paymentId, metadata) ->
-                        when (paymentId) {
-                            is WalletPaymentId.IncomingPaymentId -> paymentDb.getIncomingPayment(paymentId.paymentHash)?.let {
-                                IncomingPayment(it, metadata)
-                            }
-                            else -> null
-                        }
-                    }.let { payments ->
-                        call.respond(payments)
+                get("payments/outgoing") {
+                    val listAll = call.parameters["all"]?.toBoolean() ?: false // by default, only list outgoing payments that have been successfully sent, or are pending
+                    val from = call.parameters.getOptionalLong("from") ?: 0L
+                    val to = call.parameters.getOptionalLong("to") ?: currentTimestampMillis()
+                    val limit = call.parameters.getOptionalLong("limit") ?: 20
+                    val offset = call.parameters.getOptionalLong("offset") ?: 0
+                    val payments = paymentDb.listLightningOutgoingPayments(from, to, limit, offset, listAll).map {
+                        OutgoingPayment(it)
                     }
+                    call.respond(payments)
                 }
                 get("payments/outgoing/{uuid}") {
                     val uuid = call.parameters.getUUID("uuid")

src/commonMain/kotlin/fr/acinq/lightning/bin/db/SqlitePaymentsDb.kt

@@ -136,12 +136,10 @@ class SqlitePaymentsDb(val database: PhoenixDatabase) : PaymentsDb {
         lightningOutgoingQueries.getPaymentFromPartId(partId)
     }
 
-    // ---- list outgoing
-
     override suspend fun listLightningOutgoingPayments(
         paymentHash: ByteVector32
     ): List<LightningOutgoingPayment> = withContext(Dispatchers.Default) {
-        lightningOutgoingQueries.listLightningOutgoingPayments(paymentHash)
+        lightningOutgoingQueries.listPaymentsForPaymentHash(paymentHash)
     }
 
     // ---- incoming payments
@@ -254,4 +252,35 @@ class SqlitePaymentsDb(val database: PhoenixDatabase) : PaymentsDb {
         inQueries.deleteIncomingPayment(paymentHash)
     }
 
+    // ---- list payments with filter
+
+    suspend fun listIncomingPayments(from: Long, to: Long, limit: Long, offset: Long, listAll: Boolean): List<Pair<IncomingPayment, String?>> {
+        return withContext(Dispatchers.Default) {
+            if (listAll) {
+                inQueries.listPayments(from, to, limit, offset)
+            } else {
+                inQueries.listReceivedPayments(from, to, limit, offset)
+            }
+        }
+    }
+
+    suspend fun listIncomingPaymentsForExternalId(externalId: String, from: Long, to: Long, limit: Long, offset: Long, listAll: Boolean): List<Pair<IncomingPayment, String?>> {
+        return withContext(Dispatchers.Default) {
+            if (listAll) {
+                inQueries.listPaymentsForExternalId(externalId, from, to, limit, offset)
+            } else {
+                inQueries.listReceivedPaymentsForExternalId(externalId, from, to, limit, offset)
+            }
+        }
+    }
+
+    suspend fun listLightningOutgoingPayments(from: Long, to: Long, limit: Long, offset: Long, listAll: Boolean): List<LightningOutgoingPayment> {
+        return withContext(Dispatchers.Default) {
+            if (listAll) {
+                lightningOutgoingQueries.listPayments(from, to, limit, offset)
+            } else {
+                lightningOutgoingQueries.listSuccessfulOrPendingPayments(from, to, limit, offset)
+            }
+        }
+    }
 }

src/commonMain/kotlin/fr/acinq/lightning/bin/db/payments/IncomingQueries.kt

@@ -21,6 +21,7 @@ import app.cash.sqldelight.coroutines.mapToList
 import fr.acinq.bitcoin.ByteVector32
 import fr.acinq.bitcoin.byteVector32
 import fr.acinq.lightning.db.IncomingPayment
+import fr.acinq.lightning.utils.currentTimestampMillis
 import fr.acinq.phoenix.db.PhoenixDatabase
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.IO
@@ -130,9 +131,34 @@ class IncomingQueries(private val database: PhoenixDatabase) {
         return queries.listAllNotConfirmed(Companion::mapIncomingPayment).asFlow().mapToList(Dispatchers.IO)
     }
 
+    fun listPayments(from: Long, to: Long, limit: Long, offset: Long): List<Pair<IncomingPayment, String?>> {
+        return queries.listCreatedWithin(from = from, to = to, limit, offset).executeAsList().map {
+            mapIncomingPayment(it.payment_hash, it.preimage, it.created_at, it.origin_type, it.origin_blob, it.received_amount_msat, it.received_at, it.received_with_type, it.received_with_blob) to it.external_id
+        }
+    }
+
+    fun listPaymentsForExternalId(externalId: String, from: Long, to: Long, limit: Long, offset: Long): List<Pair<IncomingPayment, String?>> {
+        return queries.listCreatedForExternalIdWithin(externalId, from, to, limit, offset).executeAsList().map {
+            mapIncomingPayment(it.payment_hash, it.preimage, it.created_at, it.origin_type, it.origin_blob, it.received_amount_msat, it.received_at, it.received_with_type, it.received_with_blob) to it.external_id
+        }
+    }
+
+    fun listReceivedPayments(from: Long, to: Long, limit: Long, offset: Long): List<Pair<IncomingPayment, String?>> {
+        return queries.listReceivedWithin(from = from, to = to, limit, offset).executeAsList().map {
+            mapIncomingPayment(it.payment_hash, it.preimage, it.created_at, it.origin_type, it.origin_blob, it.received_amount_msat, it.received_at, it.received_with_type, it.received_with_blob) to it.external_id
+        }
+    }
+
+    fun listReceivedPaymentsForExternalId(externalId: String, from: Long, to: Long, limit: Long, offset: Long): List<Pair<IncomingPayment, String?>> {
+        return queries.listReceivedForExternalIdWithin(externalId, from, to, limit, offset).executeAsList().map {
+            mapIncomingPayment(it.payment_hash, it.preimage, it.created_at, it.origin_type, it.origin_blob, it.received_amount_msat, it.received_at, it.received_with_type, it.received_with_blob) to it.external_id
+        }
+    }
+
     fun listExpiredPayments(fromCreatedAt: Long, toCreatedAt: Long): List<IncomingPayment> {
-        return queries.listAllWithin(fromCreatedAt, toCreatedAt, Companion::mapIncomingPayment).executeAsList().filter {
-            it.received == null
+        return queries.listCreatedWithinNoPaging(fromCreatedAt, toCreatedAt, Companion::mapIncomingPayment).executeAsList().filter {
+            val origin = it.origin
+            it.received == null && origin is IncomingPayment.Origin.Invoice && origin.paymentRequest.isExpired()
         }
     }
 
@@ -186,13 +212,6 @@ class IncomingQueries(private val database: PhoenixDatabase) {
                 else -> throw UnreadableIncomingReceivedWith(received_at, received_with_type, received_with_blob)
             }
         }
-
-        private fun mapTxIdPaymentHash(
-            tx_id: ByteArray,
-            payment_hash: ByteArray
-        ): Pair<ByteVector32, ByteVector32> {
-            return tx_id.byteVector32() to payment_hash.byteVector32()
-        }
     }
 }
 class IncomingPaymentNotFound(paymentHash: ByteVector32) : RuntimeException("missing payment for payment_hash=$paymentHash")

src/commonMain/kotlin/fr/acinq/lightning/bin/db/payments/LightningOutgoingQueries.kt

@@ -158,7 +158,17 @@ class LightningOutgoingQueries(val database: PhoenixDatabase) {
         }
     }
 
-    fun listLightningOutgoingPayments(paymentHash: ByteVector32): List<LightningOutgoingPayment> {
+    fun listPayments(from: Long, to: Long, limit: Long, offset: Long): List<LightningOutgoingPayment> {
+        return queries.listPaymentsWithin(from, to, limit, offset, Companion::mapLightningOutgoingPayment).executeAsList()
+            .let { groupByRawLightningOutgoing(it) }
+    }
+
+    fun listSuccessfulOrPendingPayments(from: Long, to: Long, limit: Long, offset: Long): List<LightningOutgoingPayment> {
+        return queries.listSuccessfulOrPendingPaymentsWithin(from, to, limit, offset, Companion::mapLightningOutgoingPayment).executeAsList()
+            .let { groupByRawLightningOutgoing(it) }
+    }
+
+    fun listPaymentsForPaymentHash(paymentHash: ByteVector32): List<LightningOutgoingPayment> {
         return queries.listPaymentsForPaymentHash(paymentHash.toByteArray(), Companion::mapLightningOutgoingPayment).executeAsList()
             .let { groupByRawLightningOutgoing(it) }
     }
@@ -183,7 +193,7 @@ class LightningOutgoingQueries(val database: PhoenixDatabase) {
 
     companion object {
         @Suppress("UNUSED_PARAMETER")
-        fun mapLightningOutgoingPaymentWithoutParts(
+        private fun mapLightningOutgoingPaymentWithoutParts(
             id: String,
             recipient_amount_msat: Long,
             recipient_node_id: String,
@@ -207,7 +217,6 @@ class LightningOutgoingQueries(val database: PhoenixDatabase) {
             )
         }
 
-        @Suppress("UNUSED_PARAMETER")
         fun mapLightningOutgoingPayment(
             id: String,
             recipient_amount_msat: Long,

src/commonMain/kotlin/fr/acinq/lightning/bin/json/JsonSerializers.kt

@@ -22,7 +22,7 @@ import fr.acinq.lightning.MilliSatoshi
 import fr.acinq.lightning.bin.db.PaymentMetadata
 import fr.acinq.lightning.channel.states.ChannelState
 import fr.acinq.lightning.channel.states.ChannelStateWithCommitments
-import fr.acinq.lightning.db.LightningOutgoingPayment
+import fr.acinq.lightning.db.*
 import fr.acinq.lightning.json.JsonSerializers
 import fr.acinq.lightning.utils.UUID
 import kotlinx.datetime.Clock
@@ -102,10 +102,10 @@ sealed class ApiType {
     @Serializable
     @SerialName("incoming_payment")
     data class IncomingPayment(val paymentHash: ByteVector32, val preimage: ByteVector32, val externalId: String?, val description: String?, val invoice: String?, val isPaid: Boolean, val receivedSat: Satoshi, val fees: MilliSatoshi, val completedAt: Long?, val createdAt: Long) {
-        constructor(payment: fr.acinq.lightning.db.IncomingPayment, metadata: PaymentMetadata?) : this (
+        constructor(payment: fr.acinq.lightning.db.IncomingPayment, externalId: String?) : this (
             paymentHash = payment.paymentHash,
             preimage = payment.preimage,
-            externalId = metadata?.externalId,
+            externalId = externalId,
             description = (payment.origin as? fr.acinq.lightning.db.IncomingPayment.Origin.Invoice)?.paymentRequest?.description,
             invoice = (payment.origin as? fr.acinq.lightning.db.IncomingPayment.Origin.Invoice)?.paymentRequest?.write(),
             isPaid = payment.completedAt != null,
@@ -118,16 +118,17 @@ sealed class ApiType {
 
     @Serializable
     @SerialName("outgoing_payment")
-    data class OutgoingPayment(val paymentHash: ByteVector32, val preimage: ByteVector32?, val isPaid: Boolean, val sent: Satoshi, val fees: MilliSatoshi, val invoice: String?, val completedAt: Long?, val createdAt: Long) {
-        constructor(payment: LightningOutgoingPayment) : this (
+    data class OutgoingPayment(val paymentId: String, val paymentHash: ByteVector32, val preimage: ByteVector32?, val isPaid: Boolean, val sent: Satoshi, val fees: MilliSatoshi, val invoice: String?, val completedAt: Long?, val createdAt: Long) {
+        constructor(payment: LightningOutgoingPayment) : this(
+            paymentId = payment.id.toString(),
             paymentHash = payment.paymentHash,
             preimage = (payment.status as? LightningOutgoingPayment.Status.Completed.Succeeded.OffChain)?.preimage,
             invoice = (payment.details as? LightningOutgoingPayment.Details.Normal)?.paymentRequest?.write(),
-            isPaid = payment.completedAt != null,
+            isPaid = payment.status is LightningOutgoingPayment.Status.Completed.Succeeded.OffChain,
             sent = payment.amount.truncateToSatoshi(),
             fees = payment.fees,
             completedAt = payment.completedAt,
             createdAt = payment.createdAt,
         )
     }
-}
+}

src/commonMain/kotlin/fr/acinq/lightning/cli/PhoenixCli.kt

@@ -9,6 +9,7 @@ import com.github.ajalt.clikt.parameters.groups.mutuallyExclusiveOptions
 import com.github.ajalt.clikt.parameters.groups.required
 import com.github.ajalt.clikt.parameters.groups.single
 import com.github.ajalt.clikt.parameters.options.*
+import com.github.ajalt.clikt.parameters.types.boolean
 import com.github.ajalt.clikt.parameters.types.int
 import com.github.ajalt.clikt.parameters.types.long
 import com.github.ajalt.clikt.sources.MapValueSource
@@ -31,16 +32,14 @@ import io.ktor.client.statement.*
 import io.ktor.http.*
 import io.ktor.serialization.kotlinx.json.*
 import io.ktor.server.util.*
-import io.ktor.util.*
 import io.ktor.utils.io.core.*
 import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.json.Json
-import kotlin.use
 
 fun main(args: Array<String>) =
     PhoenixCli()
         .versionOption(BuildVersions.phoenixdVersion, names = setOf("--version", "-v"))
-        .subcommands(GetInfo(), GetBalance(), ListChannels(), GetOutgoingPayment(), GetIncomingPayment(), ListIncomingPayments(), CreateInvoice(), PayInvoice(), SendToAddress(), CloseChannel())
+        .subcommands(GetInfo(), GetBalance(), ListChannels(), GetOutgoingPayment(), ListOutgoingPayments(), GetIncomingPayment(), ListIncomingPayments(), CreateInvoice(), PayInvoice(), SendToAddress(), CloseChannel())
         .main(args)
 
 data class HttpConf(val baseUrl: Url, val httpClient: HttpClient)
@@ -127,19 +126,48 @@ class GetOutgoingPayment : PhoenixCliCommand(name = "getoutgoingpayment", help =
     }
 }
 
+class ListOutgoingPayments : PhoenixCliCommand(name = "listoutgoingpayments", help = "List outgoing payments") {
+    private val from by option("--from").long().help { "start timestamp in millis since epoch" }
+    private val to by option("--to").long().help { "end timestamp in millis since epoch" }
+    private val limit by option("--limit").long().default(20).help { "number of payments in the page" }
+    private val offset by option("--offset").long().default(0).help { "page offset" }
+    private val all by option("--all").boolean().default(false).help { "if true, include failed payments" }
+    override suspend fun httpRequest() = commonOptions.httpClient.use {
+        it.get(url = commonOptions.baseUrl / "payments/outgoing") {
+            url {
+                parameters.append("all", all.toString())
+                from?.let { parameters.append("from", it.toString()) }
+                to?.let { parameters.append("to", it.toString()) }
+                parameters.append("limit", limit.toString())
+                parameters.append("offset", offset.toString())
+            }
+        }
+    }
+}
+
 class GetIncomingPayment : PhoenixCliCommand(name = "getincomingpayment", help = "Get incoming payment") {
     private val paymentHash by option("--paymentHash", "--h").convert { it.toByteVector32() }.required()
     override suspend fun httpRequest() = commonOptions.httpClient.use {
         it.get(url = commonOptions.baseUrl / "payments/incoming/$paymentHash")
     }
 }
 
-class ListIncomingPayments : PhoenixCliCommand(name = "listincomingpayments", help = "List incoming payments matching the given externalId") {
-    private val externalId by option("--externalId", "--eid").required()
+class ListIncomingPayments : PhoenixCliCommand(name = "listincomingpayments", help = "List incoming payments") {
+    private val from by option("--from").long().help { "start timestamp in millis since epoch" }
+    private val to by option("--to").long().help { "end timestamp in millis since epoch" }
+    private val limit by option("--limit").long().default(20).help { "number of payments in the page" }
+    private val offset by option("--offset").long().default(0).help { "page offset" }
+    private val all by option("--all").boolean().default(false).help { "if true, include unpaid invoices" }
+    private val externalId by option("--externalId").help { "optional external id tied to the payments" }
     override suspend fun httpRequest() = commonOptions.httpClient.use {
         it.get(url = commonOptions.baseUrl / "payments/incoming") {
             url {
-                parameters.append("externalId", externalId)
+                parameters.append("all", all.toString())
+                externalId?.let { parameters.append("externalId", it) }
+                from?.let { parameters.append("from", it.toString()) }
+                to?.let { parameters.append("to", it.toString()) }
+                parameters.append("limit", limit.toString())
+                parameters.append("offset", offset.toString())
             }
         }
     }