Skip to content

Commit

Permalink
Convert .tgs with go libraries (and cgo) (telegram) (#1569)
Browse files Browse the repository at this point in the history
This commit adds support for go/cgo tgs conversion when building with the -tags `cgo`
The default binaries are still "pure" go and uses the old way of converting.

* Move lottie_convert.py conversion code to its own file

* Add optional libtgsconverter

* Update vendor

* Apply suggestions from code review

* Update bridge/helper/libtgsconverter.go

Co-authored-by: Wim <wim@42.be>
  • Loading branch information
Benau and 42wim authored Aug 24, 2021
1 parent d4195de commit 53cafa9
Show file tree
Hide file tree
Showing 310 changed files with 121,526 additions and 85 deletions.
66 changes: 0 additions & 66 deletions bridge/helper/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,7 @@ import (
"fmt"
"image/png"
"io"
"io/ioutil"
"net/http"
"os"
"os/exec"
"regexp"
"strings"
"time"
Expand Down Expand Up @@ -239,66 +236,3 @@ func ConvertWebPToPNG(data *[]byte) error {
*data = w.Bytes()
return nil
}

// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
func CanConvertTgsToX() error {
// We depend on the fact that `lottie_convert.py --help` has exit status 0.
// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
// However, there is no alternative like `lottie_convert.py --is-properly-installed`
cmd := exec.Command("lottie_convert.py", "--help")
return cmd.Run()
}

// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
// This relies on an external command, which is ugly, but works.
func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
// lottie can't handle input from a pipe, so write to a temporary file:
tmpInFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-input-*.tgs")
if err != nil {
return err
}
tmpInFileName := tmpInFile.Name()
defer func() {
if removeErr := os.Remove(tmpInFileName); removeErr != nil {
logger.Errorf("Could not delete temporary (input) file %s: %v", tmpInFileName, removeErr)
}
}()
// lottie can handle writing to a pipe, but there is no way to do that platform-independently.
// "/dev/stdout" won't work on Windows, and "-" upsets Cairo for some reason. So we need another file:
tmpOutFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-output-*.data")
if err != nil {
return err
}
tmpOutFileName := tmpOutFile.Name()
defer func() {
if removeErr := os.Remove(tmpOutFileName); removeErr != nil {
logger.Errorf("Could not delete temporary (output) file %s: %v", tmpOutFileName, removeErr)
}
}()

if _, writeErr := tmpInFile.Write(*data); writeErr != nil {
return writeErr
}
// Must close before calling lottie to avoid data races:
if closeErr := tmpInFile.Close(); closeErr != nil {
return closeErr
}

// Call lottie to transform:
cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpInFileName, tmpOutFileName)
cmd.Stdout = nil
cmd.Stderr = nil
// NB: lottie writes progress into to stderr in all cases.
_, stderr := cmd.Output()
if stderr != nil {
// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
return stderr
}
dataContents, err := ioutil.ReadFile(tmpOutFileName)
if err != nil {
return err
}

*data = dataContents
return nil
}
34 changes: 34 additions & 0 deletions bridge/helper/libtgsconverter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// +build cgo

package helper

import (
"fmt"
"github.com/Benau/tgsconverter/libtgsconverter"
"github.com/sirupsen/logrus"
)

func CanConvertTgsToX() error {
return nil
}

// ConvertTgsToX convert input data (which should be tgs format) to any format supported by libtgsconverter
func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
options := libtgsconverter.NewConverterOptions()
options.SetExtension(outputFormat)
blob, err := libtgsconverter.ImportFromData(*data, options)
if err != nil {
return fmt.Errorf("failed to run libtgsconverter.ImportFromData: %s", err.Error())
}

*data = blob
return nil
}

func SupportsFormat(format string) bool {
return libtgsconverter.SupportsExtension(format)
}

func LottieBackend() string {
return "libtgsconverter"
}
89 changes: 89 additions & 0 deletions bridge/helper/lottie_convert.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
// +build !cgo

package helper

import (
"io/ioutil"
"os"
"os/exec"
"github.com/sirupsen/logrus"
)

// CanConvertTgsToX Checks whether the external command necessary for ConvertTgsToX works.
func CanConvertTgsToX() error {
// We depend on the fact that `lottie_convert.py --help` has exit status 0.
// Hyrum's Law predicted this, and Murphy's Law predicts that this will break eventually.
// However, there is no alternative like `lottie_convert.py --is-properly-installed`
cmd := exec.Command("lottie_convert.py", "--help")
return cmd.Run()
}

// ConvertTgsToWebP convert input data (which should be tgs format) to WebP format
// This relies on an external command, which is ugly, but works.
func ConvertTgsToX(data *[]byte, outputFormat string, logger *logrus.Entry) error {
// lottie can't handle input from a pipe, so write to a temporary file:
tmpInFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-input-*.tgs")
if err != nil {
return err
}
tmpInFileName := tmpInFile.Name()
defer func() {
if removeErr := os.Remove(tmpInFileName); removeErr != nil {
logger.Errorf("Could not delete temporary (input) file %s: %v", tmpInFileName, removeErr)
}
}()
// lottie can handle writing to a pipe, but there is no way to do that platform-independently.
// "/dev/stdout" won't work on Windows, and "-" upsets Cairo for some reason. So we need another file:
tmpOutFile, err := ioutil.TempFile(os.TempDir(), "matterbridge-lottie-output-*.data")
if err != nil {
return err
}
tmpOutFileName := tmpOutFile.Name()
defer func() {
if removeErr := os.Remove(tmpOutFileName); removeErr != nil {
logger.Errorf("Could not delete temporary (output) file %s: %v", tmpOutFileName, removeErr)
}
}()

if _, writeErr := tmpInFile.Write(*data); writeErr != nil {
return writeErr
}
// Must close before calling lottie to avoid data races:
if closeErr := tmpInFile.Close(); closeErr != nil {
return closeErr
}

// Call lottie to transform:
cmd := exec.Command("lottie_convert.py", "--input-format", "lottie", "--output-format", outputFormat, tmpInFileName, tmpOutFileName)
cmd.Stdout = nil
cmd.Stderr = nil
// NB: lottie writes progress into to stderr in all cases.
_, stderr := cmd.Output()
if stderr != nil {
// 'stderr' already contains some parts of Stderr, because it was set to 'nil'.
return stderr
}
dataContents, err := ioutil.ReadFile(tmpOutFileName)
if err != nil {
return err
}

*data = dataContents
return nil
}

func SupportsFormat(format string) bool {
switch format {
case "png":
fallthrough
case "webp":
return true
default:
return false
}
return false
}

func LottieBackend() string {
return "lottie_convert.py"
}
18 changes: 4 additions & 14 deletions bridge/telegram/handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -220,20 +220,10 @@ func (b *Btelegram) handleDownloadAvatar(userid int, channel string) {
}

func (b *Btelegram) maybeConvertTgs(name *string, data *[]byte) {
var format string
switch b.GetString("MediaConvertTgs") {
case FormatWebp:
b.Log.Debugf("Tgs to WebP conversion enabled, converting %v", name)
format = FormatWebp
case FormatPng:
// The WebP to PNG converter can't handle animated webp files yet,
// and I'm not going to write a path for x/image/webp.
// The error message would be:
// conversion failed: webp: non-Alpha VP8X is not implemented
// So instead, we tell lottie to directly go to PNG.
b.Log.Debugf("Tgs to PNG conversion enabled, converting %v", name)
format = FormatPng
default:
format := b.GetString("MediaConvertTgs")
if helper.SupportsFormat(format) {
b.Log.Debugf("Format supported by %s, converting %v", helper.LottieBackend(), name)
} else {
// Otherwise, no conversion was requested. Trying to run the usual webp
// converter would fail, because '.tgs.webp' is actually a gzipped JSON
// file, and has nothing to do with WebP.
Expand Down
8 changes: 3 additions & 5 deletions bridge/telegram/telegram.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ const (
HTMLFormat = "HTML"
HTMLNick = "htmlnick"
MarkdownV2 = "MarkdownV2"
FormatPng = "png"
FormatWebp = "webp"
)

type Btelegram struct {
Expand All @@ -32,10 +30,10 @@ func New(cfg *bridge.Config) bridge.Bridger {
if tgsConvertFormat != "" {
err := helper.CanConvertTgsToX()
if err != nil {
log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but lottie does not appear to work:\n%#v", tgsConvertFormat, err)
log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but %s does not appear to work:\n%#v", tgsConvertFormat, helper.LottieBackend(), err)
}
if tgsConvertFormat != FormatPng && tgsConvertFormat != FormatWebp {
log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but only '%s' and '%s' are supported.", FormatPng, FormatWebp, tgsConvertFormat)
if !helper.SupportsFormat(tgsConvertFormat) {
log.Fatalf("Telegram bridge configured to convert .tgs files to '%s', but %s doesn't support it.", tgsConvertFormat, helper.LottieBackend())
}
}
return &Btelegram{Config: cfg, avatarMap: make(map[string]string)}
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/42wim/matterbridge
require (
github.com/42wim/go-gitter v0.0.0-20170828205020-017310c2d557
github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f
github.com/Benau/tgsconverter v0.0.0-20210809170556-99f4a4f6337f
github.com/Jeffail/gabs v1.4.0 // indirect
github.com/Philipp15b/go-steam v1.0.1-0.20200727090957-6ae9b3c0a560
github.com/Rhymen/go-whatsapp v0.1.2-0.20210615184944-2b8a3e9b8aa2
Expand Down
10 changes: 10 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ github.com/Azure/azure-sdk-for-go v26.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo
github.com/Azure/go-autorest v11.5.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f h1:2dk3eOnYllh+wUOuDhOoC2vUVoJF/5z478ryJ+wzEII=
github.com/Baozisoftware/qrcode-terminal-go v0.0.0-20170407111555-c0650d8dff0f/go.mod h1:4a58ifQTEe2uwwsaqbh3i2un5/CBPg+At/qHpt18Tmk=
github.com/Benau/go_rlottie v0.0.0-20210807002906-98c1b2421989 h1:+wrfJITuBoQOE6ST4k3c4EortNVQXVhfAbwt0M/j0+Y=
github.com/Benau/go_rlottie v0.0.0-20210807002906-98c1b2421989/go.mod h1:aDWSWjsayFyGTvHZH3v4ijGXEBe51xcEkAK+NUWeOeo=
github.com/Benau/tgsconverter v0.0.0-20210809170556-99f4a4f6337f h1:aUkwZDEMJIGRcWlSDifSLoKG37UCOH/DPeG52/xwois=
github.com/Benau/tgsconverter v0.0.0-20210809170556-99f4a4f6337f/go.mod h1:AQiQKKI/YIIctvDt3hI3c1S05/JXMM7v/sQcRd0paVE=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
Expand Down Expand Up @@ -118,6 +122,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
github.com/armon/go-metrics v0.3.4/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
github.com/av-elier/go-decimal-to-rational v0.0.0-20191127152832-89e6aad02ecf h1:csfEAyvOG4/498Q4SyF48ysFqQC9ESj3o8ppRtg+Rog=
github.com/av-elier/go-decimal-to-rational v0.0.0-20191127152832-89e6aad02ecf/go.mod h1:POPnOeaYF7U9o3PjLTb9icRfEOxjBNLRXh9BLximJGM=
github.com/avct/uasurfer v0.0.0-20191028135549-26b5daa857f1/go.mod h1:noBAuukeYOXa0aXGqxr24tADqkwDO2KRD15FsuaZ5a8=
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
github.com/aws/aws-sdk-go v1.19.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
Expand Down Expand Up @@ -501,6 +507,8 @@ github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4
github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U=
github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw=
github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0=
github.com/kettek/apng v0.0.0-20191108220231-414630eed80f h1:dnCYnTSltLuPMfc7dMrkz2uBUcEf/OFBR8yRh3oRT98=
github.com/kettek/apng v0.0.0-20191108220231-414630eed80f/go.mod h1:x78/VRQYKuCftMWS0uK5e+F5RJ7S4gSlESRWI0Prl6Q=
github.com/keybase/go-keybase-chat-bot v0.0.0-20200505163032-5cacf52379da h1:LK+8uBG3kNikj664cjFt88RBmuGmonxkXv2rUVfbqz4=
github.com/keybase/go-keybase-chat-bot v0.0.0-20200505163032-5cacf52379da/go.mod h1:xJA+X9ZVyT/irGldcb7q1XnJBq5F9s5H9h2L44Y+poY=
github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
Expand Down Expand Up @@ -838,6 +846,8 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sizeofint/webpanimation v0.0.0-20210809145948-1d2b32119882 h1:A7o8tOERTtpD/poS+2VoassCjXpjHn916luXbf5QKD0=
github.com/sizeofint/webpanimation v0.0.0-20210809145948-1d2b32119882/go.mod h1:5IwJoz9Pw7JsrCN4/skkxUtSWT7myuUPLhCgv6Q5vvQ=
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
github.com/slack-go/slack v0.9.3 h1:H1UwldF1zWQakjaSymbHMgG3Pg1BiClez/a7JRLdxKc=
Expand Down
24 changes: 24 additions & 0 deletions vendor/github.com/Benau/go_rlottie/LICENSE

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions vendor/github.com/Benau/go_rlottie/README.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 53cafa9

Please # to comment.