From 12817411e69ca73d444750619552d87ce564ace4 Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Mon, 24 Jan 2022 17:25:18 +0000 Subject: [PATCH 1/3] Bump libgit2 to 1.3.0 and git2go to V33. Downstream breaking changes introduced since git2go@V31: - git2go.ErrorCode was deprecated in favour of the native error type. - FetchOptions no longer expects a pointer, but rather the actual value of git2go.FetchOptions. Signed-off-by: Paulo Gomes --- go.mod | 2 +- go.sum | 4 +- pkg/git/libgit2/checkout.go | 10 ++-- pkg/git/libgit2/checkout_test.go | 2 +- pkg/git/libgit2/transport.go | 42 +++++++-------- pkg/git/libgit2/transport_test.go | 86 ++++++++++++++++++++----------- 6 files changed, 86 insertions(+), 60 deletions(-) diff --git a/go.mod b/go.mod index 160c278ae..576ae15fd 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/go-git/go-billy/v5 v5.3.1 github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v1.2.2 - github.com/libgit2/git2go/v31 v31.7.6 + github.com/libgit2/git2go/v33 v33.0.6 github.com/minio/minio-go/v7 v7.0.15 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.17.0 diff --git a/go.sum b/go.sum index 5581610f4..ddb126abd 100644 --- a/go.sum +++ b/go.sum @@ -622,8 +622,8 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libgit2/git2go/v31 v31.7.6 h1:jg/pNomrQULnafmfF6XTkozPX5ypyELoWErWkJuYPcI= -github.com/libgit2/git2go/v31 v31.7.6/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec= +github.com/libgit2/git2go/v33 v33.0.6 h1:F//bA3/pgSTVq2hLNahhnof9NxyCzFF/c3MB6lb93Qo= +github.com/libgit2/git2go/v33 v33.0.6/go.mod h1:KdpqkU+6+++4oHna/MIOgx4GCQ92IPCdpVRMRI80J+4= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= diff --git a/pkg/git/libgit2/checkout.go b/pkg/git/libgit2/checkout.go index 60b2830eb..f279c5870 100644 --- a/pkg/git/libgit2/checkout.go +++ b/pkg/git/libgit2/checkout.go @@ -25,7 +25,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/go-logr/logr" - git2go "github.com/libgit2/git2go/v31" + git2go "github.com/libgit2/git2go/v33" "github.com/fluxcd/pkg/gitutil" "github.com/fluxcd/pkg/version" @@ -61,7 +61,7 @@ type CheckoutBranch struct { func (c *CheckoutBranch) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) { repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ - FetchOptions: &git2go.FetchOptions{ + FetchOptions: git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsNone, RemoteCallbacks: RemoteCallbacks(ctx, opts), ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, @@ -91,7 +91,7 @@ type CheckoutTag struct { func (c *CheckoutTag) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) { repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ - FetchOptions: &git2go.FetchOptions{ + FetchOptions: git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsAll, RemoteCallbacks: RemoteCallbacks(ctx, opts), ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, @@ -115,7 +115,7 @@ type CheckoutCommit struct { func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, opts *git.AuthOptions) (*git.Commit, error) { repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ - FetchOptions: &git2go.FetchOptions{ + FetchOptions: git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsNone, RemoteCallbacks: RemoteCallbacks(ctx, opts), ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, @@ -147,7 +147,7 @@ func (c *CheckoutSemVer) Checkout(ctx context.Context, path, url string, opts *g } repo, err := git2go.Clone(url, path, &git2go.CloneOptions{ - FetchOptions: &git2go.FetchOptions{ + FetchOptions: git2go.FetchOptions{ DownloadTags: git2go.DownloadTagsAll, RemoteCallbacks: RemoteCallbacks(ctx, opts), ProxyOptions: git2go.ProxyOptions{Type: git2go.ProxyTypeAuto}, diff --git a/pkg/git/libgit2/checkout_test.go b/pkg/git/libgit2/checkout_test.go index 0e82986d0..e55b87ade 100644 --- a/pkg/git/libgit2/checkout_test.go +++ b/pkg/git/libgit2/checkout_test.go @@ -25,7 +25,7 @@ import ( "testing" "time" - git2go "github.com/libgit2/git2go/v31" + git2go "github.com/libgit2/git2go/v33" . "github.com/onsi/gomega" ) diff --git a/pkg/git/libgit2/transport.go b/pkg/git/libgit2/transport.go index ab36130b6..a6a7c73eb 100644 --- a/pkg/git/libgit2/transport.go +++ b/pkg/git/libgit2/transport.go @@ -31,7 +31,7 @@ import ( "strings" "time" - git2go "github.com/libgit2/git2go/v31" + git2go "github.com/libgit2/git2go/v33" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" @@ -61,16 +61,16 @@ func RemoteCallbacks(ctx context.Context, opts *git.AuthOptions) git2go.RemoteCa // libgit2 it should stop the transfer when the given context is closed (due to // e.g. a timeout). func transferProgressCallback(ctx context.Context) git2go.TransferProgressCallback { - return func(p git2go.TransferProgress) git2go.ErrorCode { + return func(p git2go.TransferProgress) error { // Early return if all the objects have been received. if p.ReceivedObjects == p.TotalObjects { - return git2go.ErrorCodeOK + return nil } select { case <-ctx.Done(): - return git2go.ErrorCodeUser + return fmt.Errorf("transport close - potentially due to a timeout") default: - return git2go.ErrorCodeOK + return nil } } } @@ -79,12 +79,12 @@ func transferProgressCallback(ctx context.Context) git2go.TransferProgressCallba // libgit2 it should cancel the network operation when the given context is // closed. func transportMessageCallback(ctx context.Context) git2go.TransportMessageCallback { - return func(_ string) git2go.ErrorCode { + return func(_ string) error { select { case <-ctx.Done(): - return git2go.ErrorCodeUser + return fmt.Errorf("transport closed") default: - return git2go.ErrorCodeOK + return nil } } } @@ -93,16 +93,16 @@ func transportMessageCallback(ctx context.Context) git2go.TransportMessageCallba // signals libgit2 it should stop the push transfer when the given context is // closed (due to e.g. a timeout). func pushTransferProgressCallback(ctx context.Context) git2go.PushTransferProgressCallback { - return func(current, total uint32, _ uint) git2go.ErrorCode { + return func(current, total uint32, _ uint) error { // Early return if current equals total. if current == total { - return git2go.ErrorCodeOK + return nil } select { case <-ctx.Done(): - return git2go.ErrorCodeUser + return fmt.Errorf("transport close - potentially due to a timeout") default: - return git2go.ErrorCodeOK + return nil } } } @@ -155,10 +155,10 @@ func certificateCallback(opts *git.AuthOptions) git2go.CertificateCheckCallback // x509Callback returns a CertificateCheckCallback that verifies the // certificate against the given caBundle for git.HTTPS Transports. func x509Callback(caBundle []byte) git2go.CertificateCheckCallback { - return func(cert *git2go.Certificate, valid bool, hostname string) git2go.ErrorCode { + return func(cert *git2go.Certificate, valid bool, hostname string) error { roots := x509.NewCertPool() if ok := roots.AppendCertsFromPEM(caBundle); !ok { - return git2go.ErrorCodeCertificate + return fmt.Errorf("x509 cert could not be appended") } opts := x509.VerifyOptions{ @@ -167,9 +167,9 @@ func x509Callback(caBundle []byte) git2go.CertificateCheckCallback { CurrentTime: now(), } if _, err := cert.X509.Verify(opts); err != nil { - return git2go.ErrorCodeCertificate + return fmt.Errorf("x509 cert could not be verified") } - return git2go.ErrorCodeOK + return nil } } @@ -177,10 +177,10 @@ func x509Callback(caBundle []byte) git2go.CertificateCheckCallback { // the key of Git server against the given host and known_hosts for // git.SSH Transports. func knownHostsCallback(host string, knownHosts []byte) git2go.CertificateCheckCallback { - return func(cert *git2go.Certificate, valid bool, hostname string) git2go.ErrorCode { + return func(cert *git2go.Certificate, valid bool, hostname string) error { kh, err := parseKnownHosts(string(knownHosts)) if err != nil { - return git2go.ErrorCodeCertificate + return fmt.Errorf("failed to parse known_hosts: %w", err) } // First, attempt to split the configured host and port to validate @@ -195,7 +195,7 @@ func knownHostsCallback(host string, knownHosts []byte) git2go.CertificateCheckC // Check if the configured host matches the hostname given to // the callback. if h != hostname { - return git2go.ErrorCodeUser + return fmt.Errorf("host mismatch: %q %q\n", h, hostname) } // We are now certain that the configured host and the hostname @@ -205,10 +205,10 @@ func knownHostsCallback(host string, knownHosts []byte) git2go.CertificateCheckC h = knownhosts.Normalize(host) for _, k := range kh { if k.matches(h, cert.Hostkey) { - return git2go.ErrorCodeOK + return nil } } - return git2go.ErrorCodeCertificate + return fmt.Errorf("hostkey could not be verified") } } diff --git a/pkg/git/libgit2/transport_test.go b/pkg/git/libgit2/transport_test.go index 4a14b3af5..a5b330aeb 100644 --- a/pkg/git/libgit2/transport_test.go +++ b/pkg/git/libgit2/transport_test.go @@ -23,10 +23,11 @@ import ( "encoding/base64" "encoding/pem" "errors" + "fmt" "testing" "time" - git2go "github.com/libgit2/git2go/v31" + git2go "github.com/libgit2/git2go/v33" . "github.com/onsi/gomega" ) @@ -144,42 +145,42 @@ func Test_x509Callback(t *testing.T) { certificate string host string caBundle []byte - want git2go.ErrorCode + want error }{ { name: "Valid certificate authority bundle", certificate: googleLeafFixture, host: "www.google.com", caBundle: []byte(giag2IntermediateFixture + "\n" + geoTrustRootFixture), - want: git2go.ErrorCodeOK, + want: nil, }, { name: "Invalid certificate", certificate: googleLeafWithInvalidHashFixture, host: "www.google.com", caBundle: []byte(giag2IntermediateFixture + "\n" + geoTrustRootFixture), - want: git2go.ErrorCodeCertificate, + want: fmt.Errorf("x509 cert could not be verified"), }, { name: "Invalid certificate authority bundle", certificate: googleLeafFixture, host: "www.google.com", caBundle: bytes.Trim([]byte(giag2IntermediateFixture+"\n"+geoTrustRootFixture), "-"), - want: git2go.ErrorCodeCertificate, + want: fmt.Errorf("x509 cert could not be appended"), }, { name: "Missing intermediate in bundle", certificate: googleLeafFixture, host: "www.google.com", caBundle: []byte(geoTrustRootFixture), - want: git2go.ErrorCodeCertificate, + want: fmt.Errorf("x509 cert could not be verified"), }, { name: "Invalid host", certificate: googleLeafFixture, host: "www.google.co", caBundle: []byte(giag2IntermediateFixture + "\n" + geoTrustRootFixture), - want: git2go.ErrorCodeCertificate, + want: fmt.Errorf("x509 cert could not be verified"), }, } for _, tt := range tests { @@ -194,7 +195,12 @@ func Test_x509Callback(t *testing.T) { } callback := x509Callback(tt.caBundle) - g.Expect(callback(cert, false, tt.host)).To(Equal(tt.want)) + result := g.Expect(callback(cert, false, tt.host)) + if tt.want == nil { + result.To(BeNil()) + } else { + result.To(Equal(tt.want)) + } }) } } @@ -206,7 +212,7 @@ func Test_knownHostsCallback(t *testing.T) { expectedHost string knownHosts []byte hostkey git2go.HostkeyCertificate - want git2go.ErrorCode + want error }{ { name: "Match", @@ -214,7 +220,7 @@ func Test_knownHostsCallback(t *testing.T) { knownHosts: []byte(knownHostsFixture), hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeySHA1 | git2go.HostkeyMD5, HashSHA1: sha1Fingerprint("v2toJdKXfFEaR1u++4iq1UqSrHM")}, expectedHost: "github.com", - want: git2go.ErrorCodeOK, + want: nil, }, { name: "Match with port", @@ -222,7 +228,7 @@ func Test_knownHostsCallback(t *testing.T) { knownHosts: []byte(knownHostsFixture), hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeySHA1 | git2go.HostkeyMD5, HashSHA1: sha1Fingerprint("v2toJdKXfFEaR1u++4iq1UqSrHM")}, expectedHost: "github.com:22", - want: git2go.ErrorCodeOK, + want: nil, }, { name: "Hostname mismatch", @@ -230,7 +236,7 @@ func Test_knownHostsCallback(t *testing.T) { knownHosts: []byte(knownHostsFixture), hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeySHA1 | git2go.HostkeyMD5, HashSHA1: sha1Fingerprint("v2toJdKXfFEaR1u++4iq1UqSrHM")}, expectedHost: "example.com", - want: git2go.ErrorCodeUser, + want: fmt.Errorf("host mismatch: %q %q\n", "example.com", "github.com"), }, { name: "Hostkey mismatch", @@ -238,7 +244,7 @@ func Test_knownHostsCallback(t *testing.T) { knownHosts: []byte(knownHostsFixture), hostkey: git2go.HostkeyCertificate{Kind: git2go.HostkeyMD5, HashMD5: md5Fingerprint("\xb6\x03\x0e\x39\x97\x9e\xd0\xe7\x24\xce\xa3\x77\x3e\x01\x42\x09")}, expectedHost: "github.com", - want: git2go.ErrorCodeCertificate, + want: fmt.Errorf("hostkey could not be verified"), }, } for _, tt := range tests { @@ -247,7 +253,12 @@ func Test_knownHostsCallback(t *testing.T) { cert := &git2go.Certificate{Hostkey: tt.hostkey} callback := knownHostsCallback(tt.expectedHost, tt.knownHosts) - g.Expect(callback(cert, false, tt.host)).To(Equal(tt.want)) + result := g.Expect(callback(cert, false, tt.host)) + if tt.want == nil { + result.To(BeNil()) + } else { + result.To(Equal(tt.want)) + } }) } } @@ -352,7 +363,7 @@ func Test_transferProgressCallback(t *testing.T) { name string progress git2go.TransferProgress cancelFunc func(context.CancelFunc) - wantErr git2go.ErrorCode + wantErr error }{ { name: "ok - in progress", @@ -361,7 +372,7 @@ func Test_transferProgressCallback(t *testing.T) { ReceivedObjects: 21, }, cancelFunc: func(cf context.CancelFunc) {}, - wantErr: git2go.ErrorCodeOK, + wantErr: nil, }, { name: "ok - transfer complete", @@ -370,7 +381,7 @@ func Test_transferProgressCallback(t *testing.T) { ReceivedObjects: 30, }, cancelFunc: func(cf context.CancelFunc) {}, - wantErr: git2go.ErrorCodeOK, + wantErr: nil, }, { name: "ok - transfer complete, context cancelled", @@ -379,7 +390,7 @@ func Test_transferProgressCallback(t *testing.T) { ReceivedObjects: 30, }, cancelFunc: func(cf context.CancelFunc) { cf() }, - wantErr: git2go.ErrorCodeOK, + wantErr: nil, }, { name: "error - context cancelled", @@ -388,7 +399,7 @@ func Test_transferProgressCallback(t *testing.T) { ReceivedObjects: 21, }, cancelFunc: func(cf context.CancelFunc) { cf() }, - wantErr: git2go.ErrorCodeUser, + wantErr: fmt.Errorf("transport close - potentially due to a timeout"), }, } @@ -403,7 +414,12 @@ func Test_transferProgressCallback(t *testing.T) { tt.cancelFunc(cancel) - g.Expect(tpcb(tt.progress)).To(Equal(tt.wantErr)) + result := g.Expect(tpcb(tt.progress)) + if tt.wantErr == nil { + result.To(BeNil()) + } else { + result.To(Equal(tt.wantErr)) + } }) } } @@ -412,17 +428,17 @@ func Test_transportMessageCallback(t *testing.T) { tests := []struct { name string cancelFunc func(context.CancelFunc) - wantErr git2go.ErrorCode + wantErr error }{ { name: "ok - transport open", cancelFunc: func(cf context.CancelFunc) {}, - wantErr: git2go.ErrorCodeOK, + wantErr: nil, }, { name: "error - transport closed", cancelFunc: func(cf context.CancelFunc) { cf() }, - wantErr: git2go.ErrorCodeUser, + wantErr: fmt.Errorf("transport closed"), }, } @@ -437,7 +453,12 @@ func Test_transportMessageCallback(t *testing.T) { tt.cancelFunc(cancel) - g.Expect(tmcb("")).To(Equal(tt.wantErr)) + result := g.Expect(tmcb("")) + if tt.wantErr == nil { + result.To(BeNil()) + } else { + result.To(Equal(tt.wantErr)) + } }) } } @@ -452,31 +473,31 @@ func Test_pushTransferProgressCallback(t *testing.T) { name string progress pushProgress cancelFunc func(context.CancelFunc) - wantErr git2go.ErrorCode + wantErr error }{ { name: "ok - in progress", progress: pushProgress{current: 20, total: 25}, cancelFunc: func(cf context.CancelFunc) {}, - wantErr: git2go.ErrorCodeOK, + wantErr: nil, }, { name: "ok - transfer complete", progress: pushProgress{current: 25, total: 25}, cancelFunc: func(cf context.CancelFunc) {}, - wantErr: git2go.ErrorCodeOK, + wantErr: nil, }, { name: "ok - transfer complete, context cancelled", progress: pushProgress{current: 25, total: 25}, cancelFunc: func(cf context.CancelFunc) { cf() }, - wantErr: git2go.ErrorCodeOK, + wantErr: nil, }, { name: "error - context cancelled", progress: pushProgress{current: 20, total: 25}, cancelFunc: func(cf context.CancelFunc) { cf() }, - wantErr: git2go.ErrorCodeUser, + wantErr: fmt.Errorf("transport close - potentially due to a timeout"), }, } @@ -491,7 +512,12 @@ func Test_pushTransferProgressCallback(t *testing.T) { tt.cancelFunc(cancel) - g.Expect(ptpcb(tt.progress.current, tt.progress.total, tt.progress.bytes)).To(Equal(tt.wantErr)) + result := g.Expect(ptpcb(tt.progress.current, tt.progress.total, tt.progress.bytes)) + if tt.wantErr == nil { + result.To(BeNil()) + } else { + result.To(Equal(tt.wantErr)) + } }) } } From 4348159255dbf5393c9586ce965e429641630e12 Mon Sep 17 00:00:00 2001 From: Sunny Date: Mon, 24 Jan 2022 23:48:18 +0000 Subject: [PATCH 2/3] Add libgit2 checkout test with ED25519 key This adds a test to detect any regression in libgit2's ED25519 key support. go-git supports ED25519 but not the current version of libgit2 used in flux. The updates to libgit2 in v1.2.0 adds support for ED25519. This test would help ensure the right version of libgit2 is used. Signed-off-by: Sunny Signed-off-by: Paulo Gomes --- pkg/git/libgit2/checkout_test.go | 71 +++++++++++++++++++++++ pkg/git/libgit2/testdata/git/repo/foo.txt | 1 + 2 files changed, 72 insertions(+) create mode 100644 pkg/git/libgit2/testdata/git/repo/foo.txt diff --git a/pkg/git/libgit2/checkout_test.go b/pkg/git/libgit2/checkout_test.go index e55b87ade..7d96eb1b6 100644 --- a/pkg/git/libgit2/checkout_test.go +++ b/pkg/git/libgit2/checkout_test.go @@ -20,13 +20,19 @@ import ( "context" "errors" "fmt" + "net/url" "os" "path/filepath" "testing" "time" + "github.com/fluxcd/pkg/gittestserver" + "github.com/fluxcd/pkg/ssh" git2go "github.com/libgit2/git2go/v33" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + + "github.com/fluxcd/source-controller/pkg/git" ) func TestCheckoutBranch_Checkout(t *testing.T) { @@ -444,3 +450,68 @@ func mockSignature(time time.Time) *git2go.Signature { When: time, } } + +// This test is specifically to detect regression in libgit2's ED25519 key +// support for client authentication. +// Refer: https://github.com/fluxcd/source-controller/issues/399 +func TestCheckout_ED25519(t *testing.T) { + g := NewWithT(t) + timeout := 5 * time.Second + + // Create a git test server. + server, err := gittestserver.NewTempGitServer() + g.Expect(err).ToNot(HaveOccurred()) + defer os.RemoveAll(server.Root()) + server.Auth("test-user", "test-pswd") + server.AutoCreate() + + server.KeyDir(filepath.Join(server.Root(), "keys")) + g.Expect(server.ListenSSH()).To(Succeed()) + + go func() { + server.StartSSH() + }() + defer server.StopSSH() + + repoPath := "test.git" + + err = server.InitRepo("testdata/git/repo", git.DefaultBranch, repoPath) + g.Expect(err).NotTo(HaveOccurred()) + + sshURL := server.SSHAddress() + repoURL := sshURL + "/" + repoPath + + // Fetch host key. + u, err := url.Parse(sshURL) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(u.Host).ToNot(BeEmpty()) + knownHosts, err := ssh.ScanHostKey(u.Host, timeout) + g.Expect(err).ToNot(HaveOccurred()) + + kp, err := ssh.NewEd25519Generator().Generate() + g.Expect(err).ToNot(HaveOccurred()) + + secret := corev1.Secret{ + Data: map[string][]byte{ + "identity": kp.PrivateKey, + "known_hosts": knownHosts, + }, + } + + authOpts, err := git.AuthOptionsFromSecret(repoURL, &secret) + g.Expect(err).ToNot(HaveOccurred()) + + // Prepare for checkout. + branchCheckoutStrat := &CheckoutBranch{Branch: git.DefaultBranch} + tmpDir, _ := os.MkdirTemp("", "test") + defer os.RemoveAll(tmpDir) + + ctx, cancel := context.WithTimeout(context.TODO(), timeout) + defer cancel() + + // Checkout the repo. + // This should always fail because the generated key above isn't present in + // the git server. + _, err = branchCheckoutStrat.Checkout(ctx, tmpDir, repoURL, authOpts) + g.Expect(err).To(BeNil()) +} diff --git a/pkg/git/libgit2/testdata/git/repo/foo.txt b/pkg/git/libgit2/testdata/git/repo/foo.txt new file mode 100644 index 000000000..16b14f5da --- /dev/null +++ b/pkg/git/libgit2/testdata/git/repo/foo.txt @@ -0,0 +1 @@ +test file From 0d43766f18d486ff524e3defcff172a93cb5e65f Mon Sep 17 00:00:00 2001 From: Paulo Gomes Date: Tue, 25 Jan 2022 18:43:07 +0000 Subject: [PATCH 3/3] Bump libgit2 docker image Streamline the process of generating images by using a libgit source image that has pre-built static libraries. Signed-off-by: Paulo Gomes --- .github/actions/run-tests/Dockerfile | 4 +- Dockerfile | 97 +++++++++------------------- Makefile | 4 +- 3 files changed, 34 insertions(+), 71 deletions(-) diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index a67d9b060..ddba605ba 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,9 +1,9 @@ ARG BASE_VARIANT=bullseye -ARG GO_VERSION=1.17.5 +ARG GO_VERSION=1.17.6 ARG XX_VERSION=1.1.0 ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2 -ARG LIBGIT2_TAG=libgit2-1.1.1-3 +ARG LIBGIT2_TAG=libgit2-1.3.0 FROM tonistiigi/xx:${XX_VERSION} AS xx FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2 diff --git a/Dockerfile b/Dockerfile index 11eda1696..703eef3fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,55 +1,11 @@ -ARG BASE_VARIANT=bullseye +ARG BASE_VARIANT=alpine ARG GO_VERSION=1.17 ARG XX_VERSION=1.1.0 ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2 -ARG LIBGIT2_TAG=libgit2-1.1.1-3 +ARG LIBGIT2_TAG=libgit2-1.3.0 -FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx -FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2 - -FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable - -FROM gostable AS go-linux - -FROM go-${TARGETOS} AS build-base-bullseye - -# Copy the build utilities -COPY --from=xx / / - -# Align golang base image with bookworm. -# TODO: Replace this with a golang bookworm variant, once that is released. -RUN echo "deb http://deb.debian.org/debian bookworm main" > /etc/apt/sources.list.d/bookworm.list \ - && echo "deb-src http://deb.debian.org/debian bookworm main" /etc/apt/sources.list.d/bookworm.list \ - && xx-apt update \ - && xx-apt -t bookworm upgrade -y \ - && xx-apt -t bookworm install -y curl - -COPY --from=libgit2 /Makefile /libgit2/ - -# Install the libgit2 build dependencies -RUN make -C /libgit2 cmake - -ARG TARGETPLATFORM -RUN make -C /libgit2 dependencies - -FROM build-base-${BASE_VARIANT} as libgit2-bullseye - -ARG TARGETPLATFORM - -# First build libgit2 statically, this ensures that all its dependencies -# will be statically available as well. -ARG BUILD_SHARED_LIBS=OFF -RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2 - -# Rebuild libgit2 this time to generate the shared libraries. -ARG BUILD_SHARED_LIBS=ON -RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2 -# Logs glibc version used at built time. The final image must be compatible with it. -RUN ldd --version ldd > /libgit2/built-on-glibc-version - - -FROM libgit2-${BASE_VARIANT} as build +FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as build # Configure workspace WORKDIR /workspace @@ -64,35 +20,42 @@ COPY go.sum go.sum # Cache modules RUN go mod download -# Copy source code -COPY main.go main.go -COPY controllers/ controllers/ -COPY pkg/ pkg/ -COPY internal/ internal/ +RUN apk add clang lld pkgconfig ca-certificates -# Build the binary ENV CGO_ENABLED=1 ARG TARGETPLATFORM -# The dependencies being statically built are: libgit2, libssh2, libssl, libcrypto and libz. -# Others (such as libc, librt, libdl and libpthread) are resolved at run-time. -# To decrease the likelihood of such dependencies being out of sync, the base build image -# should be aligned with the target (i.e. same debian variant). -RUN FLAGS=$(pkg-config --static --libs --cflags libssh2 libgit2 libssl libcrypto zlib openssl) \ +RUN xx-apk add --no-cache \ + musl-dev gcc lld binutils-gold + +# Performance related changes: +# - Use read-only bind instead of copying go source files. +# - Cache go packages. +RUN --mount=target=. \ + --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg \ + export LIBRARY_PATH="/usr/local/$(xx-info triple)/lib:/usr/local/$(xx-info triple)/lib64:${LIBRARY_PATH}" && \ + export PKG_CONFIG_PATH="/usr/local/$(xx-info triple)/lib/pkgconfig:/usr/local/$(xx-info triple)/lib64/pkgconfig" && \ + export FLAGS="$(pkg-config --static --libs --cflags libssh2 openssl libgit2)" && \ + CGO_LDFLAGS="${FLAGS} -static" \ xx-go build \ - -ldflags "-s -w -extldflags \"/usr/lib/$(xx-info triple)/libssh2.a /usr/lib/$(xx-info triple)/libssl.a /usr/lib/$(xx-info triple)/libcrypto.a /usr/lib/$(xx-info triple)/libz.a -Wl,--unresolved-symbols=ignore-in-object-files -Wl,-allow-shlib-undefined ${FLAGS} -static\"" \ + -ldflags "-s -w" \ -tags 'netgo,osusergo,static_build' \ - -o source-controller -trimpath main.go; + -o /source-controller -trimpath main.go; -# The target image must aligned with apt sources used for libgit2. -FROM debian:bookworm-slim as controller +# Ensure that the binary was cross-compiled correctly to the target platform. +RUN xx-verify --static /source-controller -ARG TARGETPLATFORM -RUN apt update && apt install -y ca-certificates -# Copy over binary from build -COPY --from=build /workspace/source-controller /usr/local/bin/ -COPY --from=libgit2-bullseye /libgit2/built-on-glibc-version / +FROM gcr.io/distroless/static + +# Link repo to the GitHub Container Registry image +LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller" + +ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=build /source-controller /usr/local/bin/ COPY ATTRIBUTIONS.md / USER 65534:65534 diff --git a/Makefile b/Makefile index d3a63670d..d983a2391 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ TAG ?= latest # Base image used to build the Go binary LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 -LIBGIT2_TAG ?= libgit2-1.1.1-3 +LIBGIT2_TAG ?= libgit2-1.3.0 # Allows for defining additional Docker buildx arguments, # e.g. '--push'. @@ -19,7 +19,7 @@ CRD_OPTIONS ?= crd:crdVersions=v1 REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) # Libgit2 version -LIBGIT2_VERSION ?= 1.1.1 +LIBGIT2_VERSION ?= 1.3.0 # Other dependency versions ENVTEST_BIN_VERSION ?= 1.19.2