From ac2bf4a938a1d187e5303a67f6419589f1e5cc2f Mon Sep 17 00:00:00 2001 From: Igor Shishkin Date: Sun, 11 Aug 2024 19:08:55 +0300 Subject: [PATCH] Add CLI support for direct YUM repository interaction Signed-off-by: Igor Shishkin --- cli/service/service.go | 180 ++++++++++++++-- cli/service/service_test.go | 152 ++++++++++++- cli/yum/models/packages.go | 8 + cli/yum/models/primarymd.go | 75 +++++++ cli/yum/models/repomd.go | 51 +++++ .../repo-sha1/Packages/testpkg-1-1.src.rpm | Bin 0 -> 6156 bytes .../repo-sha1/Packages/testpkg-1-1.x86_64.rpm | Bin 0 -> 6722 bytes ...08f41e5578d702d2bea21a2e7-filelists.xml.gz | Bin 0 -> 282 bytes ...5a77124d370de1d08deae8f1cc6-primary.xml.gz | Bin 0 -> 688 bytes ...5b59b27c27859bb1c17ac573e-other.sqlite.bz2 | Bin 0 -> 669 bytes ...9b3dd9f40e61c65af499e-filelists.sqlite.bz2 | Bin 0 -> 787 bytes ...894718ea227fea60f2b78ba-primary.sqlite.bz2 | Bin 0 -> 1937 bytes ...127d52228d01b0239010ddca14c8f-other.xml.gz | Bin 0 -> 247 bytes .../testdata/repo-sha1/repodata/repomd.xml | 55 +++++ .../repo/Packages/testpkg-1-1.src.rpm | Bin 0 -> 6156 bytes .../repo/Packages/testpkg-1-1.x86_64.rpm | Bin 0 -> 6722 bytes ...94b3f296cce8bff166ad8ed-primary.sqlite.bz2 | Bin 0 -> 1995 bytes ...3b3bb3e802ee4d5cd0090651091-primary.xml.gz | Bin 0 -> 720 bytes ...2922a6d91d574e617d2f6-filelists.sqlite.bz2 | Bin 0 -> 858 bytes ...9698f232fa9ab5810ec531de1-filelists.xml.gz | Bin 0 -> 313 bytes ...42403bd07105214e1c9f4f0d7-other.sqlite.bz2 | Bin 0 -> 749 bytes ...633d2d5d5d8ba05c9720ad59046e7-other.xml.gz | Bin 0 -> 281 bytes cli/yum/testdata/repo/repodata/repomd.xml | 55 +++++ cli/yum/yum.go | 199 ++++++++++++++++++ cli/yum/yum_test.go | 116 ++++++++++ cmd/cli/main.go | 4 +- docker-compose.yaml | 2 +- 27 files changed, 876 insertions(+), 21 deletions(-) create mode 100644 cli/yum/models/packages.go create mode 100644 cli/yum/models/primarymd.go create mode 100644 cli/yum/models/repomd.go create mode 100644 cli/yum/testdata/repo-sha1/Packages/testpkg-1-1.src.rpm create mode 100644 cli/yum/testdata/repo-sha1/Packages/testpkg-1-1.x86_64.rpm create mode 100644 cli/yum/testdata/repo-sha1/repodata/4a11e3eeb25d21b08f41e5578d702d2bea21a2e7-filelists.xml.gz create mode 100644 cli/yum/testdata/repo-sha1/repodata/80779e2ab55e25a77124d370de1d08deae8f1cc6-primary.xml.gz create mode 100644 cli/yum/testdata/repo-sha1/repodata/b31561a27d014d35b59b27c27859bb1c17ac573e-other.sqlite.bz2 create mode 100644 cli/yum/testdata/repo-sha1/repodata/c66ce2caa41ed83879f9b3dd9f40e61c65af499e-filelists.sqlite.bz2 create mode 100644 cli/yum/testdata/repo-sha1/repodata/e7a8a53e7398f6c22894718ea227fea60f2b78ba-primary.sqlite.bz2 create mode 100644 cli/yum/testdata/repo-sha1/repodata/fdedb6ce109127d52228d01b0239010ddca14c8f-other.xml.gz create mode 100644 cli/yum/testdata/repo-sha1/repodata/repomd.xml create mode 100644 cli/yum/testdata/repo/Packages/testpkg-1-1.src.rpm create mode 100644 cli/yum/testdata/repo/Packages/testpkg-1-1.x86_64.rpm create mode 100644 cli/yum/testdata/repo/repodata/1b4aca205bffe8d65f33b066e3f9965cb4c009e3c94b3f296cce8bff166ad8ed-primary.sqlite.bz2 create mode 100644 cli/yum/testdata/repo/repodata/2267234d92017b049818be743f720f37c176a3b3bb3e802ee4d5cd0090651091-primary.xml.gz create mode 100644 cli/yum/testdata/repo/repodata/2623c0a1472f574989dcba85417e8ce27b87983bba12922a6d91d574e617d2f6-filelists.sqlite.bz2 create mode 100644 cli/yum/testdata/repo/repodata/314e73564000b8a68848551ce0fa9b36e11ed609698f232fa9ab5810ec531de1-filelists.xml.gz create mode 100644 cli/yum/testdata/repo/repodata/64f4875d92a3672f62a2d15d5f0ae6f0806451f42403bd07105214e1c9f4f0d7-other.sqlite.bz2 create mode 100644 cli/yum/testdata/repo/repodata/e3984def0f3b5ce1b174fad2f6eb3c05829633d2d5d5d8ba05c9720ad59046e7-other.xml.gz create mode 100644 cli/yum/testdata/repo/repodata/repomd.xml create mode 100644 cli/yum/yum.go create mode 100644 cli/yum/yum_test.go diff --git a/cli/service/service.go b/cli/service/service.go index 0758f5c..e42b685 100644 --- a/cli/service/service.go +++ b/cli/service/service.go @@ -17,6 +17,7 @@ import ( log "github.com/sirupsen/logrus" cache "github.com/teran/archived/cli/service/stat_cache" + "github.com/teran/archived/cli/yum" v1proto "github.com/teran/archived/presenter/manager/grpc/proto/v1" ) @@ -25,7 +26,7 @@ type Service interface { ListContainers() func(ctx context.Context) error DeleteContainer(containerName string) func(ctx context.Context) error - CreateVersion(containerName string, shouldPublish bool, fromDir *string) func(ctx context.Context) error + CreateVersion(containerName string, shouldPublish bool, fromDir, fromYumRepo *string) func(ctx context.Context) error DeleteVersion(containerName, versionID string) func(ctx context.Context) error ListVersions(containerName string) func(ctx context.Context) error PublishVersion(containerName, versionID string) func(ctx context.Context) error @@ -89,7 +90,7 @@ func (s *service) DeleteContainer(containerName string) func(ctx context.Context } } -func (s *service) CreateVersion(containerName string, shouldPublish bool, fromDir *string) func(ctx context.Context) error { +func (s *service) CreateVersion(containerName string, shouldPublish bool, fromDir, fromYumRepo *string) func(ctx context.Context) error { return func(ctx context.Context) error { resp, err := s.cli.CreateVersion(ctx, &v1proto.CreateVersionRequest{ Container: containerName, @@ -106,6 +107,12 @@ func (s *service) CreateVersion(containerName string, shouldPublish bool, fromDi if err != nil { return errors.Wrap(err, "error creating objects") } + } else if fromYumRepo != nil && *fromYumRepo != "" { + log.Tracef("--from-yum-repo is requested with `%s`", *fromYumRepo) + err := s.createVersionFromYUMRepository(ctx, containerName, versionID, *fromYumRepo) + if err != nil { + return errors.Wrap(err, "error creating objects") + } } if shouldPublish { @@ -126,6 +133,138 @@ func (s *service) CreateVersion(containerName string, shouldPublish bool, fromDi } } +func (s *service) createVersionFromYUMRepository(ctx context.Context, containerName, versionID, url string) error { + repo := yum.New(url) + + packages, err := repo.Packages(ctx) + if err != nil { + return errors.Wrap(err, "error getting repository data") + } + + for k, v := range repo.Metadata() { + size := len(v) + + hasher := sha256.New() + n, err := hasher.Write(v) + if err != nil { + return err + } + + if n != size { + return io.ErrShortWrite + } + + checksum := hex.EncodeToString(hasher.Sum(nil)) + + log.Tracef("rpc CreateObject(%s, %s, %s, %s, %d)", containerName, versionID, k, checksum, size) + resp, err := s.cli.CreateObject(ctx, &v1proto.CreateObjectRequest{ + Container: containerName, + Version: versionID, + Key: k, + Checksum: checksum, + Size: int64(size), + }) + if err != nil { + return errors.Wrap(err, "error creating object") + } + + if uploadURL := resp.GetUploadUrl(); uploadURL != "" { + err := func(url, uploadURL string, rd io.Reader) error { + log.Tracef("Upload URL: `%s`", uploadURL) + return uploadBlob(ctx, uploadURL, rd) + }(url, uploadURL, bytes.NewReader(v)) + if err != nil { + return err + } + } + } + + for _, pkg := range packages { + name := pkg.Name + checksum := pkg.Checksum + size := int64(pkg.Size) + sourceURL := strings.TrimSuffix(url, "/") + "/" + strings.TrimPrefix(pkg.Name, "/") + + if pkg.ChecksumType != "sha256" { + err := func(sourceURL string) error { + log.Tracef("Fetching source artefact: %s ...", sourceURL) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil) + if err != nil { + return errors.Wrap(err, "error constructing request object") + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return errors.Wrap(err, "error requesting object") + } + defer resp.Body.Close() + + buf := bytes.NewBuffer(nil) + if _, err := io.Copy(buf, resp.Body); err != nil { + return errors.Wrap(err, "error on data copy") + } + + h := sha256.New() + n, err := io.Copy(h, buf) + if err != nil { + return errors.Wrap(err, "error calculating checksum") + } + + if n != int64(pkg.Size) { + return io.ErrShortWrite + } + + checksum = hex.EncodeToString(h.Sum(nil)) + + return nil + }(sourceURL) + if err != nil { + return err + } + } + + log.Tracef("rpc CreateObject(%s, %s, %s, %s, %d)", containerName, versionID, name, checksum, size) + resp, err := s.cli.CreateObject(ctx, &v1proto.CreateObjectRequest{ + Container: containerName, + Version: versionID, + Key: name, + Checksum: checksum, + Size: size, + }) + if err != nil { + return errors.Wrap(err, "error creating object") + } + + if uploadURL := resp.GetUploadUrl(); uploadURL != "" { + err := func(url, uploadURL string) error { + log.Tracef("Upload URL: `%s`", uploadURL) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, sourceURL, nil) + if err != nil { + return errors.Wrap(err, "error constructing request object") + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return errors.Wrap(err, "error requesting object") + } + defer resp.Body.Close() + + buf := bytes.NewBuffer(nil) + if _, err := io.Copy(buf, resp.Body); err != nil { + return errors.Wrap(err, "error on data copy") + } + + return uploadBlob(ctx, uploadURL, buf) + }(url, uploadURL) + if err != nil { + return err + } + } + } + return nil +} + func (s *service) DeleteVersion(containerName, versionID string) func(ctx context.Context) error { return func(ctx context.Context) error { _, err := s.cli.DeleteVersion(ctx, &v1proto.DeleteVersionRequest{ @@ -251,19 +390,9 @@ func (s *service) CreateObject(containerName, versionID, directoryPath string) f return errors.Wrap(err, "error on data copy") } - req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, buf) - if err != nil { - return errors.Wrap(err, "error constructing request") + if err := uploadBlob(ctx, url, buf); err != nil { + return err } - - req.Header.Set("Content-Type", "multipart/form-data") - - c := &http.Client{} - uploadResp, err := c.Do(req) - if err != nil { - return errors.Wrap(err, "error uploading file") - } - log.Debugf("upload HTTP response code: %s", uploadResp.Status) } return nil @@ -344,3 +473,26 @@ func checksumFile(filename string) (string, error) { return hex.EncodeToString(h.Sum(nil)), nil } + +func uploadBlob(ctx context.Context, url string, rd io.Reader) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, rd) + if err != nil { + return errors.Wrap(err, "error constructing request") + } + + req.Header.Set("Content-Type", "multipart/form-data") + + c := &http.Client{} + uploadResp, err := c.Do(req) + if err != nil { + return errors.Wrap(err, "error uploading file") + } + + log.Debugf("upload HTTP response code: %s", uploadResp.Status) + + if uploadResp.StatusCode > 299 { + return errors.Errorf("unexpected status code on upload: %s", uploadResp.Status) + } + + return nil +} diff --git a/cli/service/service_test.go b/cli/service/service_test.go index 3b48340..7cb3f1f 100644 --- a/cli/service/service_test.go +++ b/cli/service/service_test.go @@ -7,6 +7,8 @@ import ( "net/http/httptest" "testing" + echo "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" ptr "github.com/teran/go-ptr" @@ -42,7 +44,7 @@ func (s *serviceTestSuite) TestDeleteContainer() { func (s *serviceTestSuite) TestCreateVersion() { s.cliMock.On("CreateVersion", "container1").Return("version_id", nil).Once() - fn := s.svc.CreateVersion("container1", false, nil) + fn := s.svc.CreateVersion("container1", false, nil, nil) s.Require().NoError(fn(s.ctx)) } @@ -50,7 +52,7 @@ func (s *serviceTestSuite) TestCreateVersionAndPublish() { s.cliMock.On("CreateVersion", "container1").Return("version_id", nil).Once() s.cliMock.On("PublishVersion", "container1", "version_id").Return(nil).Once() - fn := s.svc.CreateVersion("container1", true, nil) + fn := s.svc.CreateVersion("container1", true, nil, nil) s.Require().NoError(fn(s.ctx)) } @@ -58,7 +60,7 @@ func (s *serviceTestSuite) TestCreateVersionAndPublishWithEmptyPath() { s.cliMock.On("CreateVersion", "container1").Return("version_id", nil).Once() s.cliMock.On("PublishVersion", "container1", "version_id").Return(nil).Once() - fn := s.svc.CreateVersion("container1", true, ptr.String("")) + fn := s.svc.CreateVersion("container1", true, ptr.String(""), nil) s.Require().NoError(fn(s.ctx)) } @@ -68,7 +70,7 @@ func (s *serviceTestSuite) TestCreateVersionFromDirAndPublish() { s.cacheMock.On("Put", "testdata/somefile1", "a883dafc480d466ee04e0d6da986bd78eb1fdd2178d04693723da3a8f95d42f4").Return(nil).Once() s.cacheMock.On("Put", "testdata/somefile2", "ff5a972ba33179c7ec67c73e00a362b629c489f9d7c86489644db2bcd8c62c61").Return(nil).Once() - s.cliMock.On("CreateVersion", "container1").Return("version_id", nil).Once() + s.cliMock.On("CreateVersion", "container1").Return("version_id", nil, nil).Once() s.cliMock. On("CreateObject", "container1", "version_id", "somefile1", "a883dafc480d466ee04e0d6da986bd78eb1fdd2178d04693723da3a8f95d42f4", int64(5)). Return(ptr.String(""), nil). @@ -79,7 +81,147 @@ func (s *serviceTestSuite) TestCreateVersionFromDirAndPublish() { Once() s.cliMock.On("PublishVersion", "container1", "version_id").Return(nil).Once() - fn := s.svc.CreateVersion("container1", true, ptr.String("testdata")) + fn := s.svc.CreateVersion("container1", true, ptr.String("testdata"), nil) + s.Require().NoError(fn(s.ctx)) +} + +func (s *serviceTestSuite) TestCreateVersionFromYumRepoAndPublish() { + e := echo.New() + e.Use(middleware.Logger()) + e.Use(middleware.Recover()) + e.Static("/", "../yum/testdata/repo") + + srv := httptest.NewServer(e) + defer srv.Close() + + e2 := echo.New() + e2.Use(middleware.Logger()) + e2.Use(middleware.Recover()) + e2.PUT("/upload", func(c echo.Context) error { + if c.Request().Header.Get("Content-Length") != "6156" { + return c.NoContent(http.StatusConflict) + } + + if c.Request().Header.Get("Content-Type") != "multipart/form-data" { + return c.NoContent(http.StatusConflict) + } + return nil + }) + + uploadSrv := httptest.NewServer(e2) + defer uploadSrv.Close() + + s.cliMock.On("CreateVersion", "container1").Return("version_id", nil, nil).Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/2267234d92017b049818be743f720f37c176a3b3bb3e802ee4d5cd0090651091-primary.xml.gz", "2267234d92017b049818be743f720f37c176a3b3bb3e802ee4d5cd0090651091", int64(720)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/1b4aca205bffe8d65f33b066e3f9965cb4c009e3c94b3f296cce8bff166ad8ed-primary.sqlite.bz2", "1b4aca205bffe8d65f33b066e3f9965cb4c009e3c94b3f296cce8bff166ad8ed", int64(1995)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/314e73564000b8a68848551ce0fa9b36e11ed609698f232fa9ab5810ec531de1-filelists.xml.gz", "314e73564000b8a68848551ce0fa9b36e11ed609698f232fa9ab5810ec531de1", int64(313)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/2623c0a1472f574989dcba85417e8ce27b87983bba12922a6d91d574e617d2f6-filelists.sqlite.bz2", "2623c0a1472f574989dcba85417e8ce27b87983bba12922a6d91d574e617d2f6", int64(858)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/e3984def0f3b5ce1b174fad2f6eb3c05829633d2d5d5d8ba05c9720ad59046e7-other.xml.gz", "e3984def0f3b5ce1b174fad2f6eb3c05829633d2d5d5d8ba05c9720ad59046e7", int64(281)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/64f4875d92a3672f62a2d15d5f0ae6f0806451f42403bd07105214e1c9f4f0d7-other.sqlite.bz2", "64f4875d92a3672f62a2d15d5f0ae6f0806451f42403bd07105214e1c9f4f0d7", int64(749)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/repomd.xml", "ad1ff2a7e93b614596a9c432f85b141df86e2c010b6591a04c8b011051bd739c", int64(3069)). + Return(ptr.String(""), nil). + Once() + + s.cliMock. + On("CreateObject", "container1", "version_id", "Packages/testpkg-1-1.src.rpm", "684303227d799ffe1f0b39e030a12ad249931a11ec1690e2079f981cc16d8c52", int64(6156)). + Return(ptr.String(uploadSrv.URL+"/upload"), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "Packages/testpkg-1-1.x86_64.rpm", "d9ae5e56ea38d2ac470f320cade63663dae6ab8b8e1630b2fd5a3c607f45e2ee", int64(6722)). + Return(ptr.String(""), nil). + Once() + s.cliMock.On("PublishVersion", "container1", "version_id").Return(nil).Once() + + fn := s.svc.CreateVersion("container1", true, ptr.String(""), ptr.String(srv.URL)) + s.Require().NoError(fn(s.ctx)) +} + +func (s *serviceTestSuite) TestCreateVersionFromYumRepoAndPublishSHA1() { + e := echo.New() + e.Use(middleware.Logger()) + e.Use(middleware.Recover()) + e.Static("/", "../yum/testdata/repo-sha1") + + srv := httptest.NewServer(e) + defer srv.Close() + + e2 := echo.New() + e2.Use(middleware.Logger()) + e2.Use(middleware.Recover()) + e2.PUT("/upload", func(c echo.Context) error { + if c.Request().Header.Get("Content-Length") != "6156" { + return c.NoContent(http.StatusConflict) + } + + if c.Request().Header.Get("Content-Type") != "multipart/form-data" { + return c.NoContent(http.StatusConflict) + } + return nil + }) + + uploadSrv := httptest.NewServer(e2) + defer uploadSrv.Close() + + s.cliMock.On("CreateVersion", "container1").Return("version_id", nil, nil).Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/80779e2ab55e25a77124d370de1d08deae8f1cc6-primary.xml.gz", "1c07f3f3f0e6d09972c1d7852d1dbc9715d6fbdceee66c50e8356d1e69502d3b", int64(688)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/e7a8a53e7398f6c22894718ea227fea60f2b78ba-primary.sqlite.bz2", "c9b8ce03b503e29d9ec2faa2328e4f2082f0a5f71478ca6cb2f1a3ab75e676bc", int64(1937)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/4a11e3eeb25d21b08f41e5578d702d2bea21a2e7-filelists.xml.gz", "b56801c0a86f9a0136953e8c8e59cd35c1f18fc41e70ba8fcdcccfee068dfc8a", int64(282)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/c66ce2caa41ed83879f9b3dd9f40e61c65af499e-filelists.sqlite.bz2", "59bd3edd4edacac87e5e15494698f34a7f52277691635f927c185e92a681d9ee", int64(787)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/fdedb6ce109127d52228d01b0239010ddca14c8f-other.xml.gz", "56e566dfc63b0a7056b21cec661717a411f68cf98747d9a719557bce3a8ac41a", int64(247)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/b31561a27d014d35b59b27c27859bb1c17ac573e-other.sqlite.bz2", "7eec446e0036d356d8e5694047d9fdb6af00f2fc62993b854232830cf9dbcff8", int64(669)). + Return(ptr.String(""), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "repodata/repomd.xml", "9f18801e8532f631e308a130a347f66eb3900d054df1d66dff53a69aa5b9e7d3", int64(2601)). + Return(ptr.String(""), nil). + Once() + + s.cliMock. + On("CreateObject", "container1", "version_id", "Packages/testpkg-1-1.src.rpm", "684303227d799ffe1f0b39e030a12ad249931a11ec1690e2079f981cc16d8c52", int64(6156)). + Return(ptr.String(uploadSrv.URL+"/upload"), nil). + Once() + s.cliMock. + On("CreateObject", "container1", "version_id", "Packages/testpkg-1-1.x86_64.rpm", "d9ae5e56ea38d2ac470f320cade63663dae6ab8b8e1630b2fd5a3c607f45e2ee", int64(6722)). + Return(ptr.String(""), nil). + Once() + s.cliMock.On("PublishVersion", "container1", "version_id").Return(nil).Once() + + fn := s.svc.CreateVersion("container1", true, ptr.String(""), ptr.String(srv.URL)) s.Require().NoError(fn(s.ctx)) } diff --git a/cli/yum/models/packages.go b/cli/yum/models/packages.go new file mode 100644 index 0000000..6314f63 --- /dev/null +++ b/cli/yum/models/packages.go @@ -0,0 +1,8 @@ +package models + +type Package struct { + Name string + Checksum string + ChecksumType string + Size uint64 +} diff --git a/cli/yum/models/primarymd.go b/cli/yum/models/primarymd.go new file mode 100644 index 0000000..420ea4e --- /dev/null +++ b/cli/yum/models/primarymd.go @@ -0,0 +1,75 @@ +package models + +import "encoding/xml" + +type PrimaryMDPackageVersion struct { + Text string `xml:",chardata"` + Epoch string `xml:"epoch,attr"` + Ver string `xml:"ver,attr"` + Rel string `xml:"rel,attr"` +} + +type PrimaryMDPackageChecksum struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` + PkgID string `xml:"pkgid,attr"` +} + +type PrimaryMDPackageTime struct { + Text string `xml:",chardata"` + File string `xml:"file,attr"` + Build string `xml:"build,attr"` +} + +type PrimaryMDPackageSize struct { + Text string `xml:",chardata"` + Package uint64 `xml:"package,attr"` + Installed string `xml:"installed,attr"` + Archive string `xml:"archive,attr"` +} + +type PrimaryMDPackageLocation struct { + Text string `xml:",chardata"` + Href string `xml:"href,attr"` +} + +type PrimaryMDPackageFormatFile struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` +} + +type PrimaryMDPackageFormat struct { + Text string `xml:",chardata"` + License string `xml:"license"` + Vendor string `xml:"vendor"` + Group string `xml:"group"` + BuildHost string `xml:"buildhost"` + SourceRPM string `xml:"sourcerpm"` + File []PrimaryMDPackageFormatFile `xml:"file"` +} + +type PrimaryMDPackage struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` + Name string `xml:"name"` + Arch string `xml:"arch"` + Version PrimaryMDPackageVersion `xml:"version"` + Checksum PrimaryMDPackageChecksum `xml:"checksum"` + Summary string `xml:"summary"` + Description string `xml:"description"` + Packager string `xml:"packager"` + URL string `xml:"url"` + Time PrimaryMDPackageTime `xml:"time"` + Size PrimaryMDPackageSize `xml:"size"` + Location PrimaryMDPackageLocation `xml:"location"` + Format PrimaryMDPackageFormat `xml:"format"` +} + +type PrimaryMD struct { + XMLName xml.Name `xml:"metadata"` + Text string `xml:",chardata"` + Xmlns string `xml:"xmlns,attr"` + Rpm string `xml:"rpm,attr"` + Packages string `xml:"packages,attr"` + Package []PrimaryMDPackage `xml:"package"` +} diff --git a/cli/yum/models/repomd.go b/cli/yum/models/repomd.go new file mode 100644 index 0000000..7848b45 --- /dev/null +++ b/cli/yum/models/repomd.go @@ -0,0 +1,51 @@ +package models + +import ( + "encoding/xml" + + "github.com/pkg/errors" +) + +type RepoMDDataChecksum struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` +} + +type RepoMDDataOpenChecksum struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` +} + +type RepoMDDataLocation struct { + Text string `xml:",chardata"` + Href string `xml:"href,attr"` +} + +type RepoMDData struct { + Text string `xml:",chardata"` + Type string `xml:"type,attr"` + Checksum RepoMDDataChecksum `xml:"checksum"` + OpenChecksum RepoMDDataOpenChecksum `xml:"open-checksum"` + Location RepoMDDataLocation `xml:"location"` + Timestamp string `xml:"timestamp"` + Size string `xml:"size"` + OpenSize string `xml:"open-size"` +} + +type RepoMD struct { + XMLName xml.Name `xml:"repomd"` + Text string `xml:",chardata"` + Xmlns string `xml:"xmlns,attr"` + Rpm string `xml:"rpm,attr"` + Revision string `xml:"revision"` + Data []RepoMDData `xml:"data"` +} + +func (r *RepoMD) GetPrimary() (RepoMDData, error) { + for _, md := range r.Data { + if md.Type == "primary" { + return md, nil + } + } + return RepoMDData{}, errors.New("no primary MD found") +} diff --git a/cli/yum/testdata/repo-sha1/Packages/testpkg-1-1.src.rpm b/cli/yum/testdata/repo-sha1/Packages/testpkg-1-1.src.rpm new file mode 100644 index 0000000000000000000000000000000000000000..6b82ce96f27da1019a8c4fcd65a9bf4e61869cc4 GIT binary patch literal 6156 zcmeI0U2GIp6vywj#g>9tr9?r*l_*fit}}CI@5~@Up)9sw4Kz?o;@8~yC|kDe+AeLS zjkMUBh=3v>J~V!W5+ASxN)-}8AT0>N6e36oY6uYp3M~+@q5;-(wwHwX=&Li$+5i6T zIrq%m|91D`^lj{KVCjEfR!mN$&~lvIp8_q zIp8_qIp8_qIp8_qIp8_qIp8_qIp8_)e|I1`6G-PzM02K3Gg3(jt!-7YFV*N3}4})U1pYS)WP~;bg`eMPT*nbfy z*1s$m75lv|_!Ush+k#&e{Dxp$$LFmS>G#UztdlG+TMMaz> zUkHKkcH-`d;KRwpfCrC8_@vy5vRI{4Rpr>TBT?r?v&^Z=AtHw>vmzyMOjx5qDX7A9 zRt>VClvJ4oZ9@sBy3Z=Rs>3Z_$aCkJ}p7$AdC%oc$g6Ih$RxDCMN|r@p#275j zSQ>W&I#FHCRwaenx@PH`OSuw~xXnYF;kc%ux;9K!H7+yCWtG_$b6|y=22?SL+LDrN zmLW6QhWV>3Ys^*@W^;vGlyj3B7Hl&$MR#-~q)3u#DK0fFSut$Ml_XiFrplOxU6Mb5 ziNtOEqGmn*9{exCm6E!;uwZWIb6I_FEam%hFMKz=yPO@j4^Q~! zax^ghPEC(GDiHWPxTE6tJ)_4(^%38h+5Pi-^1IT{MrLiAujHrqyi`|RRI}>F)mUD4 z_M-QPuPnOs$;R5$I(6j0s*hT?_EvZC1&4+{I?xfATYjeW>mSZF*pcj+w|1p;hWAr@ zZqBySvDXhADLq&`G5k=`nDc+Of3-LDo!C|V`-Zg8-j=OZTe0$4*8Cf0Eru!O|9h+L<7wjK? CcA&oi literal 0 HcmV?d00001 diff --git a/cli/yum/testdata/repo-sha1/Packages/testpkg-1-1.x86_64.rpm b/cli/yum/testdata/repo-sha1/Packages/testpkg-1-1.x86_64.rpm new file mode 100644 index 0000000000000000000000000000000000000000..42da835b9ba3d62e0965c72a8afbd025c4b067ce GIT binary patch literal 6722 zcmeI1U2GIp6vuDbMOr|u76anT_z}}aW;@>>iB+MrPzBnwfCWEb?wvc`f$h#_W|p?t zXdw_Hi3Xw%_@pLAi9*B#5)%VxM52lK5qTg1Vn{>_0mK(I5vb>E@7Bf_6Q6h5yZ`yk zx%b?2&%M($XRm&6?QDj?RhRN4Eo~lQUX)4e1n-T=eFW|R8t8zT}yRsLt}<-2Aoq}H5qqJTl0LEDom%k z=}?^-%+R^(2AopMv@~1i231{V!i0p>zdv@d;e`W7F8OB~-rU}~Z*M<@B!QuJ)iO}a zKrI8c4Ae4E%RnsywG7lUP|H9q1GNm)GEmDvEd!OB$i&3N9!Le$GzghiH*ui6@ishq z92(R$vU@?Xj}NUL8eUlUz&Z)mO=bwbBzU&q%Yx?$#yUy^v^xY}75uQ^YoJ(Hf%z-` zvCe}29~Jx?DB4#D{v8zUJ%axPMgMCB-vmWF6?{u@ev-533tLGr_QldvDj0nsZxM{^ z!v4d8=YV29Mg`+Kk&g<-IurK)RPddkXg?tsbBg?x;CY}J!#9HO0mbn@37!v%{l}p} z&a(?Z!46wNunv~J7ZmJu=m&YB;9Y{7LBW6Bd!R5V+bQ^v;ANob?@PhU1%EBL3l#l- zC%9kOPlIAS>xBKB;Ae%s8vk>`J`RfWKQH(X!MINl-wa$3+Wkq+j0xs~cM1*!?-pDT z{EA@QM_5m0zu>aq1A=j1VLh4Z`iF$Q8ZYiMj<3c$BKQg@#RV7UCs3z)lhw#+otNOuJ1EX zQ*FwrqN}QH2e?m_D~_U+R6eHFdUHw4au71~fWJQz)l~{@hInMlXaVaJnB>{hTWdLq zTWMHqrCyRJ;G+=I=Jsf?6!RqEerH(VtGq#;v{Y^QWNA16N&4qmk2g|?yh`U}xwnP) z#5@Rxr-le~SF%1yeF#I(DLDfw)JD$ADHX*}GiT&fr-G`LQ*%0*o^wgHHC=aH#c?nX zu!Ks(d)R+cj}`A~)jQ7%g-9NX;)0)pMNKV{$laPG2Zc0h^U7%iw}F?+fO14Pah4|V zgby=bO2eoKFaW4whk-ql^5>o)Xtujy^?XoKk|pD$a{wWPhjf5YOwz7FWoy{b==@fam-m1toT zT466*WA5j@REV7SVxJeeFEge>s7VSZRxZU+3b9AAoTPD7gqZ)+k5?>4u;mpul00Y` z@ZrTQ-!^$rvP^kq3(S_hjl?5|!g3mlqFsVNptOSEYo-VJTfC zE$(aY>XK4EoJxEca$X5#Y9O4AUV!G&<1@8`mpkTMSbOO3`A6`*tuLp6><~qF6yH*) z@2JeA%vY%9QP`+?)Zw+RY3 z(coQn4Q_D7c2%mV8shmP+#SASds=UfebOB4=sTWgk~8vVMC&gde}8-3X?{Asw7qrYY>0s+y$>B9 zVw?^_IO~@+!{tgD;UaDn0U?6;(Q`I|xx$5#r{F1RTiJBISnN0gK|{6Nt25MfQGu2U z`V>;ByfG?AEM?zF#hHg2bv#mt$;uhbX6SgBO$ueK-6}Ae#``_hNS9QIvzgp|iihmZ zb=)$tFp+5=sEJE#U{4USFQ8XQl`akC`@S%`fVwJ3qdC{*zR@V9(SIPWT79VM|CC}0 gvp8EX26ArPeT}ojEDXgqgWdvSS2MI zD*I)7nkYeR%}`H?Ov&*m+l`VL{e1b6=8^+nLFmNMnunnm@WhFDf#(h)cO@cMO5Z6~ zPL#4a(mpJHYpx&i>(=zdsJg3we7L!ut4!HUuw)5FXw~(2-24`6BUtXtT&C>QZRwLo z=P5fZEk5W9A^U5sUe!PfRbf$X?D=uv`+-NvdQg=dAwO9;Niy}psP8c6%xL6>5hbcI zJ+CT2e&AsPZEw}ix}$kp)e3G8MoJAuMsKDevS~yMci}E^E=xPWaYxbWn)h~z2Eyl+ z63`gPuikySzD(IEbR_Mdk*%%x#5j64x*fXwt}z`5RVa`nZ-kGueeESqBoBS?R)OoR zowYl1kG><{w1XCat10i%K5|On67)jz1}`hNOZV8U&k1F^R`qf_T7DJs4}HC(=@&)i`Q!hQ~&vboQPOY$;N-6zqPD0l^3$-?7BBq2zA z0d8D)BJp?_c#uf34*htQNXHS$f2OP0^B(ESi%w+aC4VBTaWen9vVQSUyjW~e(cS@k)>!9>+4$lUo*;+Mh%}x0@+~d8ey{`FzUP4v1i*K#22_a|LXt${Numx-)Nu!1A-m^0RR99 zKmt9pxB<}=+K2!KfB*mh05kvqXaE7GgiR72sf{%C0$|b_Fa*E=OieK`36YZk0Tnzb z(TFqvGynhq00000001PQni(N5QJE8JG}9o^+Ks5t)6@esG|*^0O{m7Egr)u`reY5g zLHl5iUTpTw8j7VE4`2cm7$Q_JUXVW%h_Fx$Kp_YS7fTBDTIPCxqpFclad1WbV(fN* zso-=TaSLTcN_466qXA<###iF%hf+Gl)iK~yEl9>@AY3*9Mp`@LyswVtaJ=mxXNNVc8=)0L#xOL34->^% zoEn2P%N@hMwy>Sy#@R8AXG?6HoVatP%#rDu-WubAa|&)FaFL+(3`XIzZm=_$ZAm*_ zXYwf!Mi#Eb>k7s(e! z9JbZ#;-v!|ktijd<-k{L>F?^T?=$N>WQ%O2fAa)2TxGl|X+k^dKRML1B9p8UA7 D^8+io literal 0 HcmV?d00001 diff --git a/cli/yum/testdata/repo-sha1/repodata/c66ce2caa41ed83879f9b3dd9f40e61c65af499e-filelists.sqlite.bz2 b/cli/yum/testdata/repo-sha1/repodata/c66ce2caa41ed83879f9b3dd9f40e61c65af499e-filelists.sqlite.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..cb0a5971bead4654f05eb81430c9b2a0f2ce31f7 GIT binary patch literal 787 zcmV+u1MK`lT4*&fL0KkKSwn2FHvj?rf8YQ2$cn;m|LXtm{Numx-{>F!1A;&R01O@g z01G1wKmwg;${GOC1t;oYq3R5cHlql{0ibAT4Gji>003w-7=%+KnUhaQ^%`WD0EU_x z8ZZDfMi3b^G{Ou_02}}SGz|?64Ke^U00E#H8UO%f005FfH8KN4OlmST8Vw9gCL|U#`Kg1nvK?~jS!zM>E{q*`$hK_1ep-x;U3x~SvC=@oDl(uQ33@j z3LDJ5Hs;FtH)mxP@K%1p>+rJG+8>{P(JP(hgd#bXifo!nE&*yC(7|;m!jkWAc@L5k+i!{z})JI2zg>ahh`D_NT~WNrxw(7o2Es zBtJ&XyZO}JV2F-8*fHKVu3tI!VrK6b4dV9cv|hSu8{-@P_}8GKsh&ZK(@4jOP?1`r z$qzScjv!Sc4}2<>Pr%}mf?M_CPf2Ko90nT?hZ_*w2@DJppVgs#AeD(Ui5=ykPf88S zIgdKlP!zaD#p;Urq3#w=(04uhRFxaJ@xcv2${G8J~i4rG)UrR8NBTW7!vW=S5Xq6O- zpOZmDNYWxa(XbG6W4>(&&RU8Sh5b#h2k(fc+ zV3VyxZXgr3M;X(yEXkbBYBpAiImNoLHV$Q@oH4Lfx2ttuXSv2IZ`ou;x)T1=DlocY zS7Tes$@5?a5RG{yYaf;&`4#NdW@LtJ1d2b=wuljw6WQrfNvl7N_{B8Lu;_*(;B>Az zJ3?XhCbC!x0ZJ})Bm_R};p-&#|}TAkTO#e)x$VHElD%;x0Kv@CoKPXR+%k`dgPg2dol?=+b9!Y@=b!WU^ZDcb=l9PqBV6D@p@-Q7hwYi4N(5jj z+P~XSYqjswwOaK9tyZ)1A-K;+0(n8>ZABHt_2PN3JuVlUmww9>13;YrjG&GfJnDS2 zeQF5nxsSB)qY6v(BvHcbEFPgxi4x1@aGe^!3M`1G^0A;m98fyU+^6(UqpOAL zG${=LKwjVZo=(-@f>8h0NO2%suWB&Q03B1h?lk1-Cs?Svu7>6*(kgxHkJSjxf|Z;? zKwr977H8l*^hPb;7aZqyfcV1cPkM0sk|GB(!iUrQdVD{weM^s+{o~`;Yh$Vn@Y1P@vaXKCqsW(;93S*hNR>6PIi~@}@DtE1XczzvxwnX5&k^ER zmH%02mJ~Y7saDaNi;)>J%>!KTwIyVI2HgveUWRk7g_G$EqjGZoeTcW)%hpFtuwR}n z&ZVdA(3P9B^gO-{y`Zs2OI490O{52e`7DW3!G>*OuzAO~pK~wrQP8L-j}DmR*)`t#8)9FU_+N61L#9iTH}Jq5SdA<#sB zicZ;oo3sv>8fs2@?w*Nu+K_mN$y2<745mTEF$8n>!|323LGx5Mb=w>Mu<(h+g~=u-t_;e#t2e*7fO3nR7g2* zojhLhoZ`dJ0dFYp)W(Ij@W>?9vZT`76zKBhZA7| zcL(+>Q@)hvAS0sm?>Rjgl(yZkTF6hP=+;5714=sE)WkbZ5hW+^4JCK;f=eS+Qur9R zk}Ad=bz}ZEA%Q#@F6iqB-Qt+izOoI}FPDmGCX|(*+7J5$mP+Ymkg#8g_S}(62M(*3 zYmq^%Sp*@Rfrf{OL@qS}=3Zx4lTE9BcY1hT1Z`$^>!;R@IZ0~9_gCI7+@)Y}T*uje zocl!HBMRjyLSkJ8TqB2*Oddz!_?(XM{#)BHjZ~!jO03XPMJ%T4?R|Gio`nsfl4Si{ z)6Pv{1Dy>`POEOh50I6WJUiBn8#SScr_LxS<-Jj~gv{qNSynH%-^36CM4MNM(2cI{ z*myf?)1mAW@z1)?%#q#ib!xqO)^>a5GoY_)TSZ;nY{I+TKH6Z4IOCl+T}AUbvZpI? z7(8n=OK0aT7ui}`KmWCB;|iT$qpc(iV&mCX#@{qiTju2t?g2pvCga4{Kb`Fm*+*k~(x@ZcMStp(dKz`sVZ|7x7x&qi-8)x z89%#f(5S#mss^Qd@Bd4egzUk{v9~}z);2RM2U{`|B3o(UuEyfm*Til_sWBG@wURgb z0v@su5Q|yJg746{8;Vy!SXph8u*ADhA_{~6JP6YegajonZc{3q_g9G>aMR5ZHWxle z5K`YEcTQT}D2CEkk?v?Fsn|be@^pZseO6B47TP{`2|}G60N0&VLb05d11tVz9q)-l z=o2NOA9(iR4v&rNFB}$55q`+pp^HH8uh1b?2%YLBDa}%2uq;&1YO{T-%9xa%NkmqO&Y?a9-T^`Snt8UwhqOPq4F&> zQq3VO%OL6l_~dc(l)2+~Dgs+&w|9%%w=?iZCJgZW@#MdR<#5U8AkM#UH;P>xrN>(1 zuj!l)-(F9{m&(%4LQa0%SGP5-=Sscr*_Qhz`W_b5Qegdj32P0>WiXH=J)bs-E+Gmq SPJcXK6l6>?`qRbi;r{?+*F%T^ literal 0 HcmV?d00001 diff --git a/cli/yum/testdata/repo-sha1/repodata/fdedb6ce109127d52228d01b0239010ddca14c8f-other.xml.gz b/cli/yum/testdata/repo-sha1/repodata/fdedb6ce109127d52228d01b0239010ddca14c8f-other.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..0c159dec1c6856688b6f65f39a02afdf9c00dd33 GIT binary patch literal 247 zcmVGKEI(t zo!X_7<)h>CaD6|fk$uv-^|LpSCIT3FJGi2}Jw zE2~oHGSO5*0rNPKfk0b`r@(Nvoq=t&QIxUziqB}7PdrwLNGt* x*eFN2&Mc*6BdwNL + + 1723748954 + + 80779e2ab55e25a77124d370de1d08deae8f1cc6 + 5f05d943af6b59a3a4b432b76eb417862335a35f + + 1723748954 + 688 + 2096 + + + 4a11e3eeb25d21b08f41e5578d702d2bea21a2e7 + c183131be21f5f7c5ddd9706c59e0589bf91a770 + + 1723748954 + 282 + 465 + + + fdedb6ce109127d52228d01b0239010ddca14c8f + d2838f0858a8a20576d663f9996b586272e6e6b7 + + 1723748954 + 247 + 394 + + + e7a8a53e7398f6c22894718ea227fea60f2b78ba + aa51549db14b4d21b28f426cf93fd25142aeedab + + 1723748954 + 1937 + 106496 + 10 + + + c66ce2caa41ed83879f9b3dd9f40e61c65af499e + 5d2cc3bac662e0dbea5a034cf99e236b1d9affcf + + 1723748954 + 787 + 28672 + 10 + + + b31561a27d014d35b59b27c27859bb1c17ac573e + 2dddccb3427bb9b2d5d81a3ae4b2e6668c57f651 + + 1723748954 + 669 + 24576 + 10 + + diff --git a/cli/yum/testdata/repo/Packages/testpkg-1-1.src.rpm b/cli/yum/testdata/repo/Packages/testpkg-1-1.src.rpm new file mode 100644 index 0000000000000000000000000000000000000000..6b82ce96f27da1019a8c4fcd65a9bf4e61869cc4 GIT binary patch literal 6156 zcmeI0U2GIp6vywj#g>9tr9?r*l_*fit}}CI@5~@Up)9sw4Kz?o;@8~yC|kDe+AeLS zjkMUBh=3v>J~V!W5+ASxN)-}8AT0>N6e36oY6uYp3M~+@q5;-(wwHwX=&Li$+5i6T zIrq%m|91D`^lj{KVCjEfR!mN$&~lvIp8_q zIp8_qIp8_qIp8_qIp8_qIp8_qIp8_)e|I1`6G-PzM02K3Gg3(jt!-7YFV*N3}4})U1pYS)WP~;bg`eMPT*nbfy z*1s$m75lv|_!Ush+k#&e{Dxp$$LFmS>G#UztdlG+TMMaz> zUkHKkcH-`d;KRwpfCrC8_@vy5vRI{4Rpr>TBT?r?v&^Z=AtHw>vmzyMOjx5qDX7A9 zRt>VClvJ4oZ9@sBy3Z=Rs>3Z_$aCkJ}p7$AdC%oc$g6Ih$RxDCMN|r@p#275j zSQ>W&I#FHCRwaenx@PH`OSuw~xXnYF;kc%ux;9K!H7+yCWtG_$b6|y=22?SL+LDrN zmLW6QhWV>3Ys^*@W^;vGlyj3B7Hl&$MR#-~q)3u#DK0fFSut$Ml_XiFrplOxU6Mb5 ziNtOEqGmn*9{exCm6E!;uwZWIb6I_FEam%hFMKz=yPO@j4^Q~! zax^ghPEC(GDiHWPxTE6tJ)_4(^%38h+5Pi-^1IT{MrLiAujHrqyi`|RRI}>F)mUD4 z_M-QPuPnOs$;R5$I(6j0s*hT?_EvZC1&4+{I?xfATYjeW>mSZF*pcj+w|1p;hWAr@ zZqBySvDXhADLq&`G5k=`nDc+Of3-LDo!C|V`-Zg8-j=OZTe0$4*8Cf0Eru!O|9h+L<7wjK? CcA&oi literal 0 HcmV?d00001 diff --git a/cli/yum/testdata/repo/Packages/testpkg-1-1.x86_64.rpm b/cli/yum/testdata/repo/Packages/testpkg-1-1.x86_64.rpm new file mode 100644 index 0000000000000000000000000000000000000000..42da835b9ba3d62e0965c72a8afbd025c4b067ce GIT binary patch literal 6722 zcmeI1U2GIp6vuDbMOr|u76anT_z}}aW;@>>iB+MrPzBnwfCWEb?wvc`f$h#_W|p?t zXdw_Hi3Xw%_@pLAi9*B#5)%VxM52lK5qTg1Vn{>_0mK(I5vb>E@7Bf_6Q6h5yZ`yk zx%b?2&%M($XRm&6?QDj?RhRN4Eo~lQUX)4e1n-T=eFW|R8t8zT}yRsLt}<-2Aoq}H5qqJTl0LEDom%k z=}?^-%+R^(2AopMv@~1i231{V!i0p>zdv@d;e`W7F8OB~-rU}~Z*M<@B!QuJ)iO}a zKrI8c4Ae4E%RnsywG7lUP|H9q1GNm)GEmDvEd!OB$i&3N9!Le$GzghiH*ui6@ishq z92(R$vU@?Xj}NUL8eUlUz&Z)mO=bwbBzU&q%Yx?$#yUy^v^xY}75uQ^YoJ(Hf%z-` zvCe}29~Jx?DB4#D{v8zUJ%axPMgMCB-vmWF6?{u@ev-533tLGr_QldvDj0nsZxM{^ z!v4d8=YV29Mg`+Kk&g<-IurK)RPddkXg?tsbBg?x;CY}J!#9HO0mbn@37!v%{l}p} z&a(?Z!46wNunv~J7ZmJu=m&YB;9Y{7LBW6Bd!R5V+bQ^v;ANob?@PhU1%EBL3l#l- zC%9kOPlIAS>xBKB;Ae%s8vk>`J`RfWKQH(X!MINl-wa$3+Wkq+j0xs~cM1*!?-pDT z{EA@QM_5m0zu>aq1A=j1VLh4Z`iF$Q8ZYiMj<3c$BKQg@#RV7UCs3z)lhw#+otNOuJ1EX zQ*FwrqN}QH2e?m_D~_U+R6eHFdUHw4au71~fWJQz)l~{@hInMlXaVaJnB>{hTWdLq zTWMHqrCyRJ;G+=I=Jsf?6!RqEerH(VtGq#;v{Y^QWNA16N&4qmk2g|?yh`U}xwnP) z#5@Rxr-le~SF%1yeF#I(DLDfw)JD$ADHX*}GiT&fr-G`LQ*%0*o^wgHHC=aH#c?nX zu!Ks(d)R+cj}`A~)jQ7%g-9NX;)0)pMNKV{$laPG2Zc0h^U7%iw}F?+fO14Pah4|V zgby=bO2eoKFaW4whk-ql^5>o)Xtujy^?XoKk|pD$a{wWPhjf5YOwz7FWoy{b==@fam-m1toT zT466*WA5j@REV7SVxJeeFEge>s7VSZRxZU+3b9AAoTPD7gqZ)+k5?>4u;mpul00Y` z@ZrTQ-!^$rvP^kq3(S_hjl?5|!g3mlqFsVNptOSEYo-VJTfC zE$(aY>XK4EoJxEca$X5#Y9O4AUV!G&<1@8`mpkTMSbOO3`A6`*tuLp6><~qF6yH*) z@2JeA%vY%9QP`+?)Zw+RY3 z(coQn4Q_D7c2%mV8shIXT4*&fL0KkKS*dwp5C8|MfB*mg|NrX$|LTAL{^-B||M)h*B*a7l1^7T< z7ZHzY!yI4-o%05k8(?jP04NGjfubgp)S04r38sdXLZ~$D~K7Xc-y+dW`|0 zqd)^`r~n!dPz?jrG(nRD(XX!RNglzN__>OD0cqtwvTAOK_l4^z|wPbQ$vLemtAp*dlKp29$- zPS7mNK!g@-Na^|gKOd*t@qz04yg;u2Fx0gMmvNZN1T3WJ1R+?!Y#A$0AaN?|%U)S! zuTHIan(+nX^!m^n4JaSQ86jjq8x^ic)&b5v_@#?zAd28@m}_sgVbwF*|ozG1L2uB10BM~4+pJIi=p#E;usfl zTM?AIZt@v&=4*@5`_R_5L?K=~xmHc??v)IT9G&E=F%iqjx~?W?Y@-md5K>`=lE8^L#27Dk-2Km2``xd3iw%&+W0=};ExC$) zb0JiF+WPgjG-jl9m}(YI-bQtl-NS=i!t@$-v`?_` z(GY{9Erg^6SQM!);tGM7tcIO; z&R6`PC9|2m;FkCB0$P=ev`Ys>jQ_d!EZoYX;th{U*NVhf120@F#J(<1xK z+G{gOIgIM^3~wlDBwA`a<`@`meA<_qFCfKcG-jOi7>|x56Vj|d&}uEIQVVD_tSVo4 z#w5y)Q1$&zdJWLNASw-H@*tmpcL>1(M|F#2K;P~tS(s!uqeqMnhJ>M7Ku#>FW;W-kA!p` z9AtwALEdiy4J51pJY(KuHE^7egS{bZNcAugnDBJ@vk$N`GV>2Cdt{S|RfG?z26V8* zf;=r2E*@i{FY+O8=iEVQo05AA$H@sPq+Ei6O@gx!PcV{I#MB8Ozkw>Tp(T%d(`rw0 z`1|*D9Yqk#Nsi9}@lM%7zZ_$tk<~oZ2Kd9Ba>J-~PT~>tfY*2QHVGNPi(WNa!X&4qiFR-oy>#fm360rUk6iK<`jf z*hHF=sD_GDev>9lnK{j#B{{Ri5fM4rIZkmmX-bLYPK^_0p-3R{$stGyh`NGZBp`i9 zA#%;$NwDD@+&1mm+T2dD&Z-~^Ne^IaOJG8>K*vnd7!Xmw11J&$8UX0%CdaDB5QZ}W zVFy5y@5_M)t_8SE&M=U-S*l}@HU?) zW5?%@9xU{m`4=I1{4cfL-QC^U-JRTV&9;3qKyptACtQXBr=U9!YOS0x$g^iwFP!000001Km|kZ`3dlz4I$9pWv|T*h#!em87Ru;!uf?R)Q09?8(O3 zd|@Z0<=10xvfUO@DvEl5%f>VFJmY!e=gm%U+6G;NvAXYaOxFQM(8*ruZkyw)%kx!& zPm5Ps3!_kC6v)9lo8x*MhZ8~?-Ay-ZHSJ&xY9a>|k$v0t9iGuAW@w+>;V>8tLhi&C zTz-aK(ZxjL?EpFUh~gDOStnX3MzG_s+h$~MdABgKF07Fm*{{57=?lTo%R0xdH|KF( zVq~E4U$TeE8syGS?NZ3rg2mjOX1CQU$DhwW;({kp7=(;%)Fw@<3TPFSVG2PQ2+D+F zQJRKSPzsXrGyoQC(ke};bf!vV%rbH)vajgAtuVL6MQ=xBbvrbGKD@g)5Rj1rz=IW7 zY4k980=oL$!e_$M2T;l4W`;x}s}0 zvbCS>yrb$(D@MO`hI5}ZT0&=`IDh-;;w&TgT}RM0bgK6q-!qQUt?8$sxa#ZxQdb(( zjGNs@*uMALO{9oJU};2yGAQYcJ;siB+fPPOp;G#^*1Sk)Kl2s}eq2$4)ieg{@ z|C!`Aj6IMXt${oZS7n7e4^5jocJw2mk<@ CcW0ph literal 0 HcmV?d00001 diff --git a/cli/yum/testdata/repo/repodata/2623c0a1472f574989dcba85417e8ce27b87983bba12922a6d91d574e617d2f6-filelists.sqlite.bz2 b/cli/yum/testdata/repo/repodata/2623c0a1472f574989dcba85417e8ce27b87983bba12922a6d91d574e617d2f6-filelists.sqlite.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..9ec9ea2af16902db5cc4814195c3686dc511fa69 GIT binary patch literal 858 zcmV-g1Eu^zT4*&fL0KkKS+{vVcK`w^fB*mY$cn;m|LXtm{Ncav-{>F!AOHXYKsW#l z0ssIDBMd+S-I?4pIywy!YDcMv^wcsMJx0+qWHcK@^nlRQ5M%~MAY^EInUoruAdG1} zO-JPu(t3t~27qJ+fB*mkO#lN;15nVZ`lqORk02w|&w7XlMX5$Yf~=Kl~vh$iTh6@5bKe`+tbS%wD~GgCIg#Y>|Gd z>q=9FDe(!yff&HR(3Qhyv_5Whn+{K+11O~d=-=#Xf#VCl-gL{a5^rn0G14VSxSFze z8`8c+$l!qGu0i!F_9F%6TOyxYKX%)Fe)$i~r)c5qe!9)M{hN=05km$YQ4|pIs9DUH z3Bww&QWRYY(kG~-mVnT#V$7=4fTR?b(V)Yot+Sc}Lw@+JzWP-J;VDN;Sct;)Mlg_& z#dxV_B@xCR$^ostg4qXhSV7c3hb4o-Ug@2oudiv|G|yUaa)`j#I+|qW$K-PqINJvh zJDk-BVzFyj0K|;P*z(zty7f_CV<5E50QYffv6rTBSYez!z7tfNRe!}dFV!`y|}@p zY=|2mam`$Sx>EoMFHWI5{6Y1R6$E|e$@68<$KAH8At7{unIBsek7l0DLjx!>ri5i} zL|Z9f+S}xi2YJV1pK^GN&GdsJfOr!Sji{sR(CHuy$k_nZKATLRLIJ3TOwFkV4wuh z;32G^0W4aHQdmUX@qs_UIOtHw$eo%#v!eVYSN@Upa42@nFE(3Yd*wGGw?mq%}0B`F!xPl_7~^z!pO)y0)c+>({XNGI Lb)USq0RjL3Z#2_S|LXt${Ncav-)Nu!AOHda05}2w z0RR99KmvWT(YvgWN)ZX9{Y@$945NH4c zL68qnWMmCA$%r)5N2Fv2JNUxGkTGH+2hxNtdtk>h;URPm7N7(XiV$;N_7)QoQyPgx zp#l&=HCn9f{k7Mj)!xO$#N)%J)uv5ZFyy(R!09LRvQsV?Jcb%5yC}O@yqWW%T#?uC z>+H8;wzyEJSvNG9d=6!w8%L-ygoR9e>8l`Q9pA*wwbto=tFDhcS@~J^oTPlNKej3A zf~-MIf0rwmm9=RkN)H(lP{C8Gvu~s|3eo&dE0PD4ULF{4kFXjvFEoD1;5=EZ@dE-` zB#Fx+65%w;mP;fE5Ed;EDa17^CTS?q&?gx}$Bmf@jYOjclanlvl-KB_RHP`UZb6G4 z*$IdV7)2qv(5=~()vi^f|PSYJ(|U0Vq;TaaPh<=jTs9A zZy2S}1&je<1q!d`|Ut~<{`{eQF?0e|DTg@1~vlpg7 f*W$`Gv2m??ILU%F%;yLqAMtl2Q-ui#Fs1YWZy8ar literal 0 HcmV?d00001 diff --git a/cli/yum/testdata/repo/repodata/e3984def0f3b5ce1b174fad2f6eb3c05829633d2d5d5d8ba05c9720ad59046e7-other.xml.gz b/cli/yum/testdata/repo/repodata/e3984def0f3b5ce1b174fad2f6eb3c05829633d2d5d5d8ba05c9720ad59046e7-other.xml.gz new file mode 100644 index 0000000000000000000000000000000000000000..a5aea9f270a796847a1c4f9f750db8874e121379 GIT binary patch literal 281 zcmV+!0p|W6iwFP!000001D%n*Zo@DPg!ev$z`fR=ElWk5Y<+@uE@+WTj3l;UD@7i^ zauIYXx^yCrI2^wNcZd7+B0hOphiU9UR|*7=aiU>7cJTi8`t0Gbf7(s?#0z0YQT*fD z!71nYMas)C-tHCM&Rj9wTF=IUAk&E9zNqI;ypN|7|;CyYBHKwIDgp^n( z)!V?zD%1wa)FD`nT65GcC^o7MDR>==Bp(|CV#F(Vka^7|5(F;s)WN#Ma!iQbqn6-# z+V-Vd*_4tX7QSp#7AE + + 1723389162 + + 2267234d92017b049818be743f720f37c176a3b3bb3e802ee4d5cd0090651091 + c0ceb44a83d3394ea0b1a7df447c58090258495a2142033ec11b1233c0e9468d + + 1723389162 + 720 + 2150 + + + 314e73564000b8a68848551ce0fa9b36e11ed609698f232fa9ab5810ec531de1 + 8d2dd430c787f6e91f887bce856d052c40f94bee11fcd0d8e63386774c92b982 + + 1723389162 + 313 + 513 + + + e3984def0f3b5ce1b174fad2f6eb3c05829633d2d5d5d8ba05c9720ad59046e7 + 7a87a6966494a42b6bfdb33523e585e5c23ccd6bd5276a9e85cf1f062d0f8331 + + 1723389162 + 281 + 442 + + + 1b4aca205bffe8d65f33b066e3f9965cb4c009e3c94b3f296cce8bff166ad8ed + ff9e7a7458e6c53241789a0ef5c26af1c7cb790fa4964f9304865ab4cd1b91a5 + + 1723389162 + 1995 + 106496 + 10 + + + 2623c0a1472f574989dcba85417e8ce27b87983bba12922a6d91d574e617d2f6 + eb8cdb89350f437f388d70ca6225f64a63010885ba785452ad8af42dec1fc897 + + 1723389162 + 858 + 28672 + 10 + + + 64f4875d92a3672f62a2d15d5f0ae6f0806451f42403bd07105214e1c9f4f0d7 + 90217313ce29d957aeb15a3171f50678b8158cc8f61b3a34cf2f07cada777fc3 + + 1723389162 + 749 + 24576 + 10 + + diff --git a/cli/yum/yum.go b/cli/yum/yum.go new file mode 100644 index 0000000..a780570 --- /dev/null +++ b/cli/yum/yum.go @@ -0,0 +1,199 @@ +package yum + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "encoding/xml" + "hash" + "io" + "net/http" + "strings" + "sync" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + + "github.com/teran/archived/cli/yum/models" +) + +var ( + _ YumRepo = (*yumRepo)(nil) + + ErrFileNotFound = errors.New("file not found") + ErrChecksumMismatch = errors.New("checksum mismatch") + ErrNotSupportedChecksumAlgo = errors.New("not supported checksum algorithm is in use") + + hashFunctionsByName map[string]func() hash.Hash = map[string]func() hash.Hash{ + "md5": md5.New, + "sha": sha1.New, + "sha256": sha256.New, + "sha512": sha512.New, + } +) + +type YumRepo interface { + Packages(ctx context.Context) ([]models.Package, error) + Metadata() map[string][]byte +} + +type yumRepo struct { + mutex *sync.RWMutex + url string + metadata map[string][]byte +} + +func New(url string) *yumRepo { + return &yumRepo{ + url: strings.TrimSuffix(url, "/"), + metadata: make(map[string][]byte), + mutex: &sync.RWMutex{}, + } +} + +func (y *yumRepo) Packages(ctx context.Context) ([]models.Package, error) { + rd, err := fetch(ctx, y.url+"/repodata/repomd.xml") + if err != nil { + return nil, err + } + defer rd.Close() + + y.mutex.Lock() + defer y.mutex.Unlock() + y.metadata["repodata/repomd.xml"], err = io.ReadAll(rd) + if err != nil { + return nil, errors.Wrap(err, "error reading repomd.xml") + } + + repomd := models.RepoMD{} + if err := xml.Unmarshal(y.metadata["repodata/repomd.xml"], &repomd); err != nil { + return nil, errors.Wrap(err, "error decoding repomd XML") + } + + if err := y.fetchRepoMetadata(ctx, repomd); err != nil { + return nil, errors.Wrap(err, "error fetching repository metadata") + } + + primary, err := repomd.GetPrimary() + if err != nil { + return nil, err + } + + return y.fetchPackageIndex(ctx, primary.Location, primary.Checksum) +} + +func (y *yumRepo) Metadata() map[string][]byte { + y.mutex.RLock() + defer y.mutex.RUnlock() + + out := map[string][]byte{} + for k, v := range y.metadata { + out[k] = append(out[k], v...) + } + + return out +} + +func (y *yumRepo) fetchRepoMetadata(ctx context.Context, repomd models.RepoMD) error { + for _, md := range repomd.Data { + filename := strings.TrimPrefix(md.Location.Href, "/") + + rd, err := fetch(ctx, y.url+"/"+filename) + if err != nil { + return err + } + defer rd.Close() + + data, err := io.ReadAll(rd) + if err != nil { + return errors.Wrap(err, "error reading file") + } + + y.metadata[filename] = append(y.metadata[filename], data...) + } + + return nil +} + +func (y *yumRepo) fetchPackageIndex(ctx context.Context, href models.RepoMDDataLocation, checksum models.RepoMDDataChecksum) ([]models.Package, error) { + log.Tracef("primary index url: %s", href.Href) + + indexFileName := strings.TrimPrefix(href.Href, "/") + rd, err := fetch(ctx, y.url+"/"+indexFileName) + if err != nil { + return nil, err + } + defer rd.Close() + + hfn, err := hasherByName(checksum.Type) + if err != nil { + return nil, err + } + + buf := &bytes.Buffer{} + hasher := hfn() + trd := io.TeeReader(io.TeeReader(rd, buf), hasher) + + if strings.HasSuffix(href.Href, ".gz") { + log.Tracef(".gz extension detected. Wrapping ...") + trd, err = gzip.NewReader(trd) + if err != nil { + return nil, errors.Wrap(err, "error creating gzip decoder") + } + } + + y.metadata[indexFileName] = buf.Bytes() + + primaryMD := models.PrimaryMD{} + if err := xml.NewDecoder(trd).Decode(&primaryMD); err != nil { + return nil, errors.Wrap(err, "error decoding XML") + } + + if hex.EncodeToString(hasher.Sum(nil)) != checksum.Text { + return nil, ErrChecksumMismatch + } + + packages := []models.Package{} + for _, pkg := range primaryMD.Package { + packages = append(packages, models.Package{ + Name: pkg.Location.Href, + Checksum: pkg.Checksum.Text, + ChecksumType: pkg.Checksum.Type, + Size: pkg.Size.Package, + }) + } + + return packages, nil +} + +func fetch(ctx context.Context, url string) (io.ReadCloser, error) { + log.Tracef("requesting `%s` ...", url) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, ErrFileNotFound + } + + return resp.Body, nil +} + +func hasherByName(algo string) (func() hash.Hash, error) { + if h, ok := hashFunctionsByName[algo]; ok { + return h, nil + } + return nil, errors.Wrapf(ErrNotSupportedChecksumAlgo, "requested algo is `%s`", algo) +} diff --git a/cli/yum/yum_test.go b/cli/yum/yum_test.go new file mode 100644 index 0000000..b711f7f --- /dev/null +++ b/cli/yum/yum_test.go @@ -0,0 +1,116 @@ +package yum + +import ( + "context" + "net/http/httptest" + "testing" + + echo "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/teran/archived/cli/yum/models" +) + +func init() { + log.SetLevel(log.TraceLevel) +} + +func TestPackages(t *testing.T) { + r := require.New(t) + + e := echo.New() + e.Use(middleware.Logger()) + e.Use(middleware.Recover()) + e.Static("/", "testdata/repo") + + srv := httptest.NewServer(e) + defer srv.Close() + + repo := New(srv.URL) + + packages, err := repo.Packages(context.Background()) + r.NoError(err) + r.Equal([]models.Package{ + { + Name: "Packages/testpkg-1-1.src.rpm", + Checksum: "684303227d799ffe1f0b39e030a12ad249931a11ec1690e2079f981cc16d8c52", + ChecksumType: "sha256", + Size: 6156, + }, + { + Name: "Packages/testpkg-1-1.x86_64.rpm", + Checksum: "d9ae5e56ea38d2ac470f320cade63663dae6ab8b8e1630b2fd5a3c607f45e2ee", + ChecksumType: "sha256", + Size: 6722, + }, + }, packages) +} + +func TestMetadataSHA256(t *testing.T) { + r := require.New(t) + + e := echo.New() + e.Use(middleware.Logger()) + e.Use(middleware.Recover()) + e.Static("/", "testdata/repo") + + srv := httptest.NewServer(e) + defer srv.Close() + + repo := New(srv.URL) + + _, err := repo.Packages(context.Background()) + r.NoError(err) + + md := repo.Metadata() + r.Equal(map[string]int{ + "repodata/1b4aca205bffe8d65f33b066e3f9965cb4c009e3c94b3f296cce8bff166ad8ed-primary.sqlite.bz2": 1995, + "repodata/2267234d92017b049818be743f720f37c176a3b3bb3e802ee4d5cd0090651091-primary.xml.gz": 720, + "repodata/2623c0a1472f574989dcba85417e8ce27b87983bba12922a6d91d574e617d2f6-filelists.sqlite.bz2": 858, + "repodata/314e73564000b8a68848551ce0fa9b36e11ed609698f232fa9ab5810ec531de1-filelists.xml.gz": 313, + "repodata/64f4875d92a3672f62a2d15d5f0ae6f0806451f42403bd07105214e1c9f4f0d7-other.sqlite.bz2": 749, + "repodata/e3984def0f3b5ce1b174fad2f6eb3c05829633d2d5d5d8ba05c9720ad59046e7-other.xml.gz": 281, + "repodata/repomd.xml": 3069, + }, func() map[string]int { + keys := map[string]int{} + for k, v := range md { + keys[k] = len(v) + } + return keys + }()) +} + +func TestMetadataSHA1(t *testing.T) { + r := require.New(t) + + e := echo.New() + e.Use(middleware.Logger()) + e.Use(middleware.Recover()) + e.Static("/", "testdata/repo-sha1") + + srv := httptest.NewServer(e) + defer srv.Close() + + repo := New(srv.URL) + + _, err := repo.Packages(context.Background()) + r.NoError(err) + + md := repo.Metadata() + r.Equal(map[string]int{ + "repodata/repomd.xml": 2601, + "repodata/4a11e3eeb25d21b08f41e5578d702d2bea21a2e7-filelists.xml.gz": 282, + "repodata/fdedb6ce109127d52228d01b0239010ddca14c8f-other.xml.gz": 247, + "repodata/e7a8a53e7398f6c22894718ea227fea60f2b78ba-primary.sqlite.bz2": 1937, + "repodata/c66ce2caa41ed83879f9b3dd9f40e61c65af499e-filelists.sqlite.bz2": 787, + "repodata/b31561a27d014d35b59b27c27859bb1c17ac573e-other.sqlite.bz2": 669, + "repodata/80779e2ab55e25a77124d370de1d08deae8f1cc6-primary.xml.gz": 688, + }, func() map[string]int { + keys := map[string]int{} + for k, v := range md { + keys[k] = len(v) + } + return keys + }()) +} diff --git a/cmd/cli/main.go b/cmd/cli/main.go index cfa7898..01da040 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -75,6 +75,8 @@ var ( Bool() versionCreateFromDir = versionCreate.Flag("from-dir", "create version right from directory"). String() + versionCreateFromYumRepo = versionCreate.Flag("from-yum-repo", "create version right from yum repository"). + String() versionDelete = version.Command("delete", "delete the given version") versionDeleteContainer = versionDelete.Arg("container", "name of the container to delete version of").Required().String() @@ -176,7 +178,7 @@ func main() { r.Register(containerDelete.FullCommand(), cliSvc.DeleteContainer(*containerDeleteName)) r.Register(versionList.FullCommand(), cliSvc.ListVersions(*versionListContainer)) - r.Register(versionCreate.FullCommand(), cliSvc.CreateVersion(*versionCreateContainer, *versionCreatePublish, versionCreateFromDir)) + r.Register(versionCreate.FullCommand(), cliSvc.CreateVersion(*versionCreateContainer, *versionCreatePublish, versionCreateFromDir, versionCreateFromYumRepo)) r.Register(versionDelete.FullCommand(), cliSvc.DeleteVersion(*versionDeleteContainer, *versionDeleteVersion)) r.Register(versionPublish.FullCommand(), cliSvc.PublishVersion(*versionPublishContainer, *versionPublishVersion)) diff --git a/docker-compose.yaml b/docker-compose.yaml index e4dfbdb..b0f41c7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,6 @@ services: postgresql: - image: postgres:16.3 + image: index.docker.io/library/postgres:16.3 environment: POSTGRES_PASSWORD: password ports: