Migrate artifact hosting to cloudsmith

BUILD

@@ -41,7 +41,6 @@ stardoc(
         "//npm/deploy:lib",
         "//packer:lib",
         "//pip:lib",
-        "//rpm:lib",
     ],
     symbol_names = [
         # From: //apt:rules.bzl
@@ -106,9 +105,5 @@ stardoc(
         # From: //pip:rules.bzl
         "assemble_pip",
         "deploy_pip",
-
-        # From: //rpm:rules.bzl
-        "assemble_rpm",
-        "deploy_rpm",
     ],
 )

README.md

@@ -275,9 +275,9 @@ deploy_npm(<a href="#deploy_npm-name">name</a>, <a href="#deploy_npm-release">re
     ## How to generate an auth token
 
     ### Using the command line (`npm adduser`)
-    1. Run `npm adduser &lt;repo_url&gt;` (example: `npm adduser --registry=https://repo.vaticle.com/repository/npm-private`)
+    1. Run `npm adduser &lt;repo_url&gt;` (example: `npm adduser --registry=https://npm.cloudsmith.io/typedb/private/`)
     2. When prompted, provide login credentials to # to the user account that is used in your CI and has permissions to publish the package
-    3. If successful, a line will be added to your `.npmrc` file (`$HOME/.npmrc` on Unix) which looks like: `//repo.vaticle.com/repository/npm-snapshot/:_authToken=NpmToken.00000000-0000-0000-0000-000000000000`. The token is the value of `_authToken`, in this case `NpmToken.00000000-0000-0000-0000-000000000000`.
+    3. If successful, a line will be added to your `.npmrc` file (`$HOME/.npmrc` on Unix) which looks like: `//npm.cloudsmith.io/typedb/private/:_authToken=NpmToken.00000000-0000-0000-0000-000000000000`. The token is the value of `_authToken`, in this case `NpmToken.00000000-0000-0000-0000-000000000000`.
     4. Save the auth token somewhere safe and then delete it from your `.npmrc` file
 
     ### Using a UI
@@ -321,30 +321,6 @@ Execute Packer to perform deployment
 | <a id="deploy_packer-target"></a>target |  <code>assemble_packer</code> label to be deployed.   | <a href="https://bazel.build/concepts/labels">Label</a> | optional | <code>None</code> |
 
 
-<a id="deploy_rpm"></a>
-
-## deploy_rpm
-
-<pre>
-deploy_rpm(<a href="#deploy_rpm-name">name</a>, <a href="#deploy_rpm-release">release</a>, <a href="#deploy_rpm-snapshot">snapshot</a>, <a href="#deploy_rpm-target">target</a>)
-</pre>
-
-Deploy package built with `assemble_rpm` to RPM repository.
-
-    Select deployment to `snapshot` or `release` repository with `bazel run //:some-deploy-rpm -- [snapshot|release]
-
-
-**ATTRIBUTES**
-
-
-| Name  | Description | Type | Mandatory | Default |
-| :------------- | :------------- | :------------- | :------------- | :------------- |
-| <a id="deploy_rpm-name"></a>name |  A unique name for this target.   | <a href="https://bazel.build/concepts/labels#target-names">Name</a> | required |  |
-| <a id="deploy_rpm-release"></a>release |  Remote repository to deploy rpm release to   | String | required |  |
-| <a id="deploy_rpm-snapshot"></a>snapshot |  Remote repository to deploy rpm snapshot to   | String | required |  |
-| <a id="deploy_rpm-target"></a>target |  <code>assemble_rpm</code> target to deploy   | <a href="https://bazel.build/concepts/labels">Label</a> | optional | <code>None</code> |
-
-
 <a id="doxygen_docs"></a>
 
 ## doxygen_docs
@@ -651,36 +627,6 @@ Assemble files for HashiCorp Packer deployment
 | <a id="assemble_packer-files"></a>files |  Files to include into deployment   |  <code>{}</code> |
 
 
-<a id="assemble_rpm"></a>
-
-## assemble_rpm
-
-<pre>
-assemble_rpm(<a href="#assemble_rpm-name">name</a>, <a href="#assemble_rpm-package_name">package_name</a>, <a href="#assemble_rpm-spec_file">spec_file</a>, <a href="#assemble_rpm-version_file">version_file</a>, <a href="#assemble_rpm-workspace_refs">workspace_refs</a>, <a href="#assemble_rpm-installation_dir">installation_dir</a>,
-             <a href="#assemble_rpm-archives">archives</a>, <a href="#assemble_rpm-empty_dirs">empty_dirs</a>, <a href="#assemble_rpm-files">files</a>, <a href="#assemble_rpm-permissions">permissions</a>, <a href="#assemble_rpm-symlinks">symlinks</a>, <a href="#assemble_rpm-tags">tags</a>)
-</pre>
-
-Assemble package for installation with RPM
-
-**PARAMETERS**
-
-
-| Name  | Description | Default Value |
-| :------------- | :------------- | :------------- |
-| <a id="assemble_rpm-name"></a>name |  A unique name for this target.   |  none |
-| <a id="assemble_rpm-package_name"></a>package_name |  Package name for built .rpm package   |  none |
-| <a id="assemble_rpm-spec_file"></a>spec_file |  The RPM spec file to use   |  none |
-| <a id="assemble_rpm-version_file"></a>version_file |  File containing version number of a package. Alternatively, pass --define version=VERSION to Bazel invocation. Not specifying version defaults to '0.0.0'   |  <code>None</code> |
-| <a id="assemble_rpm-workspace_refs"></a>workspace_refs |  JSON file with other Bazel workspace references   |  <code>None</code> |
-| <a id="assemble_rpm-installation_dir"></a>installation_dir |  directory into which .rpm package is unpacked at installation   |  <code>None</code> |
-| <a id="assemble_rpm-archives"></a>archives |  Bazel labels of archives that go into .rpm package   |  <code>[]</code> |
-| <a id="assemble_rpm-empty_dirs"></a>empty_dirs |  list of empty directories created at package installation   |  <code>[]</code> |
-| <a id="assemble_rpm-files"></a>files |  mapping between Bazel labels of archives that go into .rpm package and their resulting location on .rpm package installation   |  <code>{}</code> |
-| <a id="assemble_rpm-permissions"></a>permissions |  mapping between paths and UNIX permissions   |  <code>{}</code> |
-| <a id="assemble_rpm-symlinks"></a>symlinks |  mapping between source and target of symbolic links created at installation   |  <code>{}</code> |
-| <a id="assemble_rpm-tags"></a>tags |  additional tags passed to all wrapped rules   |  <code>[]</code> |
-
-
 <a id="assemble_targz"></a>
 
 ## assemble_targz

WORKSPACE

@@ -95,3 +95,9 @@ http_archive(
 
 load("@bazel_stardoc//:setup.bzl", "stardoc_repositories")
 stardoc_repositories()
+
+# Load @vaticle_bazel_distribution_uploader
+load("//common/uploader:deps.bzl", uploader_deps = "deps")
+uploader_deps()
+load("@vaticle_bazel_distribution_uploader//:requirements.bzl", install_uploader_deps = "install_deps")
+install_uploader_deps()

apt/rules.bzl

@@ -187,27 +187,25 @@ def assemble_apt(name,
 
 
 def _deploy_apt_impl(ctx):
+    _deploy_script = ctx.actions.declare_file(ctx.attr.deploy_script_name)
+    package_path = ctx.files.target[0].short_path
     ctx.actions.expand_template(
         template = ctx.file._deployment_script,
-        output = ctx.outputs.deployment_script,
+        output = _deploy_script,
         substitutions = {
             '{snapshot}' : ctx.attr.snapshot,
-            '{release}' : ctx.attr.release
+            '{release}' : ctx.attr.release,
+            '{package_path}' : package_path,
         },
         is_executable = True
     )
 
-    symlinks = {
-        'package.deb': ctx.files.target[0],
-    }
+    deployment_lib_files = ctx.attr._deployment_wrapper_lib[DefaultInfo].default_runfiles.files.to_list()
+    return DefaultInfo(executable = _deploy_script,
+                       runfiles = ctx.runfiles(files=[ctx.files.target[0]] + deployment_lib_files))
 
-    return DefaultInfo(executable = ctx.outputs.deployment_script,
-                       runfiles = ctx.runfiles(
-                           files=[ctx.files.target[0]],
-                           symlinks = symlinks))
 
-
-deploy_apt = rule(
+_deploy_apt = rule(
     attrs = {
         "target": attr.label(
             doc = 'assemble_apt label to deploy'
@@ -220,13 +218,17 @@ deploy_apt = rule(
             mandatory = True,
             doc = 'Release repository to deploy apt artifact to'
         ),
+        "_deployment_wrapper_lib": attr.label(
+            default = "//common/uploader:uploader",
+        ),
         "_deployment_script": attr.label(
             allow_single_file = True,
             default = "//apt/templates:deploy.py"
         ),
-    },
-    outputs = {
-        "deployment_script": "%{name}.sh",
+        "deploy_script_name": attr.string(
+            mandatory = True,
+            doc = 'Name of instantiated deployment script'
+        ),
     },
     implementation = _deploy_apt_impl,
     executable = True,
@@ -235,3 +237,22 @@ deploy_apt = rule(
     Select deployment to `snapshot` or `release` repository with `bazel run //:some-deploy-apt -- [snapshot|release]
     """
 )
+
+def deploy_apt(name, target, snapshot, release, **kwargs):
+    deploy_script_target_name = name + "__deploy"
+    deploy_script_name = deploy_script_target_name + "-deploy.py"
+
+    _deploy_apt(
+        name = deploy_script_target_name,
+        target = target,
+        snapshot = snapshot,
+        release = release,
+        deploy_script_name = deploy_script_name,
+        **kwargs
+    )
+
+    native.py_binary(
+        name = name,
+        srcs = [deploy_script_target_name],
+        main = deploy_script_name,
+    )

apt/templates/deploy.py

@@ -31,18 +31,25 @@
 import tempfile
 from runpy import run_path
 
+import sys, glob
+# Prefer using the runfile dependency than system dependency
+runfile_deps = [path for path in map(os.path.abspath, glob.glob('external/*/*'))]
+sys.path = runfile_deps + sys.path
+
+from common.uploader.uploader import Uploader
+
 parser = argparse.ArgumentParser()
 parser.add_argument('repo_type')
 args = parser.parse_args()
 
 repo_type_key = args.repo_type
 
-apt_deployments = {
+apt_repositories = {
     'snapshot' : "{snapshot}",
     'release' : "{release}"
 }
 
-apt_registry = apt_deployments[repo_type_key]
+repo_url = apt_repositories[repo_type_key]
 
 apt_username, apt_password = (
     os.getenv('DEPLOY_APT_USERNAME'),
@@ -61,19 +68,9 @@
         '$DEPLOY_APT_PASSWORD env variable'
     )
 
-upload_status_code = subprocess.check_output([
-    'curl',
-    '--silent',
-    '--output', '/dev/stderr',
-    '--write-out', '%{http_code}',
-    '-u', '{}:{}'.format(apt_username, apt_password),
-    '-X', 'POST',
-    '-H', 'Content-Type: multipart/form-data',
-    '--data-binary', '@package.deb',
-    apt_registry
-]).decode().strip()
+package_path = "{package_path}"
 
-if upload_status_code != '201':
-    raise Exception('upload failed, got HTTP status code {}'.format(upload_status_code))
+uploader = Uploader.create(apt_username, apt_password, repo_url)
+uploader.apt(package_path)
 
 print('Deployment completed.')

artifact/rules.bzl

@@ -51,11 +51,11 @@ def _deploy_artifact_impl(ctx):
     symlinks = {
         'VERSION': version_file,
     }
-
+    deployment_lib_files = ctx.attr._deployment_wrapper_lib[DefaultInfo].default_runfiles.files.to_list()
     return DefaultInfo(
         executable = _deploy_script,
         runfiles = ctx.runfiles(
-            files = files,
+            files = files + deployment_lib_files,
             symlinks = symlinks,
         ),
     )
@@ -84,6 +84,9 @@ _deploy_artifact = rule(
             doc = "The artifact filename, automatic from the target file if not specified",
             default = '',
         ),
+        "_deployment_wrapper_lib": attr.label(
+            default = "//common/uploader:uploader",
+        ),
         "_deploy_script_template": attr.label(
             allow_single_file = True,
             default = "//artifact/templates:deploy.py",
@@ -165,7 +168,7 @@ def artifact_file(name,
 
     http_file(
         name = name,
-        urls = ["{}/{}/{}/{}".format(repository_url, group_name, version, artifact_name)],
+        urls = ["{}/names/{}/versions/{}/{}".format(repository_url.rstrip("/"), group_name, version, artifact_name)],
         downloaded_file_path = artifact_name,
         sha = sha,
         tags = tags + ["{}={}".format(versiontype, version)],

artifact/templates/deploy.py

@@ -27,20 +27,12 @@
 import sys
 from posixpath import join as urljoin
 
+import glob
+# Prefer using the runfile dependency than system dependency
+runfile_deps = [path for path in map(os.path.abspath, glob.glob('external/*/*'))]
+sys.path = runfile_deps + sys.path
 
-def upload(url, username, password, local_fn, remote_fn):
-    upload_status_code = sp.check_output([
-        'curl', '--silent',
-        '--write-out', '%{http_code}',
-        '-u', '{}:{}'.format(username, password),
-        '--upload-file', local_fn,
-        urljoin(url, remote_fn)
-    ]).decode().strip()
-
-    if upload_status_code != '201':
-        raise Exception('upload of {} failed, got HTTP status code {}'.format(
-            local_fn, upload_status_code))
-
+from common.uploader.uploader import Uploader
 
 if len(sys.argv) != 2:
     raise ValueError('Should pass only <snapshot|release> as arguments')
@@ -75,9 +67,6 @@ def upload(url, username, password, local_fn, remote_fn):
                      .format(version, repo_type, version_snapshot_regex))
 
 filename = '{artifact_filename}'
-if filename == '':
-    filename = os.path.basename('{artifact_path}')
-
 filename = filename.format(version = version)
 
 base_url = None
@@ -86,6 +75,5 @@ def upload(url, username, password, local_fn, remote_fn):
 else:
     base_url = '{snapshot}'
 
-dir_url = '{base_url}/{artifact_group}/{version}'.format(version=version, base_url=base_url)
-
-upload(dir_url, username, password, '{artifact_path}', filename)
+uploader = Uploader.create(username, password, base_url)
+uploader.artifact("{artifact_group}", version, '{artifact_path}', filename)

rpm/BUILD → common/uploader/BUILD

@@ -16,23 +16,11 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+load("@vaticle_bazel_distribution_uploader//:requirements.bzl", cloudsmith_requirement = "requirement")
 
-load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-
-bzl_library(
-    name = "lib",
-    srcs = [
-        "rules.bzl",
-        "@rules_pkg//:rpm.bzl",
-    ],
-    deps = [
-        "@rules_pkg//doc_build:rules_pkg_lib",
-    ],
-    visibility = ["//visibility:public"]
-)
-
-py_binary(
-    name = "generate_spec_file",
-    srcs = ["generate_spec_file.py"],
-    visibility = ["//visibility:public"]
+py_library(
+    name = "uploader",
+    srcs = glob(["*.py"]),
+    deps = [cloudsmith_requirement("requests")],
+    visibility = ["//visibility:public"],
 )