add prepare for redis tls config

Signed-off-by: yminer <miner.yang@broadcom.com>

make/harbor.yml.tmpl

@@ -215,6 +215,13 @@ _version: 2.12.0
 #   # username:
 #   # sentinel_master_set must be set to support redis+sentinel
 #   #sentinel_master_set:
+#   # tls configuration for redis connection
+#   # only server-authentication is supported
+#   # mtls for redis connection is not supported
+#   # tls connection will be disable by default 
+#   tlsOptions:
+#   # if the tlsOptions.rootCA has been specified, then tls connection will be enabled.
+#     rootCA:
 #   # db_index 0 is for core, it's unchangeable
 #   registry_db_index: 1
 #   jobservice_db_index: 2

make/photon/prepare/g.py

@@ -27,6 +27,7 @@
 
 storage_ca_bundle_filename = 'storage_ca_bundle.crt'
 internal_ca_filename = 'harbor_internal_ca.crt'
+redis_tls_ca_filename = 'redis_tls_ca.crt'
 
 old_private_key_pem_path = Path('/config/core/private_key.pem')
 old_crt_path = Path('/config/registry/root.crt')

make/photon/prepare/templates/registry/config.yml.jinja

@@ -40,6 +40,7 @@ redis:
   dialtimeout: 10s
   password: {{redis_password}}
   db: {{redis_db_index_reg}}
+  enableTLS: {{redis_enableTLS}}
   pool:
     maxidle: 100
     maxactive: 500

make/photon/prepare/utils/cert.py

@@ -4,7 +4,7 @@
 from subprocess import DEVNULL
 import logging
 
-from g import DEFAULT_GID, DEFAULT_UID, shared_cert_dir, storage_ca_bundle_filename, internal_tls_dir, internal_ca_filename
+from g import DEFAULT_GID, DEFAULT_UID, shared_cert_dir, storage_ca_bundle_filename, internal_tls_dir, internal_ca_filename, redis_tls_ca_filename
 from .misc import (
     mark_file,
     generate_random_string,
@@ -120,18 +120,23 @@ def prepare_trust_ca(config_dict):
 
     internal_ca_src = internal_tls_dir.joinpath(internal_ca_filename)
     ca_bundle_src = config_dict.get('registry_custom_ca_bundle_path')
+    redis_tls_ca_src = config_dict.get('redis_custom_tls_ca_path')
     for src_path, dst_filename in (
         (internal_ca_src, internal_ca_filename),
-        (ca_bundle_src, storage_ca_bundle_filename)):
+        (ca_bundle_src, storage_ca_bundle_filename),
+        (redis_tls_ca_src, redis_tls_ca_filename)):
+        print('copy {} to shared trust ca dir as name {} ...'.format(src_path, dst_filename))
         logging.info('copy {} to shared trust ca dir as name {} ...'.format(src_path, dst_filename))
         # check if source file valied
         if not src_path:
             continue
         real_src_path = get_realpath(str(src_path))
         if not real_src_path.exists():
+            print('ca file {} is not exist'.format(real_src_path))
             logging.info('ca file {} is not exist'.format(real_src_path))
             continue
         if not real_src_path.is_file():
+            print('{} is not file'.format(real_src_path))
             logging.info('{} is not file'.format(real_src_path))
             continue
 

make/photon/prepare/utils/configs.py

@@ -1,3 +1,4 @@
+from distutils.command.config import config
 import logging
 import os
 import yaml
@@ -354,6 +355,11 @@ def parse_yaml_config(config_file_path, with_trivy):
 
     return config_dict
 
+def get_redis_schema(redis=None):
+    if 'tlsOptions' in redis and redis['tlsOptions'].get('rootCA') is not None:
+        return redis.get('sentinel_master_set', None) and 'rediss+sentinel' or 'rediss'
+    else:
+        return redis.get('sentinel_master_set', None) and 'redis+sentinel' or 'redis'
 
 def get_redis_url(db, redis=None):
     """Returns redis url with format `redis://[arbitrary_username:password@]ipaddress:port/database_index?idle_timeout_seconds=30`
@@ -373,7 +379,7 @@ def get_redis_url(db, redis=None):
         'password': '',
     }
     kwargs.update(redis or {})
-    kwargs['scheme'] = kwargs.get('sentinel_master_set', None) and 'redis+sentinel' or 'redis'
+    kwargs['scheme'] = get_redis_schema(kwargs)
     kwargs['db_part'] = db and ("/%s" % db) or ""
     kwargs['sentinel_part'] = kwargs.get('sentinel_master_set', None) and ("/" + kwargs['sentinel_master_set']) or ''
     kwargs['password_part'] = quote(str(kwargs.get('password', None)), safe='') and (':%s@' % quote(str(kwargs['password']), safe='')) or ''
@@ -458,5 +464,8 @@ def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True)
 
     if with_trivy:
         configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)
+
+    if 'tlsOptions' in redis and redis['tlsOptions'].get('rootCA') is not None:
+        configs['redis_custom_tls_ca_path'] = redis['tlsOptions']['rootCA']
 
     return configs

make/photon/prepare/utils/registry.py

@@ -48,13 +48,30 @@ def parse_redis(redis_url):
             'redis_host': u.netloc.split('@')[-1],
             'redis_password': '' if u.password is None else unquote(u.password),
             'redis_db_index_reg': u.path and int(u.path[1:]) or 0,
+            'redis_enableTLS': 'false',
+        }
+    elif u.scheme == 'rediss':
+        return {
+            'redis_host': u.netloc.split('@')[-1],
+            'redis_password': '' if u.password is None else unquote(u.password),
+            'redis_db_index_reg': u.path and int(u.path[1:]) or 0,
+            'redis_enableTLS': 'true',
         }
     elif u.scheme == 'redis+sentinel':
         return {
             'sentinel_master_set': u.path.split('/')[1],
             'redis_host': u.netloc.split('@')[-1],
             'redis_password': '' if u.password is None else unquote(u.password),
             'redis_db_index_reg': len(u.path.split('/')) == 3 and int(u.path.split('/')[2]) or 0,
+            'redis_enableTLS': 'false',
+        }
+    elif u.scheme == 'rediss+sentinel':
+        return {
+            'sentinel_master_set': u.path.split('/')[1],
+            'redis_host': u.netloc.split('@')[-1],
+            'redis_password': '' if u.password is None else unquote(u.password),
+            'redis_db_index_reg': len(u.path.split('/')) == 3 and int(u.path.split('/')[2]) or 0,
+            'redis_enableTLS': 'true',
         }
     else:
         raise Exception('bad redis url for registry:' + redis_url)