Supporting Arctic Backend Provider & Orderbook, Tick Data Example

qlib/__init__.py

@@ -11,12 +11,12 @@
 import platform
 import subprocess
 from .log import get_module_logger
-
+from arctic import Arctic
 
 # init qlib
 def init(default_conf="client", **kwargs):
-    from .config import C
-    from .data.cache import H
+    from .config import C, Arctic_Connection_List
+    from .data.cache import H, HZ
 
     # FIXME: this logger ignored the level in config
     logger = get_module_logger("Initialization", level=logging.INFO)
@@ -28,7 +28,11 @@ def init(default_conf="client", **kwargs):
         logger.warning("Skip initialization because `skip_if_reg is True`")
         return
 
+    if len(Arctic_Connection_List) == 0:
+        Arctic_Connection_List.append(Arctic(C.arctic_uri))
+
     H.clear()
+    HZ.clear()
     C.set(default_conf, **kwargs)
 
     # mount nfs

qlib/config.py

@@ -77,6 +77,9 @@ def set_conf_from_C(self, config_c):
 PROTOCOL_VERSION = 4
 
 NUM_USABLE_CPU = max(multiprocessing.cpu_count() - 2, 1)
+NUM_USABLE_CPU_FOR_ARCTIC = min(NUM_USABLE_CPU, 30)
+ARCTIC_TIME_INTERVAL = 1
+ARCTIC_RETRY_TIME = 5
 
 DISK_DATASET_CACHE = "DiskDatasetCache"
 SIMPLE_DATASET_CACHE = "SimpleDatasetCache"
@@ -103,13 +106,19 @@ def set_conf_from_C(self, config_c):
     #   2. backend_config: backend_obj["kwargs"]["provider_uri_map"]
     #   3. qlib.init: provider_uri
     "provider_uri": "",
+    "arctic_uri": "",
+    "arctic_time_interval": ARCTIC_TIME_INTERVAL,
+    "arctic_retry_time": ARCTIC_RETRY_TIME,
     # cache
     "expression_cache": None,
     "dataset_cache": None,
     "calendar_cache": None,
+    # for market transaction time
+    "market_transaction_time_list": [("09:15", "11:30"), ("13:00", "15:00")],
     # for simple dataset cache
     "local_cache_path": None,
     "kernels": NUM_USABLE_CPU,
+    "arctic_kernels": NUM_USABLE_CPU_FOR_ARCTIC,
     # pickle.dump protocol version
     "dump_protocol_version": PROTOCOL_VERSION,
     # How many tasks belong to one process. Recommend 1 for high-frequency data and None for daily data.
@@ -122,7 +131,7 @@ def set_conf_from_C(self, config_c):
     # default 1 hour
     "mem_cache_expire": 60 * 60,
     # memory cache space limit, default 5GB, only in used client
-    "mem_cache_space_limit": 1024 * 1024 * 1024 * 5,
+    "mem_cache_space_limit": 1024 * 1024 * 1024,
     # cache dir name
     "dataset_cache_dir_name": "dataset_cache",
     "features_cache_dir_name": "features_cache",
@@ -232,6 +241,31 @@ def set_conf_from_C(self, config_c):
         # if element of custom_ops is dict, it represents the config of custom operator and should include `class` and `module_path` keys.
         "custom_ops": [],
     },
+    "arctic_client": {
+        # data provider config
+        "calendar_provider": "LocalCalendarProvider",
+        "instrument_provider": "LocalInstrumentProvider",
+        "feature_provider": "LocalFeatureProvider",
+        "expression_provider": "LocalExpressionProvider",
+        "dataset_provider": "LocalDatasetProvider",
+        "provider": "LocalProvider",
+        # config it in user's own code
+        "provider_uri": "~/.qlib/qlib_data/cn_data",
+        # cache
+        # Using parameter 'remote' to announce the client is using server_cache, and the writing access will be disabled.
+        "expression_cache": None,
+        "dataset_cache": None,
+        "calendar_cache": None,
+        # client config
+        "kernels": NUM_USABLE_CPU,
+        "mount_path": None,
+        "auto_mount": False,  # The nfs is already mounted on our server[auto_mount: False].
+        # The nfs should be auto-mounted by qlib on other
+        # serversS(such as PAI) [auto_mount:True]
+        "timeout": 100,
+        "logging_level": "INFO",
+        "region": REG_CN,
+    },
 }
 
 HIGH_FREQ_CONFIG = {
@@ -450,3 +484,5 @@ def registered(self):
 
 # global config
 C = QlibConfig(_default_config)
+# for arctic connection
+Arctic_Connection_List = []

qlib/contrib/model/pytorch_nn.py

@@ -26,7 +26,6 @@
 
 class DNNModelPytorch(Model):
     """DNN Model
-
     Parameters
     ----------
     input_dim : int