Update models

lib/procedures/q_exps.py

@@ -6,7 +6,7 @@ import inspect
 import os
 import pprint
 import logging
-
+from copy import deepcopy
 import qlib
 from qlib.utils import init_instance_by_config
 from qlib.workflow import R
@@ -33,11 +33,14 @@ def set_log_basic_config(filename=None, format=None, level=None):
     if format is None:
         format = C.logging_config["formatters"]["logger_format"]["format"]
 
+    # Remove all handlers associated with the root logger object.
+    for handler in logging.root.handlers[:]:
+        logging.root.removeHandler(handler)
     logging.basicConfig(filename=filename, format=format, level=level)
 
 
 def update_gpu(config, gpu):
-    config = config.copy()
+    config = deepcopy(config)
     if "task" in config and "model" in config["task"]:
         if "GPU" in config["task"]["model"]:
             config["task"]["model"]["GPU"] = gpu
@@ -59,13 +62,20 @@ def update_gpu(config, gpu):
 
 
 def update_market(config, market):
-    config = config.copy()
+    config = deepcopy(config.copy())
     config["market"] = market
     config["data_handler_config"]["instruments"] = market
     return config
 
 
-def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
+def run_exp(
+    task_config,
+    dataset,
+    experiment_name,
+    recorder_name,
+    uri,
+    model_obj_name="model.pkl",
+):
 
     model = init_instance_by_config(task_config["model"])
     model_fit_kwargs = dict(dataset=dataset)
@@ -80,6 +90,7 @@ def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
         # Setup log
         recorder_root_dir = R.get_recorder().get_local_dir()
         log_file = os.path.join(recorder_root_dir, "{:}.log".format(experiment_name))
+
         set_log_basic_config(log_file)
         logger = get_module_logger("q.run_exp")
         logger.info("task_config::\n{:}".format(pprint.pformat(task_config, indent=2)))
@@ -87,20 +98,29 @@ def run_exp(task_config, dataset, experiment_name, recorder_name, uri):
         logger.info("dataset={:}".format(dataset))
 
         # Train model
-        R.log_params(**flatten_dict(task_config))
-        if "save_path" in inspect.getfullargspec(model.fit).args:
-            model_fit_kwargs["save_path"] = os.path.join(recorder_root_dir, "model.ckp")
-        elif "save_dir" in inspect.getfullargspec(model.fit).args:
-            model_fit_kwargs["save_dir"] = os.path.join(recorder_root_dir, "model-ckps")
-        model.fit(**model_fit_kwargs)
+        try:
+            model = R.load_object(model_obj_name)
+            logger.info("[Find existing object from {:}]".format(model_obj_name))
+        except OSError:
+            R.log_params(**flatten_dict(task_config))
+            if "save_path" in inspect.getfullargspec(model.fit).args:
+                model_fit_kwargs["save_path"] = os.path.join(
+                    recorder_root_dir, "model.ckp"
+                )
+            elif "save_dir" in inspect.getfullargspec(model.fit).args:
+                model_fit_kwargs["save_dir"] = os.path.join(
+                    recorder_root_dir, "model-ckps"
+                )
+            model.fit(**model_fit_kwargs)
+            R.save_objects(**{model_obj_name: model})
+        except:
+            raise ValueError("Something wrong.")
         # Get the recorder
         recorder = R.get_recorder()
-        R.save_objects(**{"model.pkl": model})
 
         # Generate records: prediction, backtest, and analysis
-        import pdb; pdb.set_trace()
         for record in task_config["record"]:
-            record = record.copy()
+            record = deepcopy(record)
             if record["class"] == "SignalRecord":
                 srconf = {"model": model, "dataset": dataset, "recorder": recorder}
                 record["kwargs"].update(srconf)

lib/trade_models/transformers.py

@@ -193,19 +193,15 @@ def get_transformer(config):
         raise ValueError("Invalid Configuration: {:}".format(config))
     name = config.get("name", "basic")
     if name == "basic":
-        model = TransformerModel(
+        model = SuperTransformer(
             d_feat=config.get("d_feat"),
-            embed_dim=config.get("embed_dim"),
-            depth=config.get("depth"),
+            stem_dim=config.get("stem_dim"),
+            embed_dims=config.get("embed_dims"),
             num_heads=config.get("num_heads"),
-            mlp_ratio=config.get("mlp_ratio"),
+            mlp_hidden_multipliers=config.get("mlp_hidden_multipliers"),
             qkv_bias=config.get("qkv_bias"),
-            qk_scale=config.get("qkv_scale"),
             pos_drop=config.get("pos_drop"),
-            mlp_drop_rate=config.get("mlp_drop_rate"),
-            attn_drop_rate=config.get("attn_drop_rate"),
-            drop_path_rate=config.get("drop_path_rate"),
-            norm_layer=config.get("norm_layer", None),
+            other_drop=config.get("other_drop"),
         )
     else:
         raise ValueError("Unknown model name: {:}".format(name))

lib/xlayers/super_module.py

@@ -14,6 +14,13 @@ IntSpaceType = Union[int, spaces.Integer, spaces.Categorical]
 BoolSpaceType = Union[bool, spaces.Categorical]
 
 
+class LayerOrder(Enum):
+    """This class defines the enumerations for order of operation in a residual or normalization-based layer."""
+
+    PreNorm = "pre-norm"
+    PostNorm = "post-norm"
+
+
 class SuperRunMode(Enum):
     """This class defines the enumerations for Super Model Running Mode."""
 

lib/xlayers/super_transformer.py

@@ -15,6 +15,7 @@ import torch.nn.functional as F
 import spaces
 from .super_module import IntSpaceType
 from .super_module import BoolSpaceType
+from .super_module import LayerOrder
 from .super_module import SuperModule
 from .super_linear import SuperMLPv2
 from .super_norm import SuperLayerNorm1D
@@ -30,7 +31,8 @@ class SuperTransformerEncoderLayer(SuperModule):
       - PyTorch Implementation: https://pytorch.org/docs/stable/_modules/torch/nn/modules/transformer.html#TransformerEncoderLayer
 
     Details:
-      MHA -> residual -> norm -> MLP -> residual -> norm
+      the original post-norm version: MHA -> residual -> norm -> MLP -> residual -> norm
+      the pre-norm version: norm -> MHA -> residual -> norm -> MLP -> residual
     """
 
     def __init__(
@@ -42,9 +44,10 @@ class SuperTransformerEncoderLayer(SuperModule):
         mlp_hidden_multiplier: IntSpaceType = 4,
         drop: Optional[float] = None,
         act_layer: Callable[[], nn.Module] = nn.GELU,
+        order: LayerOrder = LayerOrder.PreNorm,
     ):
         super(SuperTransformerEncoderLayer, self).__init__()
-        self.mha = SuperAttention(
+        mha = SuperAttention(
             input_dim,
             input_dim,
             num_heads=num_heads,
@@ -52,17 +55,33 @@ class SuperTransformerEncoderLayer(SuperModule):
             attn_drop=drop,
             proj_drop=drop,
         )
-        self.drop1 = nn.Dropout(drop or 0.0)
-        self.norm1 = SuperLayerNorm1D(input_dim)
-        self.mlp = SuperMLPv2(
+        drop1 = nn.Dropout(drop or 0.0)
+        norm1 = SuperLayerNorm1D(input_dim)
+        mlp = SuperMLPv2(
             input_dim,
             hidden_multiplier=mlp_hidden_multiplier,
             out_features=output_dim,
             act_layer=act_layer,
             drop=drop,
         )
-        self.drop2 = nn.Dropout(drop or 0.0)
-        self.norm2 = SuperLayerNorm1D(output_dim)
+        drop2 = nn.Dropout(drop or 0.0)
+        norm2 = SuperLayerNorm1D(output_dim)
+        if order is LayerOrder.PreNorm:
+            self.norm1 = norm1
+            self.mha = mha
+            self.drop1 = drop1
+            self.norm2 = norm2
+            self.mlp = mlp
+            self.drop2 = drop2
+        elif order is LayerOrder.PostNoem:
+            self.mha = mha
+            self.drop1 = drop1
+            self.norm1 = norm1
+            self.mlp = mlp
+            self.drop2 = drop2
+            self.norm2 = norm2
+        else:
+            raise ValueError("Unknown order: {:}".format(order))
 
     @property
     def abstract_search_space(self):
@@ -89,12 +108,18 @@ class SuperTransformerEncoderLayer(SuperModule):
         return self.forward_raw(input)
 
     def forward_raw(self, input: torch.Tensor) -> torch.Tensor:
-        # multi-head attention
-        x = self.mha(input)
-        x = x + self.drop1(x)
-        x = self.norm1(x)
-        # feed-forward layer
-        x = self.mlp(x)
-        x = x + self.drop2(x)
-        x = self.norm2(x)
+        if order is LayerOrder.PreNorm:
+            x = self.norm1(input)
+            x = x + self.drop1(self.mha(x))
+            x = self.norm2(x)
+            x = x + self.drop2(self.mlp(x))
+        elif order is LayerOrder.PostNoem:
+            # multi-head attention
+            x = x + self.drop1(self.mha(input))
+            x = self.norm1(x)
+            # feed-forward layer
+            x = x + self.drop2(self.mlp(x))
+            x = self.norm2(x)
+        else:
+            raise ValueError("Unknown order: {:}".format(order))
         return x