diff --git a/docker/build.sh b/docker/build.sh index d3c823f5..56187851 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -104,13 +104,38 @@ gquant_ver=$(grep version gQuant/setup.py | sed "s/^.*version='\([^;]*\)'.*/\1/" CONTAINER="nvidia/cuda:${CUDA_STR}-runtime-${OS_STR}" D_CONT=${D_CONT:="gquant/gquant:${gquant_ver}-${CUDA_STR}_${OS_STR}_${RAPIDS_VERSION}_${MODE_STR}"} + +cat << 'EOF' > sacrebleu.patch +--- nemo/collections/nlp/metrics/sacrebleu.py ++++ sacrebleu_fix.py +@@ -61,13 +61,16 @@ + VERSION = '1.3.5' + + try: ++ import threading + # SIGPIPE is not available on Windows machines, throwing an exception. + from signal import SIGPIPE + + # If SIGPIPE is available, change behaviour to default instead of ignore. + from signal import signal, SIG_DFL + +- signal(SIGPIPE, SIG_DFL) ++ ++ if threading.current_thread() == threading.main_thread(): ++ signal(SIGPIPE, SIG_DFL) + + except ImportError: + logging.warning('Could not import signal.SIGPIPE (this is expected on Windows machines)') +EOF + + cat > $D_FILE < 0).astype('int64') + return {self.OUTPUT_PORT_NAME: input_df} + + def columns_setup(self): + name = self.conf.get('sign', 'sign') + addition = {name: "int64"} + return _PortTypesMixin.addition_columns_setup(self, addition) \ No newline at end of file diff --git a/gquant/plugin_nodes/transform/daskComputeNode.py b/gquant/plugin_nodes/transform/daskComputeNode.py new file mode 100644 index 00000000..5b8db0e8 --- /dev/null +++ b/gquant/plugin_nodes/transform/daskComputeNode.py @@ -0,0 +1,60 @@ +import pandas as pd +import cudf +from dask.dataframe import DataFrame as DaskDataFrame +import dask_cudf + +from gquant.dataframe_flow import Node +from gquant.dataframe_flow.portsSpecSchema import ConfSchema +from gquant.dataframe_flow._port_type_node import _PortTypesMixin + +from gquant.dataframe_flow.portsSpecSchema import ( + PortsSpecSchema, NodePorts) + +__all__ = ["DaskComputeNode"] + + +class DaskComputeNode(Node): + + def init(self): + _PortTypesMixin.init(self) + self.INPUT_PORT_NAME = 'in' + self.OUTPUT_PORT_NAME = 'out' + # no required columns + self.required[self.INPUT_PORT_NAME] = {} + + def conf_schema(self): + json = { + "title": "Run dask compute", + "type": "object", + "description": "If the input is a dask or dask_cudf dataframe " + "then run compute on it, otherwise pass through." + } + + return ConfSchema(json=json) + + def ports_setup(self): + port_type = PortsSpecSchema.port_type + + input_connections = self.get_connected_inports() + if (self.INPUT_PORT_NAME in input_connections): + determined_type = input_connections[self.INPUT_PORT_NAME] + inports = {self.INPUT_PORT_NAME: {port_type: determined_type}} + else: + intypes = [cudf.DataFrame, dask_cudf.DataFrame, + pd.DataFrame, DaskDataFrame] + inports = {self.INPUT_PORT_NAME: {port_type: intypes}} + + out_types = [cudf.DataFrame, pd.DataFrame] + outports = {self.OUTPUT_PORT_NAME: {port_type: out_types}} + + return NodePorts(inports=inports, outports=outports) + + def columns_setup(self): + '''Pass through columns from inputs to outputs''' + return _PortTypesMixin.columns_setup(self) + + def process(self, inputs): + din = inputs[self.INPUT_PORT_NAME] + dout = din.compute() if isinstance(din, DaskDataFrame) else din + + return {self.OUTPUT_PORT_NAME: dout} diff --git a/gquant/plugin_nodes/transform/data_obj.py b/gquant/plugin_nodes/transform/data_obj.py new file mode 100644 index 00000000..a022e774 --- /dev/null +++ b/gquant/plugin_nodes/transform/data_obj.py @@ -0,0 +1,10 @@ +class NormalizationData(object): + + def __init__(self, data): + self.data = data + + +class ProjectionData(object): + + def __init__(self, data): + self.data = data \ No newline at end of file diff --git a/gquant/plugin_nodes/transform/linearEmbedding.py b/gquant/plugin_nodes/transform/linearEmbedding.py new file mode 100644 index 00000000..a56f6c22 --- /dev/null +++ b/gquant/plugin_nodes/transform/linearEmbedding.py @@ -0,0 +1,252 @@ +from gquant.dataframe_flow import Node +from gquant.dataframe_flow._port_type_node import _PortTypesMixin +from gquant.dataframe_flow.portsSpecSchema import (ConfSchema, + PortsSpecSchema, NodePorts) +from .data_obj import ProjectionData +import cupy as cp +import copy +from collections import OrderedDict + +__all__ = ['LinearEmbeddingNode'] + +SPECIAL_OUTPUT_DIM_COL = 'OUTPUT_DIM_23b1c5ce-e0bf-11ea-afcf-80e82cc76d44' + + +class LinearEmbeddingNode(Node, _PortTypesMixin): + + def init(self): + _PortTypesMixin.init(self) + self.INPUT_PORT_NAME = 'df_in' + self.OUTPUT_PORT_NAME = 'df_out' + self.INPUT_PROJ_NAME = 'proj_data_in' + self.OUTPUT_PROJ_NAME = 'proj_data_out' + cols_required = {} + self.required = { + self.INPUT_PORT_NAME: cols_required, + self.INPUT_PROJ_NAME: cols_required + } + + def ports_setup_from_types(self, types): + """ + overwrite the _PortTypesMixin.ports_setup_from_types + method, which is invoked by the _PortTypesMixin.ports_setup + """ + port_type = PortsSpecSchema.port_type + input_ports = { + self.INPUT_PORT_NAME: { + port_type: types + }, + self.INPUT_PROJ_NAME: { + port_type: ProjectionData + } + } + + output_ports = { + self.OUTPUT_PORT_NAME: { + port_type: types + }, + self.OUTPUT_PROJ_NAME: { + port_type: ProjectionData + } + } + + input_connections = self.get_connected_inports() + if self.INPUT_PORT_NAME in input_connections: + determined_type = input_connections[self.INPUT_PORT_NAME] + input_ports.update({self.INPUT_PORT_NAME: + {port_type: determined_type}}) + output_ports.update({self.OUTPUT_PORT_NAME: { + port_type: determined_type}}) + # connected + return NodePorts(inports=input_ports, + outports=output_ports) + else: + return NodePorts(inports=input_ports, outports=output_ports) + + def columns_setup(self): + self.required = { + self.INPUT_PORT_NAME: {}, + self.INPUT_PROJ_NAME: {} + } + if 'columns' in self.conf and self.conf.get('include', True): + cols_required = {} + for col in self.conf['columns']: + cols_required[col] = None + self.required = { + self.INPUT_PORT_NAME: cols_required, + self.INPUT_PROJ_NAME: cols_required + } + output_cols = { + self.OUTPUT_PORT_NAME: self.required[self.INPUT_PORT_NAME], + self.OUTPUT_PROJ_NAME: self.required[ + self.INPUT_PROJ_NAME] + } + input_columns = self.get_input_columns() + if (self.INPUT_PROJ_NAME in input_columns and + self.INPUT_PORT_NAME in input_columns): + cols_required = copy.copy(input_columns[self.INPUT_PROJ_NAME]) + self.required = { + self.INPUT_PORT_NAME: cols_required, + self.INPUT_PROJ_NAME: cols_required + } + col_from_inport = input_columns[self.INPUT_PORT_NAME] + if SPECIAL_OUTPUT_DIM_COL in cols_required: + out_dim = cols_required[SPECIAL_OUTPUT_DIM_COL] + del cols_required[SPECIAL_OUTPUT_DIM_COL] + cols = ['em'+str(i) for i in range(out_dim)] + for col in cols: + col_from_inport[col] = None + output_cols = { + self.OUTPUT_PORT_NAME: col_from_inport, + self.OUTPUT_PROJ_NAME: cols_required + } + return output_cols + elif (self.INPUT_PROJ_NAME in input_columns and + self.INPUT_PORT_NAME not in input_columns): + cols_required = copy.copy(input_columns[self.INPUT_PROJ_NAME]) + self.required = { + self.INPUT_PORT_NAME: cols_required, + self.INPUT_PROJ_NAME: cols_required + } + output = copy.copy(cols_required) + if SPECIAL_OUTPUT_DIM_COL in cols_required: + out_dim = cols_required[SPECIAL_OUTPUT_DIM_COL] + del cols_required[SPECIAL_OUTPUT_DIM_COL] + cols = ['em'+str(i) for i in range(out_dim)] + for col in cols: + output[col] = None + output_cols = { + self.OUTPUT_PORT_NAME: output, + self.OUTPUT_PROJ_NAME: cols_required + } + return output_cols + elif (self.INPUT_PROJ_NAME not in input_columns and + self.INPUT_PORT_NAME in input_columns): + col_from_inport = input_columns[self.INPUT_PORT_NAME] + enums = [col for col in col_from_inport.keys()] + if 'columns' in self.conf: + if self.conf.get('include', True): + included_colums = self.conf['columns'] + else: + included_colums = [col for col in enums + if col not in self.conf['columns']] + cols_required = OrderedDict() + for col in included_colums: + if col in col_from_inport: + cols_required[col] = col_from_inport[col] + else: + cols_required[col] = None + self.required = { + self.INPUT_PORT_NAME: cols_required, + self.INPUT_PROJ_NAME: cols_required + } + col_dict = ['em'+str(i) for i in range( + self.conf['out_dimension'])] + for col in col_dict: + col_from_inport[col] = None + proj_out = copy.copy(cols_required) + proj_out[SPECIAL_OUTPUT_DIM_COL] = self.conf['out_dimension'] + output_cols = { + self.OUTPUT_PORT_NAME: col_from_inport, + self.OUTPUT_PROJ_NAME: proj_out + } + return output_cols + return output_cols + + def ports_setup(self): + return _PortTypesMixin.ports_setup(self) + + def conf_schema(self): + json = { + "title": "Linear Embeding configure", + "type": "object", + "description": """Project the features randomly and linearly to a + space of different dimension. It generates the random projection + matrix of size feature_dim x out_dimension and does dot product + with the input dataframe""", + "properties": { + "columns": { + "type": "array", + "description": """an array of columns that need to + be normalized, or excluded from normalization depending + on the `incldue` flag state""", + "items": { + "type": "string" + } + }, + "include": { + "type": "boolean", + "description": """if set true, the `columns` need to be + normalized. if false, all dataframe columns except the + `columns` need to be normalized""", + "default": True + }, + "out_dimension": { + "type": "integer", + "minimum": 0, + "description": """the projected dimension size""" + }, + "seed": { + "type": "integer", + "description": """the seed number for random projection""" + } + }, + "required": [], + } + ui = {} + input_columns = self.get_input_columns() + if self.INPUT_PORT_NAME in input_columns: + col_from_inport = input_columns[self.INPUT_PORT_NAME] + enums = [col for col in col_from_inport.keys()] + json['properties']['columns']['items']['enum'] = enums + return ConfSchema(json=json, ui=ui) + + def process(self, inputs): + """ + normalize the data to zero mean, std 1 + + Arguments + ------- + inputs: list + list of input dataframes. + Returns + ------- + dataframe + """ + input_df = inputs[self.INPUT_PORT_NAME] + + if self.INPUT_PROJ_NAME in inputs: + data_in = inputs[self.INPUT_PROJ_NAME].data + input_columns = self.get_input_columns() + col_from_inport = input_columns[self.INPUT_PROJ_NAME] + proj_data = data_in + cols = [] + # it has required colmns that used to do mapping + for col in col_from_inport.keys(): + if col != SPECIAL_OUTPUT_DIM_COL: + cols.append(col) + else: + if self.conf.get('include', True): + cols = self.conf['columns'] + else: + cols = input_df.columns.difference( + self.conf['columns']).values.tolist() + # need to generate the random projection + if 'seed' in self.conf: + cp.random.seed(self.conf['seed']) + proj_data = cp.random.rand(len(cols), self.conf['out_dimension']) + cols.sort() + # print(self.uid, cols) + # print(self.uid, proj_data) + output_matrix = input_df[cols].values.dot(proj_data) + col_dict = {'em'+str(i): output_matrix[:, i] + for i in range(proj_data.shape[1])} + # output_df = input_df[input_df.columns.difference(cols)] + output_df = input_df.assign(**col_dict) + output = {} + if self.outport_connected(self.OUTPUT_PORT_NAME): + output.update({self.OUTPUT_PORT_NAME: output_df}) + if self.outport_connected(self.OUTPUT_PROJ_NAME): + payload = ProjectionData(proj_data) + output.update({self.OUTPUT_PROJ_NAME: payload}) + return output diff --git a/gquant/plugin_nodes/transform/normalizationNode.py b/gquant/plugin_nodes/transform/normalizationNode.py new file mode 100644 index 00000000..43cb3be1 --- /dev/null +++ b/gquant/plugin_nodes/transform/normalizationNode.py @@ -0,0 +1,208 @@ +from gquant.dataframe_flow import Node +from gquant.dataframe_flow._port_type_node import _PortTypesMixin +from gquant.dataframe_flow.portsSpecSchema import (ConfSchema, + PortsSpecSchema, NodePorts) +from .data_obj import NormalizationData +from collections import OrderedDict + + +class NormalizationNode(Node, _PortTypesMixin): + + def init(self): + _PortTypesMixin.init(self) + self.INPUT_PORT_NAME = 'df_in' + self.OUTPUT_PORT_NAME = 'df_out' + self.INPUT_NORM_MODEL_NAME = 'norm_data_in' + self.OUTPUT_NORM_MODEL_NAME = 'norm_data_out' + cols_required = {} + self.required = { + self.INPUT_PORT_NAME: cols_required, + self.INPUT_NORM_MODEL_NAME: cols_required + } + + def ports_setup_from_types(self, types): + """ + overwrite the _PortTypesMixin.ports_setup_from_types + method, which is invoked by the _PortTypesMixin.ports_setup + """ + port_type = PortsSpecSchema.port_type + input_ports = { + self.INPUT_PORT_NAME: { + port_type: types + }, + self.INPUT_NORM_MODEL_NAME: { + port_type: NormalizationData + } + } + + output_ports = { + self.OUTPUT_PORT_NAME: { + port_type: types + }, + self.OUTPUT_NORM_MODEL_NAME: { + port_type: NormalizationData + } + } + + input_connections = self.get_connected_inports() + if self.INPUT_PORT_NAME in input_connections: + determined_type = input_connections[self.INPUT_PORT_NAME] + input_ports.update({self.INPUT_PORT_NAME: + {port_type: determined_type}}) + output_ports.update({self.OUTPUT_PORT_NAME: { + port_type: determined_type}}) + # connected + return NodePorts(inports=input_ports, + outports=output_ports) + else: + return NodePorts(inports=input_ports, outports=output_ports) + + def columns_setup(self): + self.required = { + self.INPUT_PORT_NAME: {}, + self.INPUT_NORM_MODEL_NAME: {} + } + if 'columns' in self.conf and self.conf.get('include', True): + cols_required = {} + for col in self.conf['columns']: + cols_required[col] = None + self.required = { + self.INPUT_PORT_NAME: cols_required, + self.INPUT_NORM_MODEL_NAME: cols_required + } + output_cols = { + self.OUTPUT_PORT_NAME: self.required[self.INPUT_PORT_NAME], + self.OUTPUT_NORM_MODEL_NAME: self.required[ + self.INPUT_NORM_MODEL_NAME] + } + input_columns = self.get_input_columns() + if (self.INPUT_NORM_MODEL_NAME in input_columns and + self.INPUT_PORT_NAME in input_columns): + cols_required = input_columns[self.INPUT_NORM_MODEL_NAME] + self.required = { + self.INPUT_PORT_NAME: cols_required, + self.INPUT_NORM_MODEL_NAME: cols_required + } + col_from_inport = input_columns[self.INPUT_PORT_NAME] + output_cols = { + self.OUTPUT_PORT_NAME: col_from_inport, + self.OUTPUT_NORM_MODEL_NAME: cols_required + } + return output_cols + elif (self.INPUT_NORM_MODEL_NAME in input_columns and + self.INPUT_PORT_NAME not in input_columns): + cols_required = input_columns[self.INPUT_NORM_MODEL_NAME] + self.required = { + self.INPUT_PORT_NAME: cols_required, + self.INPUT_NORM_MODEL_NAME: cols_required + } + output_cols = { + self.OUTPUT_PORT_NAME: cols_required, + self.OUTPUT_NORM_MODEL_NAME: cols_required + } + return output_cols + elif (self.INPUT_NORM_MODEL_NAME not in input_columns and + self.INPUT_PORT_NAME in input_columns): + col_from_inport = input_columns[self.INPUT_PORT_NAME] + enums = [col for col in col_from_inport.keys()] + if 'columns' in self.conf: + if self.conf.get('include', True): + included_colums = self.conf['columns'] + else: + included_colums = [col for col in enums + if col not in self.conf['columns']] + cols_required = OrderedDict() + for col in included_colums: + if col in col_from_inport: + cols_required[col] = col_from_inport[col] + else: + cols_required[col] = None + self.required = { + self.INPUT_PORT_NAME: cols_required, + self.INPUT_NORM_MODEL_NAME: cols_required + } + output_cols = { + self.OUTPUT_PORT_NAME: col_from_inport, + self.OUTPUT_NORM_MODEL_NAME: cols_required + } + return output_cols + return output_cols + + def ports_setup(self): + return _PortTypesMixin.ports_setup(self) + + def conf_schema(self): + json = { + "title": "Normalization Node configure", + "type": "object", + "description": "Normalize the columns to have zero mean and std 1", + "properties": { + "columns": { + "type": "array", + "description": """an array of columns that need to + be normalized, or excluded from normalization depending + on the `incldue` flag state""", + "items": { + "type": "string" + } + }, + "include": { + "type": "boolean", + "description": """if set true, the `columns` need to be + normalized. if false, all dataframe columns except the + `columns` need to be normalized""", + "default": True + }, + }, + "required": [], + } + ui = {} + input_columns = self.get_input_columns() + if self.INPUT_PORT_NAME in input_columns: + col_from_inport = input_columns[self.INPUT_PORT_NAME] + enums = [col for col in col_from_inport.keys()] + json['properties']['columns']['items']['enum'] = enums + return ConfSchema(json=json, ui=ui) + + def process(self, inputs): + """ + normalize the data to zero mean, std 1 + + Arguments + ------- + inputs: list + list of input dataframes. + Returns + ------- + dataframe + """ + input_df = inputs[self.INPUT_PORT_NAME] + if self.INPUT_NORM_MODEL_NAME in inputs: + norm_data = inputs[self.INPUT_NORM_MODEL_NAME].data + input_columns = self.get_input_columns() + means = norm_data['mean'] + stds = norm_data['std'] + col_from_inport = input_columns[self.INPUT_NORM_MODEL_NAME] + cols = [i for i in col_from_inport.keys()] + cols.sort() + else: + # need to compute the mean and std + if self.conf.get('include', True): + cols = self.conf['columns'] + else: + cols = input_df.columns.difference( + self.conf['columns']).values.tolist() + cols.sort() + means = input_df[cols].mean() + stds = input_df[cols].std() + norm = (input_df[cols] - means) / stds + col_dict = {i: norm[i] for i in cols} + norm_df = input_df.assign(**col_dict) + output = {} + if self.outport_connected(self.OUTPUT_PORT_NAME): + output.update({self.OUTPUT_PORT_NAME: norm_df}) + if self.outport_connected(self.OUTPUT_NORM_MODEL_NAME): + norm_data = {"mean": means, "std": stds} + payload = NormalizationData(norm_data) + output.update({self.OUTPUT_NORM_MODEL_NAME: payload}) + return output diff --git a/gquant/plugin_nodes/transform/onehotEncoding.py b/gquant/plugin_nodes/transform/onehotEncoding.py new file mode 100644 index 00000000..a406fdb0 --- /dev/null +++ b/gquant/plugin_nodes/transform/onehotEncoding.py @@ -0,0 +1,96 @@ +from gquant.dataframe_flow import Node +from gquant.dataframe_flow.portsSpecSchema import ConfSchema +from gquant.dataframe_flow._port_type_node import _PortTypesMixin + +__all__ = ["OneHotEncodingNode"] + + +class OneHotEncodingNode(Node, _PortTypesMixin): + + def init(self): + _PortTypesMixin.init(self) + self.INPUT_PORT_NAME = 'in' + self.OUTPUT_PORT_NAME = 'out' + cols_required = {} + self.required = { + self.INPUT_PORT_NAME: cols_required + } + self.delayed_process = True + + def ports_setup(self): + return _PortTypesMixin.ports_setup(self) + + def conf_schema(self): + json = { + "title": "One Hot Encoding configure", + "type": "array", + "description": """Encode the categorical variable by One-hot encoding + """, + "items": { + "type": "object", + "properties": { + "column": { + "type": "string", + "description": """the source column with binary + encoding for the data.""" + }, + "prefix": { + "type": "string", + "description": "the new column name prefix." + }, + "cats": { + "type": "array", + 'items': { + "type": "integer" + }, + "description": "an arrya of categories" + }, + "prefix_sep": { + "type": "string", + "description": """the separator between the prefix + and the category.""", + "default": "_" + }, + "dtype": { + "type": "string", + "description": "the dtype for the outputs", + "enum": ["float64", "float32", "int64", "int32"], + "default": "float64" + } + } + } + } + ui = { + } + input_columns = self.get_input_columns() + if self.INPUT_PORT_NAME in input_columns: + col_from_inport = input_columns[self.INPUT_PORT_NAME] + enums = [col for col in col_from_inport.keys()] + json['items']['properties']['column']['enum'] = enums + return ConfSchema(json=json, ui=ui) + else: + return ConfSchema(json=json, ui=ui) + + def process(self, inputs): + """ + encode the categorical variables to one hot + Arguments + ------- + inputs: list + list of input dataframes. + Returns + ------- + dataframe + """ + input_df = inputs[self.INPUT_PORT_NAME] + for col in self.conf: + input_df = input_df.one_hot_encoding(**col) + return {self.OUTPUT_PORT_NAME: input_df} + + def columns_setup(self): + addition = {} + for col in self.conf: + for cat in col['cats']: + name = col.get('prefix')+col.get('prefix_sep', '_')+str(cat) + addition.update({name: col.get('dtype', 'float64')}) + return _PortTypesMixin.addition_columns_setup(self, addition) diff --git a/gquant/plugin_nodes/util/__init__.py b/gquant/plugin_nodes/util/__init__.py index 01ff37ad..547a69c9 100644 --- a/gquant/plugin_nodes/util/__init__.py +++ b/gquant/plugin_nodes/util/__init__.py @@ -1,3 +1,4 @@ from .compositeNode import CompositeNode +from .contextCompositeNode import ContextCompositeNode -__all__ = ["CompositeNode"] +__all__ = ["CompositeNode", "ContextCompositeNode"] diff --git a/gquant/plugin_nodes/util/compositeNode.py b/gquant/plugin_nodes/util/compositeNode.py index 0f115fdc..2fb9ff86 100644 --- a/gquant/plugin_nodes/util/compositeNode.py +++ b/gquant/plugin_nodes/util/compositeNode.py @@ -12,6 +12,9 @@ import uuid +__all__ = ["CompositeNode"] + + def _get_node(port_name): return port_name.split('@')[0] @@ -48,6 +51,13 @@ def group_ports(input_list): class CompositeNode(Node): + def conf_update(self): + """ + This method is used to overwrite the conf from + external sources + """ + pass + def _compute_hash_key(self): """ if hash changed, the port_setup, columns_setup @@ -55,17 +65,22 @@ def _compute_hash_key(self): In very rara case, might have the problem of hash collision, It affects the column, port and conf calculation. It won't change the computation result though. + It returns the hash code, the loaded task_graph, the replacement conf obj """ + self.conf_update() task_graph = "" inputs = () replacementObj = {} input_node = "" - self.update_replace(replacementObj) + task_graph_obj = None if 'taskgraph' in self.conf: task_graph = get_file_path(self.conf['taskgraph']) if os.path.exists(task_graph): with open(task_graph) as f: task_graph = hashlib.md5(f.read().encode()).hexdigest() + task_graph_obj = TaskGraph.load_taskgraph( + get_file_path(self.conf['taskgraph'])) + self.update_replace(replacementObj, task_graph_obj) if 'input' in self.conf: for inp in self.conf['input']: input_node += inp+"," @@ -73,8 +88,9 @@ def _compute_hash_key(self): for i in self.inputs: inputs += (hash(i['from_node']), i['to_port'], i['from_port']) - return hash((self.uid, task_graph, inputs, json.dumps(self.conf), - input_node, json.dumps(replacementObj))) + return (hash((self.uid, task_graph, inputs, json.dumps(self.conf), + input_node, json.dumps(replacementObj))), task_graph_obj, + replacementObj) def _make_sub_graph_connection(self, task_graph, inputNode_fun, @@ -134,17 +150,13 @@ def _make_sub_graph_connection(self, task_graph, outNode_fun(outNode, oup_groups[oup]) def ports_setup(self): - cache_key = self._compute_hash_key() + cache_key, task_graph, replacementObj = self._compute_hash_key() if cache_key in cache_ports: # print('cache hit') return cache_ports[cache_key] inports = {} outports = {} if 'taskgraph' in self.conf: - task_graph = TaskGraph.load_taskgraph( - get_file_path(self.conf['taskgraph'])) - replacementObj = {} - self.update_replace(replacementObj) task_graph.build(replace=replacementObj) def inputNode_fun(inputNode, in_ports): @@ -170,17 +182,13 @@ def outNode_fun(outNode, out_ports): return output_port def columns_setup(self): - cache_key = self._compute_hash_key() + cache_key, task_graph, replacementObj = self._compute_hash_key() if cache_key in cache_columns: # print('cache hit') return cache_columns[cache_key] required = {} out_columns = {} if 'taskgraph' in self.conf: - task_graph = TaskGraph.load_taskgraph( - get_file_path(self.conf['taskgraph'])) - replacementObj = {} - self.update_replace(replacementObj) task_graph.build(replace=replacementObj) def inputNode_fun(inputNode, in_ports): @@ -209,7 +217,7 @@ def outNode_fun(outNode, out_ports): return out_columns def conf_schema(self): - cache_key = self._compute_hash_key() + cache_key, task_graph, replacementObj = self._compute_hash_key() if cache_key in cache_schema: # print('cache hit') return cache_schema[cache_key] @@ -258,11 +266,7 @@ def conf_schema(self): "subnodes_conf": {} } if 'taskgraph' in self.conf: - task_graphh = TaskGraph.load_taskgraph( - get_file_path(self.conf['taskgraph'])) - replacementObj = {} - self.update_replace(replacementObj) - task_graphh.build(replace=replacementObj) + task_graph.build(replace=replacementObj) def inputNode_fun(inputNode, in_ports): pass @@ -270,16 +274,16 @@ def inputNode_fun(inputNode, in_ports): def outNode_fun(outNode, out_ports): pass - self._make_sub_graph_connection(task_graphh, + self._make_sub_graph_connection(task_graph, inputNode_fun, outNode_fun) ids_in_graph = [] in_ports = [] out_ports = [] - for t in task_graphh: + for t in task_graph: node_id = t.get('id') if node_id != '': - node = task_graphh[node_id] + node = task_graph[node_id] all_ports = node.ports_setup() for port in all_ports.inports.keys(): in_ports.append(node_id+'.'+port) @@ -291,8 +295,8 @@ def outNode_fun(outNode, out_ports): json['properties']['subnode_ids']['items']['enum'] = ids_in_graph if 'subnode_ids' in self.conf: for subnodeId in self.conf['subnode_ids']: - if subnodeId in task_graphh: - nodeObj = task_graphh[subnodeId] + if subnodeId in task_graph: + nodeObj = task_graph[subnodeId] schema = nodeObj.conf_schema() json['properties'][ "subnodes_conf"]['properties'][subnodeId] = { @@ -310,7 +314,7 @@ def outNode_fun(outNode, out_ports): cache_schema[cache_key] = out_schema return out_schema - def update_replace(self, replaceObj): + def update_replace(self, replaceObj, task_graph=None): # find the other replacment conf if 'subnodes_conf' in self.conf: for key in self.conf['subnodes_conf'].keys(): @@ -404,7 +408,7 @@ def outNode_fun(outNode, out_ports): inputNode_fun, outNode_fun) task_graph.extend(input_feeders) - self.update_replace(replaceObj) + self.update_replace(replaceObj, task_graph) result = task_graph.run(outputLists, replace=replaceObj) output = {} for key in result.get_keys(): diff --git a/gquant/plugin_nodes/util/contextCompositeNode.py b/gquant/plugin_nodes/util/contextCompositeNode.py new file mode 100644 index 00000000..6cbcf98d --- /dev/null +++ b/gquant/plugin_nodes/util/contextCompositeNode.py @@ -0,0 +1,242 @@ +from .compositeNode import CompositeNode +from gquant.dataframe_flow.cache import cache_schema +from gquant.dataframe_flow.portsSpecSchema import (ConfSchema, + PortsSpecSchema, NodePorts) +from .data_obj import ConfData +from .json_util import parse_config +from jsonpath_ng import parse + +__all__ = ["ContextCompositeNode"] + + +default_map = { + "boolean": False, + "number": 0.0, + "string": "a string", + "array": [] +} + +class ContextCompositeNode(CompositeNode): + + def init(self): + super().init() + self.INPUT_CONFIG = 'conf_in' + self.OUTPUT_CONFIG = 'conf_out' + + def ports_setup(self): + ports = super().ports_setup() + port_type = PortsSpecSchema.port_type + inports = ports.inports + outports = ports.outports + inports[self.INPUT_CONFIG] = { + port_type: ConfData + } + outports[self.OUTPUT_CONFIG] = { + port_type: ConfData + } + output_port = NodePorts(inports=inports, outports=outports) + return output_port + + def columns_setup(self): + out_columns = super().columns_setup() + out_columns[self.OUTPUT_CONFIG] = self.conf + return out_columns + + def conf_schema(self): + # import pdb + # pdb.set_trace() + cache_key, task_graph, replacementObj = self._compute_hash_key() + if cache_key in cache_schema: + return cache_schema[cache_key] + json = { + "title": "Context Composite Node configure", + "type": "object", + "description": """Use a sub taskgraph as a composite node""", + "properties": { + "taskgraph": { + "type": "string", + "description": "the taskgraph filepath" + }, + "input": { + "type": "array", + "description": "the input node ids", + "items": { + "type": "string" + } + }, + "output": { + "type": "array", + "description": "the output node ids", + "items": { + "type": "string" + } + }, + }, + "required": ["taskgraph"] + } + ui = { + "taskgraph": {"ui:widget": "TaskgraphSelector"}, + } + + types = [] + if 'taskgraph' in self.conf: + json['properties']['context'] = { + "type": "object", + "description": "context parameters", + "additionalProperties": { + "type": "object", + "description": """The context parameters for this + composite node""", + "properties": { + "type": { + "type": "string", + }, + }, + "dependencies": { + "type": { + "oneOf": [] + } + } + } + } + # import pdb + # pdb.set_trace() + all_fields = parse_config(replacementObj) + types = list(all_fields.keys()) + addional = json['properties']['context']['additionalProperties'] + addional['properties']['type']['enum'] = types + typelist = addional['dependencies']['type']['oneOf'] + + for ty in types: + ty_splits = ty.split('_') + obj_temp = { + "properties": { + "type": { + "type": "string", + "description": "the parameter data type" + }, + "value": { + "type": ty_splits[0], + "default": default_map[ty_splits[0]], + "description": "the value for this context parameter" + }, + "map": { + "type": "array", + "description": """The fields of subnode's config this + parameter maps to""", + "items": { + "type": "object", + "properties": { + "node_id": { + "type": "string", + "enum": [] + } + }, + "dependencies": { + "node_id": { + "oneOf": [], + } + } + } + } + } + } + if len(ty_splits) > 1: + obj_temp['properties']['value']['items'] = { + "type": ty_splits[1] + } + type_container = all_fields[ty] + ids = list(type_container.keys()) + obj_temp['properties']['type']['enum'] = [ty] + obj_temp['properties']['map'][ + 'items']['properties']['node_id']['enum'] = ids + idlist = obj_temp['properties']['map'][ + 'items']['dependencies']['node_id']['oneOf'] + for subid in ids: + id_obj = { + "properties": { + "node_id": { + "type": "string" + }, + "xpath": { + "type": "string", + } + } + } + content = type_container[subid] + paths = [i['path'] for i in content] + names = [i['item'] for i in content] + id_obj['properties']['node_id']['enum'] = [subid] + id_obj['properties']['xpath']['enum'] = paths + id_obj['properties']['xpath']['enumNames'] = names + idlist.append(id_obj) + typelist.append(obj_temp) + + if 'taskgraph' in self.conf: + task_graph.build(replace=replacementObj) + + def inputNode_fun(inputNode, in_ports): + pass + + def outNode_fun(outNode, out_ports): + pass + + self._make_sub_graph_connection(task_graph, + inputNode_fun, outNode_fun) + + ids_in_graph = [] + in_ports = [] + out_ports = [] + for t in task_graph: + node_id = t.get('id') + if node_id != '': + node = task_graph[node_id] + all_ports = node.ports_setup() + for port in all_ports.inports.keys(): + in_ports.append(node_id+'.'+port) + for port in all_ports.outports.keys(): + out_ports.append(node_id+'.'+port) + ids_in_graph.append(node_id) + json['properties']['input']['items']['enum'] = in_ports + json['properties']['output']['items']['enum'] = out_ports + out_schema = ConfSchema(json=json, ui=ui) + cache_schema[cache_key] = out_schema + return out_schema + + def conf_update(self): + input_columns = self.get_input_columns() + if self.INPUT_CONFIG in input_columns: + if input_columns[self.INPUT_CONFIG]: + self.conf.update(input_columns[self.INPUT_CONFIG]) + + def update_replace(self, replaceObj, task_graph): + # find the other replacment conf + if task_graph: + for task in task_graph: + key = task.get('id') + newid = key + conf = task.get('conf') + if newid in replaceObj: + replaceObj[newid].update({'conf': conf}) + else: + replaceObj[newid] = {} + replaceObj[newid].update({'conf': conf}) + # replace the numbers from the context + if 'context' in self.conf: + for key in self.conf['context'].keys(): + val = self.conf['context'][key]['value'] + for map_obj in self.conf['context'][key]['map']: + xpath = map_obj['xpath'] + expr = parse(xpath) + expr.update(replaceObj, val) + + def process(self, inputs): + if self.INPUT_CONFIG in inputs: + self.conf.update(inputs[self.INPUT_CONFIG].data) + output = {} + if self.outport_connected(self.OUTPUT_CONFIG): + conf = ConfData(self.conf) + output[self.OUTPUT_CONFIG] = conf + more_output = super().process(inputs) + output.update(more_output) + return output diff --git a/gquant/plugin_nodes/util/data_obj.py b/gquant/plugin_nodes/util/data_obj.py new file mode 100644 index 00000000..c51e21f0 --- /dev/null +++ b/gquant/plugin_nodes/util/data_obj.py @@ -0,0 +1,5 @@ +class ConfData(object): + + def __init__(self, data): + self.data = data + diff --git a/gquant/plugin_nodes/util/json_util.py b/gquant/plugin_nodes/util/json_util.py new file mode 100644 index 00000000..a17be0d2 --- /dev/null +++ b/gquant/plugin_nodes/util/json_util.py @@ -0,0 +1,49 @@ +from jsonpath_ng import parse + + +# map from python obj name to schema name +type_map = { + "dict": 'object', + 'list': 'array', + 'float': 'number', + 'str': 'string', + 'int': 'number', + 'bool': 'boolean' +} + + +def parse_config(json_obj): + expr = parse('$..*') # search for all the fields in the json file + matches = expr.find(json_obj) + map_result = {} + for match in matches: + v = match.value + if type(v) == dict: + continue + element_type = '' + if type(v) == list: + if len(v) > 0: + ele = v[0] + if type(ele) == dict or type(ele) == list: + continue + else: + element_type = type_map[ele.__class__.__name__] + if v is None: + continue + result_type = type_map[v.__class__.__name__] + result_value = v + result_element_type = element_type + result_path = str(match.full_path) + node_id = result_path.split('.')[0] + if result_element_type: + type_key = result_type + '_' + result_element_type + else: + type_key = result_type + type_container = map_result.get(type_key, {}) + map_result[type_key] = type_container + content_container = type_container.get(node_id, []) + type_container[node_id] = content_container + item_str = '.'.join(result_path.split('.')[2:])+" val: "+str(v) + content_container.append({'value': result_value, "path": result_path, + "item": item_str}) + return map_result diff --git a/gquantlab/gquantlab/gquantmodel.py b/gquantlab/gquantlab/gquantmodel.py index 85e505a2..8243c1ab 100644 --- a/gquantlab/gquantlab/gquantmodel.py +++ b/gquantlab/gquantlab/gquantmodel.py @@ -36,6 +36,7 @@ def _handle_event(self, _, content, buffers): if content.get('event', '') == 'run': self.run() elif content.get('event', '') == 'clean': + self.task_graph.run_cleanup(clean_module=True) self.sub = ipywidgets.HBox() def set_taskgraph(self, task_graph): diff --git a/gquantlab/gquantlab/server_utils.py b/gquantlab/gquantlab/server_utils.py index db5a8017..e3fdfe45 100644 --- a/gquantlab/gquantlab/server_utils.py +++ b/gquantlab/gquantlab/server_utils.py @@ -7,6 +7,7 @@ import gquant.plugin_nodes as plugin_nodes import inspect import uuid +from pathlib import Path # import sys # sys.path.append('modules') # noqa E262 @@ -92,10 +93,11 @@ def get_nodes(task_graph): num = max(int(port[2:]), num) inputs.append({'name': 'in'+str(num+1), "type": ["any"]}) out_node['inputs'] = inputs + task_graph.run_cleanup() return {'nodes': nodes, 'edges': edges} -def get_node_obj(node): +def get_node_obj(node, count_id=True): """ It is a private function to convert a Node instance into a dictionary for client to consume. @@ -117,7 +119,10 @@ def get_node_obj(node): width = 160 typeName = OUTPUT_TYPE else: - width = max(max(len(node.uid), len(typeName)) * 10, 100) + if count_id: + width = max(max(len(node.uid), len(typeName)) * 10, 100) + else: + width = max(len(typeName) * 10, 100) conf = node._task_obj.get('conf') out_node = {'width': width, 'id': node.uid, @@ -180,6 +185,7 @@ def add_nodes(): dict dictionary of all the nodes that can be added in the client """ + loaded_node_classes = [] all_modules = get_gquant_config_modules() print(all_modules) all_nodes = {} @@ -187,37 +193,66 @@ def add_nodes(): for item in inspect.getmembers(plugin_nodes): if inspect.ismodule(item[1]): print(item) - all_nodes[item[0]] = [] + labmod_pkg = 'gquant.{}'.format(item[0]) + all_nodes[labmod_pkg] = [] for node in inspect.getmembers(item[1]): - if inspect.isclass(node[1]): - task = {'id': 'random', - 'type': node[0], - 'conf': {}, - 'inputs': []} - t = Task(task) - n = node[1](t) - if issubclass(node[1], Node): - nodeObj = get_node_obj(n) - all_nodes[item[0]].append(nodeObj) + nodecls = node[1] + if not inspect.isclass(nodecls): + continue + if not issubclass(nodecls, Node): + continue + if nodecls in loaded_node_classes: + continue + + task = {'id': 'node_'+str(uuid.uuid4()), + 'type': node[0], + 'conf': {}, + 'inputs': []} + t = Task(task) + n = nodecls(t) + nodeObj = get_node_obj(n, False) + all_nodes[labmod_pkg].append(nodeObj) + loaded_node_classes.append(nodecls) + for module in all_modules: + module_file_or_path = Path(all_modules[module]) loaded = load_modules(all_modules[module], module) mod = loaded.mod modulename = module - all_nodes[modulename] = [] + # all_nodes[modulename] = [] for node in inspect.getmembers(mod): - if node[1] == Node: + nodecls = node[1] + if not inspect.isclass(nodecls): + continue + if nodecls == Node: + continue + + if not issubclass(nodecls, Node): continue - if inspect.isclass(node[1]): - if issubclass(node[1], Node): - task = {'id': 'node_'+str(uuid.uuid4())[:8], - 'type': node[0], - 'conf': {}, - 'inputs': [], - 'module': module - } - t = Task(task) - n = node[1](t) - nodeObj = get_node_obj(n) - all_nodes[modulename].append(nodeObj) + + if nodecls in loaded_node_classes: + continue + + task = {'id': 'node_'+str(uuid.uuid4()), + 'type': node[0], + 'conf': {}, + 'inputs': [], + 'module': module + } + t = Task(task) + n = nodecls(t) + nodeObj = get_node_obj(n, False) + if module_file_or_path.is_dir(): + # submod = nodecls.__module__.split('.')[1:] + # flatten out the namespace hierarchy + submod = nodecls.__module__.split('.')[1:2] + modulename_ = '.'.join([modulename, '.'.join(submod)]) \ + if submod else modulename + all_nodes.setdefault(modulename_, []).append(nodeObj) + else: + all_nodes.setdefault(modulename, []).append(nodeObj) + + loaded_node_classes.append(nodecls) + return all_nodes diff --git a/gquantlab/src/chart.tsx b/gquantlab/src/chart.tsx index 3fc3ebc3..5aaf9533 100644 --- a/gquantlab/src/chart.tsx +++ b/gquantlab/src/chart.tsx @@ -514,6 +514,7 @@ export class Chart extends React.Component { outputs: any; ui: any; schema: any; + conf: any; }; } = {}; data.nodes.forEach((d: INode) => { @@ -523,7 +524,8 @@ export class Chart extends React.Component { inputs: d.inputs, outputs: d.outputs, schema: d.schema, - ui: d.ui + ui: d.ui, + conf: d.conf }; }); nodes.forEach((d: INode) => { @@ -538,6 +540,7 @@ export class Chart extends React.Component { d.outputs = newNode[d.id].outputs; d.schema = newNode[d.id].schema; d.ui = newNode[d.id].ui; + d.conf = newNode[d.id].conf; } }); this.props.setChartState({ diff --git a/gquantlab/src/chartEngine.tsx b/gquantlab/src/chartEngine.tsx index 414120b5..49e10e02 100644 --- a/gquantlab/src/chartEngine.tsx +++ b/gquantlab/src/chartEngine.tsx @@ -429,6 +429,36 @@ export class ChartEngine extends React.Component { this.setState({ nodes: layoutNodes, edges: edges }); } + private _fixNeMoPorts(state: IState): void { + // fix the nemo train node + const NeMoTrainNode = 'NemoTrainNode'; + const NeMoInferNode = 'NemoInferNode'; + const inputTensorPort = 'input_tensor'; + + const nemoNodes = state.nodes.filter( + (d: INode) => d.type === NeMoTrainNode || d.type === NeMoInferNode + ); + + const nodeIds = nemoNodes.map(d => d.id); + + if (nemoNodes.length === 0) { + return; + } + // fixed the port and connections + state.edges.forEach((d: IEdge) => { + if (nodeIds.includes(d.to.split('.')[0])) { + if (d.to.split('.')[1] === inputTensorPort) { + d.to = + d.to.split('.')[0] + + '.' + + d.from.split('.')[0] + + '@' + + d.from.split('.')[1]; + } + } + }); + } + private _fixOutputCollectorPorts(state: IState): void { const index = state.nodes.findIndex( (d: INode) => d.id === OUTPUT_COLLECTOR @@ -460,7 +490,10 @@ export class ChartEngine extends React.Component { updateWorkFlow(state: IState, update = true): void { if (state.edges && state.nodes) { + //TODO need Need to think about how the logic to handle dynamic ports can be refactored in a generic manner in the future. + // Maybe a node could have a self state flag to indicate that it creates ports dynamically. this._fixOutputCollectorPorts(state); + this._fixNeMoPorts(state); const output = exportWorkFlowNodes(state.nodes, state.edges); if (this.props.contentHandler.privateCopy) { this.props.contentHandler.privateCopy.set('value', output); diff --git a/gquantlab/src/commands.ts b/gquantlab/src/commands.ts index a07ee8b8..77600c5b 100644 --- a/gquantlab/src/commands.ts +++ b/gquantlab/src/commands.ts @@ -11,6 +11,7 @@ import { cleanIcon, runIcon } from '.'; +import { listIcon } from '@jupyterlab/ui-components'; import { IFileBrowserFactory, FileDialog } from '@jupyterlab/filebrowser'; import { folderIcon, @@ -50,6 +51,7 @@ export const COMMAND_TOOL_BAR_CLEAN = 'gquant:toolbarcleanResult'; export const COMMAND_ADD_OUTPUT_COLLECTOR = 'gquant:addOutputCollector'; export const COMMAND_OPEN_DOC_FORWARD = 'gquant:openDocumentForward'; export const COMMAND_CREATE_CUST_NODE = 'gquant:createCustomizedNode'; +export const COMMAND_TOOL_BAR_SHOW_LOG = 'gquant:showLogBox'; function uuidv4(): string { return Math.random() @@ -696,6 +698,20 @@ export function setupToolBarCommands( } }); + commands.addCommand(COMMAND_TOOL_BAR_SHOW_LOG, { + label: '', + caption: 'Show Log Console', + icon: listIcon, + mnemonic: 0, + execute: async () => { + app.commands.execute('logconsole:open', { + insertMode: 'split-bottom', + // eslint-disable-next-line prettier/prettier + ref: app.shell.currentWidget?.id + }); + } + }); + commands.addCommand(COMMAND_TOOL_BAR_CLEAN, { label: '', caption: 'Clean Result', diff --git a/gquantlab/src/eventHandler.ts b/gquantlab/src/eventHandler.ts index 1e571e6a..b582bbad 100644 --- a/gquantlab/src/eventHandler.ts +++ b/gquantlab/src/eventHandler.ts @@ -1,5 +1,6 @@ import * as d3 from 'd3'; import { Chart } from './chart'; +import { htmlForType } from './showType'; /** * @this string */ @@ -76,34 +77,7 @@ export function handleMouseOver(that: Chart): Function { } typeHTML += ''; header += `
Port Type:${typeHTML}
`; - // only show column if it is dataframe - let dataframe = false; - for (const oneType of d[key].portType) { - if (oneType.indexOf('DataFrame') >= 0) { - dataframe = true; - break; - } - } - if (dataframe) { - const columnObj = d[key].content; - const columnKeys = Object.keys(columnObj); - if (columnKeys.length > 0) { - header += ''; - header += ''; - header += ''; - for (let i = 0; i < columnKeys.length; i++) { - header += ``; - } - header += ''; - header += ''; - header += ''; - for (let i = 0; i < columnKeys.length; i++) { - header += ``; - } - header += ''; - header += '
Column Name${columnKeys[i]}
Type${columnObj[columnKeys[i]]}
'; - } - } + header += htmlForType(d, key); return header; } diff --git a/gquantlab/src/index.ts b/gquantlab/src/index.ts index edcb4b7d..527347db 100644 --- a/gquantlab/src/index.ts +++ b/gquantlab/src/index.ts @@ -431,19 +431,48 @@ function activateFun( const addNodeMenu = new Menu({ commands }); addNodeMenu.title.label = 'Add Nodes'; addNodeMenu.title.mnemonic = 4; + const subMenuDict: { [key: string]: Menu } = {}; const allNodes = requestAPI('all_nodes'); allNodes.then((allNodes: IAllNodes) => { for (const k in allNodes) { - const submenu = new Menu({ commands }); - submenu.title.label = k; - submenu.title.mnemonic = 0; + const splits = k.split('.'); + let subMenu: Menu = null; + for (let i = 0; i < splits.length; i++) { + const key = splits.slice(0, i + 1).join('.'); + if (key in subMenuDict) { + subMenu = subMenuDict[key]; + } else { + subMenu = new Menu({ commands }); + subMenu.title.label = splits[i]; + subMenu.title.mnemonic = 0; + subMenuDict[key] = subMenu; + if (i > 0) { + // add this submenu to parent + const parentKey = splits.slice(0, i).join('.'); + const pMenu = subMenuDict[parentKey]; + pMenu.addItem({ + type: 'submenu', + submenu: subMenu + }); + } else { + addNodeMenu.addItem({ + type: 'submenu', + submenu: subMenu + }); + } + } + } + + //const submenu = new Menu({ commands }); + //submenu.title.label = k; + //submenu.title.mnemonic = 0; for (let i = 0; i < allNodes[k].length; i++) { const name = allNodes[k][i].type; const args = { name: name, node: (allNodes[k][i] as unknown) as ReadonlyJSONObject }; - submenu.addItem({ + subMenu.addItem({ command: COMMAND_ADD_NODE, args: args }); @@ -455,10 +484,10 @@ function activateFun( }); } } - addNodeMenu.addItem({ - type: 'submenu', - submenu: submenu - }); + // addNodeMenu.addItem({ + // type: 'submenu', + // submenu: submenu + // }); } }); diff --git a/gquantlab/src/mainComponent.tsx b/gquantlab/src/mainComponent.tsx index 173c8e6c..1cd0357b 100644 --- a/gquantlab/src/mainComponent.tsx +++ b/gquantlab/src/mainComponent.tsx @@ -3,10 +3,11 @@ import { ReactWidget } from '@jupyterlab/apputils'; import React from 'react'; //import { ContentHandler, IChartInput } from './document'; import { ContentHandler } from './document'; -import { Widget } from '@lumino/widgets'; +import { Panel, Widget } from '@lumino/widgets'; // eslint-disable-next-line @typescript-eslint/no-unused-vars import { ChartEngine } from './chartEngine'; import { Message } from '@lumino/messaging'; +import { Cell } from '@jupyterlab/cells'; const OUTPUT_CELL_HEIGHT = 300; @@ -21,7 +22,9 @@ export class MainView extends ReactWidget { private _calclateSize(): void { this._height = OUTPUT_CELL_HEIGHT; - this._width = this.parent.parent.parent.node.clientWidth; + const cell: Cell = this.parent.parent.parent.parent.parent.parent as Cell; + this._width = cell.inputArea.editorWidget.node.clientWidth - 5; + //this._width = this.parent.parent.parent.node.clientWidth; // console.log('nsize', this._height, this._width); if (!this._contentHandler) { return; @@ -55,6 +58,13 @@ export class MainView extends ReactWidget { width: this._width, height: this._height }); + // update the result size + const pane = this.parent as Panel; + if (pane.widgets.length === 3) { + const view = pane.widgets[2]; + view.node.style.width = this._width + 'px'; + } + // const content: IChartInput = this._contentHandler.privateCopy.get('cache'); // content['width'] = this._width; // content['height'] = this._height; diff --git a/gquantlab/src/showType.ts b/gquantlab/src/showType.ts new file mode 100644 index 00000000..e30287d2 --- /dev/null +++ b/gquantlab/src/showType.ts @@ -0,0 +1,73 @@ +//TODO need to refactor this to handle dynamically loaded template for display +export function htmlForType(d: any, key: string): string { + let header = ''; + // only show column if it is dataframe + const types = d[key].portType as string[]; + const dfType = types.findIndex(d => { + return d.indexOf('DataFrame') >= 0; + }); + + if (dfType >= 0) { + const columnObj = d[key].content; + const columnKeys = Object.keys(columnObj); + if (columnKeys.length > 0) { + header += ''; + header += ''; + header += ''; + for (let i = 0; i < columnKeys.length; i++) { + header += ``; + } + header += ''; + header += ''; + header += ''; + for (let i = 0; i < columnKeys.length; i++) { + header += ``; + } + header += ''; + header += '
Column Name${columnKeys[i]}
Type${columnObj[columnKeys[i]]}
'; + } + } + + const nmType = types.findIndex(d => { + return d.indexOf('NmTensor') >= 0; + }); + + if (nmType >= 0) { + const columnObj = d[key].content; + const axes = columnObj['axes']; + const element = columnObj['element']; + header += ''; + header += ''; + header += ''; + if ('axes' in columnObj && axes.length > 0) { + for (let i = 0; i < axes.length; i++) { + if (axes[i]['size']) { + header += ``; + } else { + header += ``; + } + } + } else { + header += ''; + } + header += ''; + header += '
Axes: ${i === 0 ? '(' : ''}${axes[i]['kind']}(${ + axes[i]['size'] + })${i === axes.length - 1 ? ')' : ''}${i === 0 ? '(' : ''}${axes[i]['kind']}${ + i === axes.length - 1 ? ')' : '' + }()
'; + header += '
    '; + if ('types' in element) { + header += `
  • Element Type: ${element['types'][0]}
    • `; + if ('fileds' in element && element['fields'] !== 'None') { + header += `
  • Element fileds: ${element['fields']}
    • `; + } + if ('parameters' in element && element['parameters'] !== '{}') { + header += `
  • Element parameters: ${element['parameters']}
    • `; + } + } + header += '
'; + } + + return header; +} diff --git a/gquantlab/src/validator.ts b/gquantlab/src/validator.ts index c63064c8..e8853679 100644 --- a/gquantlab/src/validator.ts +++ b/gquantlab/src/validator.ts @@ -1,6 +1,7 @@ import { IEdge } from './document'; import { Chart } from './chart'; +//TODO, need to refactor this to handle dynamcially added module type checks function valid(required: any, outputs: any): boolean { const keys = Object.keys(required); for (let i = 0; i < keys.length; i++) { @@ -19,6 +20,56 @@ function valid(required: any, outputs: any): boolean { return true; } +function validNmType(required: any, outputs: any): boolean { + //first check types + + const reqElement = required['element']; + const outElement = outputs['element']; + if ( + outElement['types'][0] !== 'VoidType' && + reqElement['types'][0] !== 'VoidType' + ) { + if ( + outElement['types'].findIndex( + (d: string) => d === reqElement['types'][0] + ) < 0 + ) { + // req type should be more specific, + // out type should be generic + // first out type should be the parent type of required + return false; + } + if (outElement['fields'] !== reqElement['fields']) { + return false; + } + if (outElement['parameters'] !== reqElement['parameters']) { + return false; + } + } + + const reqAxes = required['axes']; + const outAxes = outputs['axes']; + if (reqAxes.length === 0) { + return true; + } + if (reqAxes.length !== outAxes.length) { + return false; + } + for (let i = 0; i < reqAxes.length; i++) { + if (reqAxes[i]['kind'] !== outAxes[i]['kind']) { + return false; + } + if ( + reqAxes[i]['size'] !== null && + outAxes[i]['size'] !== null && + reqAxes[i]['size'] !== outAxes['size'] + ) { + return false; + } + } + return true; +} + export function validConnection(that: Chart) { return function(from: string, to: string): any { if (from === to) { @@ -59,7 +110,17 @@ export function validConnection(that: Chart) { // make sure the requirement is met if (from in that.outputColumns && to in that.inputRequriements) { - return valid(that.inputRequriements[to], that.outputColumns[from]); + const nmType = toTypes.findIndex(d => { + return d.indexOf('NmTensor') >= 0; + }); + if (nmType >= 0) { + return validNmType( + that.inputRequriements[to], + that.outputColumns[from] + ); + } else { + return valid(that.inputRequriements[to], that.outputColumns[from]); + } } else if ( !(from in that.outputColumns) && to in that.inputRequriements @@ -92,7 +153,17 @@ export function validConnection(that: Chart) { // make sure the requirement is met if (to in that.outputColumns && from in that.inputRequriements) { - return valid(that.inputRequriements[from], that.outputColumns[to]); + const nmType = toTypes.findIndex(d => { + return d.indexOf('NmTensor') >= 0; + }); + if (nmType >= 0) { + return validNmType( + that.inputRequriements[from], + that.outputColumns[to] + ); + } else { + return valid(that.inputRequriements[from], that.outputColumns[to]); + } } else if ( !(to in that.outputColumns) && from in that.inputRequriements diff --git a/gquantlab/src/widget.ts b/gquantlab/src/widget.ts index 1e5245a0..a651ce33 100644 --- a/gquantlab/src/widget.ts +++ b/gquantlab/src/widget.ts @@ -8,6 +8,7 @@ import { MainView } from './mainComponent'; import { Panel, Widget } from '@lumino/widgets'; import { CommandRegistry } from '@lumino/commands'; import { IFileBrowserFactory } from '@jupyterlab/filebrowser'; +// import { Message } from '@lumino/messaging'; import { Toolbar, CommandToolbarButton } from '@jupyterlab/apputils'; import { @@ -17,9 +18,11 @@ import { COMMAND_TOOL_BAR_OPEN_NEW_FILE, COMMAND_TOOL_BAR_CONVERT_CELL_TO_FILE, COMMAND_TOOL_BAR_INCLUDE_NEW_FILE, + COMMAND_TOOL_BAR_SHOW_LOG, setupToolBarCommands } from './commands'; import { JupyterFrontEnd } from '@jupyterlab/application'; +import { Cell } from '@jupyterlab/cells'; export class GQuantModel extends DOMWidgetModel { static serializers = { @@ -82,6 +85,10 @@ export class GQuantView extends DOMWidgetView { commands: commands, id: COMMAND_TOOL_BAR_INCLUDE_NEW_FILE }); + const item6 = new CommandToolbarButton({ + commands: commands, + id: COMMAND_TOOL_BAR_SHOW_LOG + }); toolBar.addItem('relayout', item0); toolBar.addItem('run', item1); @@ -89,6 +96,7 @@ export class GQuantView extends DOMWidgetView { toolBar.addItem('open', item3); toolBar.addItem('add', item5); toolBar.addItem('create', item4); + toolBar.addItem('log', item6); } render(): void { @@ -145,8 +153,19 @@ export class GQuantView extends DOMWidgetView { pane.layout.removeWidget(pane.widgets[2]); } pane.insertWidget(2, view.pWidget); + // const message: Message = new Message('resize'); + //this._widget.processMessage(message); + }); + this.views.update([value]).then(() => { + const pane = this.pWidget as Panel; + if (pane.widgets.length === 3) { + const w = pane.widgets[2]; + const cell: Cell = this._widget.parent.parent.parent.parent.parent + .parent as Cell; + const _width = cell.inputArea.editorWidget.node.clientWidth - 5; + w.node.style.width = _width + 'px'; + } }); - this.views.update([value]); } protected getFigureSize(): DOMRect { diff --git a/gquantrc b/gquantrc index 7ec1abbd..b21b8037 100644 --- a/gquantrc +++ b/gquantrc @@ -1,3 +1,7 @@ [ModuleFiles] + custom_port_nodes= %(MODULEPATH)s/custom_port_nodes.py -custom_port_nodes2= %(MODULEPATH)s/custom_port_nodes.py + +nemo_modules= %(MODULEPATH)s/nemo_gquant_modules + +my_node= %(MODULEPATH)s/my_node.py diff --git a/modules/my_node.py b/modules/my_node.py new file mode 100644 index 00000000..f3a4c19c --- /dev/null +++ b/modules/my_node.py @@ -0,0 +1,55 @@ +import gquant +from gquant.dataframe_flow.portsSpecSchema import ConfSchema +import json + +data = """{ + "conf": { + "input": [ + "train_norm.df_in", + "test_norm.df_in" + ], + "output": [ + "train_infer.out", + "test_infer.out", + "train_xgboost.model_out", + "train_norm.df_out", + "test_norm.df_out" + ], + "subnode_ids": [ + "train_norm", + "train_xgboost" + ], + "subnodes_conf": {}, + "taskgraph": "taskgraphs/xgboost_example/xgboost_model.gq.yaml" + } +} +""" + + +class CustXGBoostNode(gquant.plugin_nodes.util.CompositeNode): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # modify the self.conf to the one that this Composite node wants + global data + node_conf = self.conf + data_obj = json.loads(data) + data_obj['conf']['subnodes_conf'].update(node_conf) + self.conf = data_obj['conf'] + + def conf_schema(self): + full_schema = super().conf_schema() + full_schema_json = full_schema.json + ui = full_schema.ui + json = { + "title": "CustXGBoostNode configure", + "type": "object", + "description": "Enter your node description here", + "properties": { + } + } + item_dict = full_schema_json['properties']["subnodes_conf"]['properties'] + for key in item_dict.keys(): + json['properties'][key] = item_dict[key] + return ConfSchema(json=json, ui=ui) + + \ No newline at end of file diff --git a/modules/nemo_gquant_modules/__init__.py b/modules/nemo_gquant_modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/nemo_gquant_modules/asr.py b/modules/nemo_gquant_modules/asr.py new file mode 100644 index 00000000..934d8ace --- /dev/null +++ b/modules/nemo_gquant_modules/asr.py @@ -0,0 +1,154 @@ +from gquant.dataframe_flow import Node +from .nemoBaseNode import NeMoBase +import nemo +import nemo.collections.asr + + + +class AudioPreprocessorNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.audio_preprocessing.AudioPreprocessor) + + + +class AudioToMFCCPreprocessorNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.audio_preprocessing.AudioToMFCCPreprocessor) + + + +class AudioToMelSpectrogramPreprocessorNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.audio_preprocessing.AudioToMelSpectrogramPreprocessor) + + + +class AudioToSpectrogramPreprocessorNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.audio_preprocessing.AudioToSpectrogramPreprocessor) + + + +class CropOrPadSpectrogramAugmentationNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.audio_preprocessing.CropOrPadSpectrogramAugmentation) + + + +class MultiplyBatchNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.audio_preprocessing.MultiplyBatch) + + + +class SpectrogramAugmentationNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.audio_preprocessing.SpectrogramAugmentation) + + + +class TimeStretchAugmentationNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.audio_preprocessing.TimeStretchAugmentation) + + + +class BeamSearchDecoderWithLMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.beam_search_decoder.BeamSearchDecoderWithLM) + + + +class ContextNetDecoderForCTCNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.contextnet.ContextNetDecoderForCTC) + + + +class ContextNetEncoderNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.contextnet.ContextNetEncoder) + + + +class JasperEncoderNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.jasper.JasperEncoder) + + + +class AudioToSpeechLabelDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.data_layer.AudioToSpeechLabelDataLayer) + + + +class AudioToTextDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.data_layer.AudioToTextDataLayer) + + + +class KaldiFeatureDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.data_layer.KaldiFeatureDataLayer) + + + +class TarredAudioToTextDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.data_layer.TarredAudioToTextDataLayer) + + + +class TranscriptDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.data_layer.TranscriptDataLayer) + + + +class GreedyCTCDecoderNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.greedy_ctc_decoder.GreedyCTCDecoder) + + + +class JasperDecoderForCTCNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.jasper.JasperDecoderForCTC) + + + +class JasperDecoderForClassificationNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.jasper.JasperDecoderForClassification) + + + +class JasperDecoderForSpkrClassNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.jasper.JasperDecoderForSpkrClass) + + + +class CTCLossNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.losses.CTCLossNM) + + + +class ASRConvCTCModelNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.models.asrconvctcmodel.ASRConvCTCModel) + + + +class JasperNetNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.models.asrconvctcmodel.JasperNet) + + + +class QuartzNetNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.asr.models.asrconvctcmodel.QuartzNet) diff --git a/modules/nemo_gquant_modules/common.py b/modules/nemo_gquant_modules/common.py new file mode 100644 index 00000000..af43bf90 --- /dev/null +++ b/modules/nemo_gquant_modules/common.py @@ -0,0 +1,88 @@ +from gquant.dataframe_flow import Node +from .nemoBaseNode import NeMoBase +import nemo +import nemo.backends.pytorch.common + + + +class BCEWithLogitsLossNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.losses.BCEWithLogitsLossNM) + + + +class CrossEntropyLossNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.losses.CrossEntropyLossNM) + + + +class LossAggregatorNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.losses.LossAggregatorNM) + + + +class MSELossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.losses.MSELoss) + + + +class SequenceLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.losses.SequenceLoss) + + + +class SequenceEmbeddingNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.other.SequenceEmbedding) + + + +class ZerosLikeNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.other.ZerosLikeNM) + + + +class DecoderRNNNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.rnn.DecoderRNN) + + + +class EncoderRNNNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.rnn.EncoderRNN) + + + +class NeMoModelNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.core.nemo_model.NeMoModel) + + + +class NeuralModuleNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.core.neural_modules.NeuralModule) + + + +class BeamSearchNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.search.BeamSearch) + + + +class GreedySearchNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.search.GreedySearch) + + + +class ZerosDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.common.zero_data.ZerosDataLayer) diff --git a/modules/nemo_gquant_modules/cv.py b/modules/nemo_gquant_modules/cv.py new file mode 100644 index 00000000..140c2ce6 --- /dev/null +++ b/modules/nemo_gquant_modules/cv.py @@ -0,0 +1,70 @@ +from gquant.dataframe_flow import Node +from .nemoBaseNode import NeMoBase +import nemo +import nemo.collections.cv + + + +class CIFAR100DataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.data_layers.cifar100_datalayer.CIFAR100DataLayer) + + + +class CIFAR10DataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.data_layers.cifar10_datalayer.CIFAR10DataLayer) + + + +class MNISTDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.data_layers.mnist_datalayer.MNISTDataLayer) + + + +class STL10DataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.data_layers.stl10_datalayer.STL10DataLayer) + + + +class NLLLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.losses.nll_loss.NLLLoss) + + + +class NonLinearityNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.non_trainables.non_linearity.NonLinearity) + + + +class ReshapeTensorNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.non_trainables.reshape_tensor.ReshapeTensor) + + + +class ConvNetEncoderNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.trainables.convnet_encoder.ConvNetEncoder) + + + +class FeedForwardNetworkNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.trainables.feed_forward_network.FeedForwardNetwork) + + + +class ImageEncoderNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.trainables.image_encoder.ImageEncoder) + + + +class LeNet5Node(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.cv.modules.trainables.lenet5.LeNet5) diff --git a/modules/nemo_gquant_modules/nemoBaseNode.py b/modules/nemo_gquant_modules/nemoBaseNode.py new file mode 100644 index 00000000..aa5473fa --- /dev/null +++ b/modules/nemo_gquant_modules/nemoBaseNode.py @@ -0,0 +1,320 @@ +from gquant.dataframe_flow.portsSpecSchema import ConfSchema, PortsSpecSchema +from gquant.dataframe_flow.portsSpecSchema import NodePorts + +from nemo.core.neural_types import NmTensor +import inspect +from gquant.plugin_nodes.util.json_util import type_map +from collections import OrderedDict +from gquant.dataframe_flow.util import get_file_path +from nemo.backends.pytorch.nm import (DataLayerNM, + TrainableNM, LossNM) +from collections import OrderedDict +import nemo + +__all__ = ["NeMoBase"] + +defaut_type = 'number' + +share_weight = 'share_weight' + + +class FeedProperty(object): + def __init__(self, conf): + self.__dict__.update(conf) + + +def serialize_type(neural_type): + output = {} + axes = [] + if neural_type.axes is None: + pass + else: + for ax in neural_type.axes: + axes.append({'kind': str(ax.kind), + 'size': ax.size}) + output['axes'] = axes + ele = {} + ele_type = neural_type.elements_type + ele['types'] = [cla.__name__ for cla in ele_type.__class__.mro() + if cla.__name__ != 'ABC' and cla.__name__ != 'object'] + ele['fields'] = str(ele_type.fields) + ele['parameters'] = str(ele_type.type_parameters) + output['element'] = ele + return output + + +def get_parameters(class_obj, conf): + init_fun = class_obj.__init__ + sig = inspect.signature(init_fun) + hasEmpty = False + init_para = {} + for key in sig.parameters.keys(): + if key == 'self': + # ignore the self + continue + if key in conf: + init_para[key] = conf[key] + else: + hasEmpty = True + break + if not hasEmpty: + return init_para + else: + return None + + +def get_conf_parameters(class_obj): + init_fun = class_obj.__init__ + sig = inspect.signature(init_fun) + init_para = OrderedDict() + for key in sig.parameters.keys(): + if key == 'self': + # ignore the self + continue + para = sig.parameters[key] + default_val = None + if para.default == inspect._empty: + p_type = defaut_type + elif para.default is None: + p_type = defaut_type + else: + if para.default.__class__.__name__ not in type_map: + print(para.default, type(para.default)) + p_type = defaut_type + else: + p_type = type_map[para.default.__class__.__name__] + default_val = para.default + init_para[para.name] = (p_type, default_val) + return init_para + + +class NeMoBase: + + def init(self, class_obj): + if nemo.core.NeuralModuleFactory.get_default_factory() is None: + nemo.core.NeuralModuleFactory() + self.instanceClass = class_obj + self.instance = None + self.file_fields = [] + conf_para = get_conf_parameters(class_obj) + self.fix_type = {} + self.INPUT_NM = 'in_nm' + self.OUTPUT_NM = 'out_nm' + for key in conf_para.keys(): + if key.find('name') >= 0: + self.fix_type[key] = "string" + if key.find('model') >= 0: + self.fix_type[key] = "string" + if key.find('file') >= 0: + self.file_fields.append(key) + for f in self.file_fields: + self.fix_type[f] = 'string' + if f in self.conf and self.conf[f]: + self.conf[f] = get_file_path(self.conf[f]) + if not issubclass(class_obj, DataLayerNM): + try: + if issubclass(self.instanceClass, TrainableNM): + input_columns = self.get_input_columns() + if self.INPUT_NM in input_columns: + if (share_weight in self.conf and + self.conf[share_weight] == 'Reuse'): + self.conf = input_columns[self.INPUT_NM] + app = nemo.utils.app_state.AppState() + ins = None + for mod in app._module_registry: + if isinstance(mod, self.instanceClass): + ins = mod + break + if ins is None: + ins = class_obj(**self.conf) + if self.instance is None: + self.instance = ins + except Exception as e: + # print(e) + pass + + def _clean_dup(self): + app = nemo.utils.app_state.AppState() + if 'name' in self.conf: + if app._module_registry.has(self.conf['name']): + existing = app._module_registry[self.conf['name']] + app._module_registry.remove(existing) + removeList = [] + for mod in app._module_registry: + if isinstance(mod, self.instanceClass): + # remove the duplicate instances + removeList.append(mod) + for mod in removeList: + app._module_registry.remove(mod) + + def ports_setup(self): + input_columns = self.get_input_columns() + if issubclass(self.instanceClass, TrainableNM): + input_columns = self.get_input_columns() + if self.INPUT_NM in input_columns: + if (share_weight in self.conf and + self.conf[share_weight] == 'Reuse'): + self.conf = input_columns[self.INPUT_NM] + port_type = PortsSpecSchema.port_type + if self.instance is not None: + inports = self.instance.input_ports + outports = self.instance.output_ports + else: + try: + p_inports = self.instanceClass.input_ports + p_outports = self.instanceClass.output_ports + feeder = FeedProperty(self.conf) + inports = p_inports.fget(feeder) + outports = p_outports.fget(feeder) + except Exception: + inports = None + outports = None + o_inports = {} + o_outports = {} + if inports is not None: + for k in inports.keys(): + o_inports[k] = {port_type: NmTensor} + if outports is not None: + for k in outports.keys(): + o_outports[k] = {port_type: NmTensor} + if issubclass(self.instanceClass, TrainableNM): + # added the port for tying the weights + o_inports[self.INPUT_NM] = {port_type: TrainableNM} + o_outports[self.OUTPUT_NM] = {port_type: TrainableNM} + elif issubclass(self.instanceClass, LossNM): + o_outports[self.OUTPUT_NM] = {port_type: LossNM} + elif issubclass(self.instanceClass, DataLayerNM): + o_outports[self.OUTPUT_NM] = {port_type: DataLayerNM} + return NodePorts(inports=o_inports, outports=o_outports) + + def columns_setup(self): + input_columns = self.get_input_columns() + if issubclass(self.instanceClass, TrainableNM): + input_columns = self.get_input_columns() + if self.INPUT_NM in input_columns: + if (share_weight in self.conf and + self.conf[share_weight] == 'Reuse'): + self.conf = input_columns[self.INPUT_NM] + if self.instance is not None: + inports = self.instance.input_ports + outports = self.instance.output_ports + else: + try: + p_inports = self.instanceClass.input_ports + p_outports = self.instanceClass.output_ports + feeder = FeedProperty(self.conf) + inports = p_inports.fget(feeder) + outports = p_outports.fget(feeder) + except Exception: + inports = None + outports = None + self.required = {} + out_cols = {} + if inports is not None: + for k in inports.keys(): + self.required[k] = serialize_type(inports[k]) + if outports is not None: + for k in outports.keys(): + out_cols[k] = serialize_type(outports[k]) + if self.instance is not None: + out_cols[self.OUTPUT_NM] = self.conf + return out_cols + + def conf_schema(self): + conf_para = get_conf_parameters(self.instanceClass) + class_doc = self.instanceClass.__doc__ + desc = "" if class_doc is None else class_doc + init_doc = self.instanceClass.__init__.__doc__ + desc += "" if init_doc is None else init_doc + json = { + "title": "NeMo "+self.instanceClass.__name__+" Node", + "type": "object", + "description": desc, + "properties": { + + }, + } + ui = { + } + for f in self.file_fields: + if f in conf_para: + ui[f] = {"ui:widget": "FileSelector"} + for p in conf_para.keys(): + stype = conf_para[p][0] + if p in self.fix_type: + stype = self.fix_type[p] + json['properties'][p] = { + "type": stype, + "default": conf_para[p][1] + } + if issubclass(self.instanceClass, TrainableNM): + if share_weight in conf_para: + print('warning, share_weight parameter name collision') + json['properties'][share_weight] = { + "type": 'string', + "description": """Weight Sharing between Modules: Reuse, + re-use neural modules between training, evaluation and + inference graphs; Copy: copy weights betwen modules. subsequent + update of weights in one module will not affect weights in the + other module. This means that the weights will get DIFFERENT + gradients on the update step. Tying: the default one, tie + weights between two or more modules. Tied weights are identical + across all modules. Gradients to the weights will be the + SAME.""", + "enum": ['Reuse', 'Copy', 'Tying'], + "default": 'Tying' + } + return ConfSchema(json=json, ui=ui) + + def process(self, inputs): + para = get_parameters(self.instanceClass, self.conf) + app = nemo.utils.app_state.AppState() + self.instance = None + if issubclass(self.instanceClass, TrainableNM): + if self.INPUT_NM in inputs: + inputIn = inputs[self.INPUT_NM] + if (share_weight in self.conf and + self.conf[share_weight] == 'Reuse'): + self.instance = inputIn + if para is not None and self.instance is None: + self._clean_dup() + self.instance = self.instanceClass(**para) + if self.instance is None: + return {} + if issubclass(self.instanceClass, TrainableNM): + if self.INPUT_NM in inputs: + inputIn = inputs[self.INPUT_NM] + if (share_weight in self.conf and + self.conf[share_weight] == 'Reuse'): + pass + elif (share_weight in self.conf and + self.conf[share_weight] == 'Copy'): + self.instance.set_weights(inputIn.get_weights()) + else: + self.instance.tie_weights_with(inputIn, + list( + inputIn.get_weights( + ).keys())) + inputsCopy = OrderedDict() + for k in self.instance.input_ports.keys(): + if k in inputs: + inputsCopy[k] = inputs[k] + instanceName = self.instance.name + if instanceName in app.active_graph._modules: + del app.active_graph._modules[instanceName] + o = self.instance(**inputsCopy) + if isinstance(o, tuple): + output = {} + for key in self.instance.output_ports.keys(): + output[key] = getattr(o, key) + else: + key = list(self.instance.output_ports.keys())[0] + output = {key: o} + # if self.uid == 'eval_data': + # print(inputs, output) + # for k in output.keys(): + # print(output[k].name) + # print(output[k].unique_name) + + output[self.OUTPUT_NM] = self.instance + return output diff --git a/modules/nemo_gquant_modules/nemo_util/__init__.py b/modules/nemo_gquant_modules/nemo_util/__init__.py new file mode 100644 index 00000000..651538d9 --- /dev/null +++ b/modules/nemo_gquant_modules/nemo_util/__init__.py @@ -0,0 +1,8 @@ +# from .toy import TaylorNetNode +# from .toy import MSELossNode +# from .toy import RealFunctionDataNode +from .trainNemo import NemoTrainNode +from .inferNemo import NemoInferNode +from .nemoHPO import NemoHyperTuneNode + +__all__ = ["NemoTrainNode", "NemoInferNode", "NemoHyperTuneNode"] diff --git a/modules/nemo_gquant_modules/nemo_util/inferNemo.py b/modules/nemo_gquant_modules/nemo_util/inferNemo.py new file mode 100644 index 00000000..1dc98f37 --- /dev/null +++ b/modules/nemo_gquant_modules/nemo_util/inferNemo.py @@ -0,0 +1,134 @@ +from gquant.dataframe_flow import Node +from gquant.dataframe_flow.portsSpecSchema import ConfSchema, PortsSpecSchema +from gquant.dataframe_flow.portsSpecSchema import NodePorts +from gquant.dataframe_flow._port_type_node import _PortTypesMixin + +from nemo.core.neural_types import NmTensor +from .trainNemo import NemoTrainNode +import nemo +import copy + +__all__ = ["NemoInferNode"] + + +class NemoInferNode(Node, _PortTypesMixin): + def init(self): + _PortTypesMixin.init(self) + self.OUTPUT_PORT_NAME = 'torch_tensor' + self.INPUT_PORT_NAME = 'log_dir' + + def ports_setup(self): + port_type = PortsSpecSchema.port_type + o_inports = {} + o_inports[self.INPUT_PORT_NAME] = {port_type: str} + o_outports = {} + o_inports['input_tensor'] = {port_type: NmTensor} + if hasattr(self, 'inputs'): + for inp in self.inputs: + if inp['to_port'] == self.INPUT_PORT_NAME: + continue + o_inports[inp['from_node'].uid+'@'+inp['from_port']] = { + port_type: NmTensor} + o_outports = {} + o_outports[self.OUTPUT_PORT_NAME] = {port_type: list} + return NodePorts(inports=o_inports, outports=o_outports) + + def columns_setup(self): + self.required = {} + output = {} + output['axes'] = [] + output['element'] = {} + output['element']['types'] = ['VoidType'] + output['element']['fields'] = 'None' + output['element']['parameters'] = '{}' + ports = self.ports_setup() + inports = ports.inports + if inports is not None: + for k in inports.keys(): + if k == self.INPUT_PORT_NAME: + continue + self.required[k] = output + return {} + + def conf_schema(self): + json = { + "title": "NeMo Infer Node", + "type": "object", + "description": """Node used to run NeMo neural network inference + to obtain values for tensors""", + "properties": { + "tensors": { + "type": "array", + "description": """List of NeMo tensors + that we want to get values of""", + "items": { + "type": "string", + } + }, + "checkpoint_dir": { + "type": ["string", "null"], + "description": """Path to checkpoint directory. + Default is None which does not load checkpoints.""", + "default": None + }, + "ckpt_pattern": { + "type": "string", + "description": """Pattern used to check for checkpoints inside + checkpoint_dir. Default is '' which matches any checkpoints + inside checkpoint_dir.""", + "default": '', + }, + "verbose": { + "type": "boolean", + "description": """Controls printing.""", + "default": True + }, + "cache": { + "type": "boolean", + "description": """If True, cache all `tensors` and intermediate + tensors so that future calls that have use_cache set will + avoid computation.""", + "default": False + }, + "use_cache": { + "type": "boolean", + "description": """Values from `tensors` will be always re-computed. + It will re-use intermediate tensors from the DAG leading to + `tensors`. If you want something to be re-computed, put it into + `tensors` list.""", + "default": False + }, + "offload_to_cpu": { + "type": "boolean", + "description": """If True, all evaluated tensors are moved to + cpu memory after each inference batch.""", + "default": True + } + } + } + ui = { + "checkpoint_dir": {"ui:widget": "PathSelector"}, + } + enum = [] + enumNames = [] + count = 1 + if hasattr(self, 'inputs'): + for i in self.inputs: + if i['to_port'] == self.INPUT_PORT_NAME: + continue + enum.append(i['from_node'].uid+'@'+i['from_port']) + enumNames.append(i['from_node'].uid+'.'+i['from_port']) + count += 1 + json['properties']["tensors"]['items']['enum'] = enum + json['properties']["tensors"]['items']['enumNames'] = enumNames + return ConfSchema(json=json, ui=ui) + + def process(self, inputs): + nf = nemo.core.NeuralModuleFactory.get_default_factory() + input_columns = self.get_input_columns() + conf = copy.copy(self.conf) + if self.INPUT_PORT_NAME in input_columns: + conf['checkpoint_dir'] = inputs[self.INPUT_PORT_NAME] + conf['tensors'] = [inputs[i] for i in conf['tensors']] + result = nf.infer(**conf) + return {self.OUTPUT_PORT_NAME: result} diff --git a/modules/nemo_gquant_modules/nemo_util/nemoHPO.py b/modules/nemo_gquant_modules/nemo_util/nemoHPO.py new file mode 100644 index 00000000..5e79fa8c --- /dev/null +++ b/modules/nemo_gquant_modules/nemo_util/nemoHPO.py @@ -0,0 +1,378 @@ +from gquant.plugin_nodes.ml.gridRandomSearchNode import GridRandomSearchNode +from gquant.plugin_nodes.util.contextCompositeNode import ContextCompositeNode +from gquant.dataframe_flow.portsSpecSchema import (ConfSchema, + NodePorts) +from gquant.dataframe_flow import TaskGraph +from gquant.dataframe_flow import Node +from gquant.dataframe_flow.util import get_file_path +from gquant.dataframe_flow.cache import cache_schema +from gquant.dataframe_flow.taskSpecSchema import TaskSpecSchema +import cudf +import uuid +import pandas +import copy + +__all__ = ["NemoHyperTuneNode"] + + +_SCHED_CONF = { + "type": "object", + "description": """distributed implementations of early + stopping algorithms such as Median Stopping Rule, + HyperBand, and ASHA.""", + "properties": { + "name": { + "type": "string", + "enum": ["AsyncHyperBandScheduler", + "HyperBandScheduler", + "MedianStoppingRule"] + }, + }, + "dependencies": { + "name": { + "oneOf": [ + { + "properties": { + "name": { + "type": "string", + "enum": ["AsyncHyperBandScheduler"] + }, + "parameters": { + "type": "object", + "properties": { + "time_attr": { + "type": "string", + "description": """A training result + attr to use for comparing time. + Note that you can pass in something + non-temporal such as + training_iteration as a measure of + progress, the only requirement is that + the attribute should increase + monotonically.""", + "enum": ["training_iteration"], + "default": "training_iteration" + }, + "max_t": { + "type": "number", + "description": """max time units per + trial. Trials will be stopped after + max_t time units (determined by + time_attr) have passed.""", + "default": 100.0 + }, + "grace_period": { + "type": "number", + "description": """Only stop trials at + least this old in time. The units are + the same as the attribute named by + time_attr""", + "default": 1.0 + }, + "reduction_factor": { + "type": "number", + "description": """Used to set halving + rate and amount. This is simply a + unit-less scalar.""", + "default": 4.0 + }, + "brackets": { + "type": "integer", + "description": """Number of brackets. + Each bracket has a different halving + rate, specified by the reduction + factor.""", + "default": 1 + } + } + } + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["HyperBandScheduler"] + }, + "parameters": { + "type": "object", + "properties": { + "time_attr": { + "type": "string", + "description": """A training result attr to + use for comparing time. Note that you + can pass in something non-temporal such + as training_iteration as a measure of + progress, the only requirement is that + the attribute should increase + monotonically.""", + "enum": ["training_iteration"], + "default": "training_iteration" + }, + "max_t": { + "type": "number", + "description": """max time units per + trial. Trials will be stopped after + max_t time units (determined by + time_attr) have passed.""", + "default": 100.0 + }, + "reduction_factor": { + "type": "number", + "description": """Used to set halving + rate and amount. This is simply a + unit-less scalar.""", + "default": 4.0 + }, + } + } + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["MedianStoppingRule"] + }, + "parameters": { + "type": "object", + "properties": { + "time_attr": { + "type": "string", + "description": """A training result attr to + use for comparing time. Note that you + can pass in something non-temporal such + as training_iteration as a measure of + progress, the only requirement is that + the attribute should increase + monotonically.""", + "enum": ["training_iteration"], + "default": "training_iteration" + }, + "grace_period": { + "type": "number", + "description": """Only stop trials at + least this old in time. The units are + the same as the attribute named by + time_attr""", + "default": 60.0 + }, + "min_samples_required": { + "type": "integer", + "description": """Minimum number of + trials to compute median over.""", + "default": 3 + }, + "min_time_slice": { + "type": "number", + "description": """Each trial runs at + least this long before yielding + (assuming it isn’t stopped). + Note: trials ONLY yield if there + are not enough samples to evaluate + performance for the current result + AND there are other trials waiting to + run. The units are the same as the + attribute named by time_attr.""", + "default": 0.0 + }, + "hard_stop": { + "type": "boolean", + "description": """If False, pauses + trials instead of stopping them. + When all other trials are complete, + paused trials will be resumed and + allowed to run FIFO.""", + "default": True + }, + } + } + } + }, + ] + } + } +} + + +class NemoHyperTuneNode(GridRandomSearchNode): + + def init(self): + GridRandomSearchNode.init(self) + + def ports_setup(self): + return GridRandomSearchNode.ports_setup(self) + + def columns_setup(self): + return GridRandomSearchNode.columns_setup(self) + + def conf_schema(self): + cache_key, task_graph, _ = self._compute_hash_key() + if cache_key in cache_schema: + return cache_schema[cache_key] + tensors = [] + if task_graph is not None: + for task in task_graph: + if task.get('type') == 'NemoTrainNode': + conf = task.get('conf') + if ('eval_callback' in conf and + 'eval_tensors' in conf['eval_callback']): + tensors = conf['eval_callback']['eval_tensors'] + tensors = [t.split('@')[-1] for t in tensors] + print(tensors) + conf = GridRandomSearchNode.conf_schema(self) + json = conf.json + if 'properties' in json: + del json['properties']['metrics'] + json['properties']['best'][ + 'properties']['metric']['enum'] = tensors + json['properties']['scheduler'] = copy.deepcopy(_SCHED_CONF) + return ConfSchema(json=json, ui=conf.ui) + + def process(self, inputs): + _, task_graph, _ = self._compute_hash_key() + train_id = None + if task_graph is not None: + for task in task_graph: + if task.get('type') == 'NemoTrainNode': + conf = task.get('conf') + if ('eval_callback' in conf and + 'eval_tensors' in conf['eval_callback']): + tensors = conf['eval_callback']['eval_tensors'] + tensors = [t.split('@')[-1] for t in tensors] + train_id = task.get('id') + if train_id is None: + print('no train node detected') + return {} + + import ray + from ray import tune + + if self.INPUT_CONFIG in inputs: + self.conf.update(inputs[self.INPUT_CONFIG].data) + output = {} + if self.outport_connected(self.OUTPUT_CONFIG): + data_store = {} + for key in inputs.keys(): + v = inputs[key] + if isinstance(v, cudf.DataFrame): + # it is a work around, + # the ray.put doesn't support GPU cudf + data_store[key] = ray.put(v.to_pandas()) + else: + data_store[key] = ray.put(v) + + # here we need to do the hyper parameter search + def search_fun(config, checkpoint_dir=None): + myinputs = {} + for key in data_store.keys(): + v = ray.get(data_store[key]) + if isinstance(v, pandas.DataFrame): + myinputs[key] = cudf.from_pandas(v) + else: + myinputs[key] = v + task_graph = TaskGraph.load_taskgraph( + get_file_path(self.conf['taskgraph'])) + task_graph.build() + + outputLists = [train_id+'.'+'checkpoint_dir'] + replaceObj = {} + input_feeders = [] + + def inputNode_fun(inputNode, in_ports): + inports = inputNode.ports_setup().inports + + class InputFeed(Node): + + def columns_setup(self): + output = {} + for inp in inputNode.inputs: + output[inp['to_port']] = inp[ + 'from_node'].columns_setup()[ + inp['from_port']] + # it will be something like { input_port: columns } + return output + + def ports_setup(self): + # it will be something like { input_port: types } + return NodePorts(inports={}, outports=inports) + + def conf_schema(self): + return ConfSchema() + + def process(self, empty): + output = {} + for key in inports.keys(): + if (inputNode.uid+'@'+key + in myinputs): + output[key] = myinputs[ + inputNode.uid+'@'+key] + return output + + uni_id = str(uuid.uuid1()) + obj = { + TaskSpecSchema.task_id: uni_id, + TaskSpecSchema.conf: {}, + TaskSpecSchema.node_type: InputFeed, + TaskSpecSchema.inputs: [] + } + input_feeders.append(obj) + newInputs = {} + for key in inports.keys(): + if inputNode.uid+'@'+key in myinputs: + newInputs[key] = uni_id+'.'+key + for inp in inputNode.inputs: + if inp['to_port'] not in in_ports: + # need to keep the old connections + newInputs[inp['to_port']] = ( + inp['from_node'].uid + '.' + inp['from_port']) + replaceObj.update({inputNode.uid: { + TaskSpecSchema.inputs: newInputs} + }) + + def outNode_fun(outNode, out_ports): + pass + self._make_sub_graph_connection(task_graph, + inputNode_fun, outNode_fun) + + task_graph.extend(input_feeders) + self.update_conf_for_search(replaceObj, task_graph, config) + task_graph.run(outputLists, replace=replaceObj) + # metric_report = {item: result[item] for item in outputLists} + # tune.report(**metric_report) + config = {} + for para in self.conf['parameters']: + fun_name = para['search']['function'] + fun = getattr(tune, fun_name) + if fun_name == 'grid_search' or fun_name == 'choice': + config[para['name']] = fun(para['search']['args']) + else: + config[para['name']] = fun(*para['search']['args']) + + scheduler_instance = None + if 'scheduler' in self.conf and 'name' in self.conf['scheduler']: + import ray.tune.schedulers + sconf = self.conf['scheduler'] + name = sconf['name'] + scheduler = getattr(ray.tune.schedulers, name) + para = sconf['parameters'] + para.update(self.conf['best']) + print(para) + scheduler_instance = scheduler(**para) + + if scheduler_instance is None: + analysis = tune.run(search_fun, **self.conf['tune'], + config=config) + else: + analysis = tune.run(search_fun, **self.conf['tune'], + config=config, + scheduler=scheduler_instance) + best = analysis.get_best_config(**self.conf['best']) + print('best parameter', best) + for key in best.keys(): + self.conf['context'][key]['value'] = best[key] + output[self.OUTPUT_CONFIG] = self.conf + more_output = ContextCompositeNode.process(self, inputs) + output.update(more_output) + + return output diff --git a/modules/nemo_gquant_modules/nemo_util/trainNemo.py b/modules/nemo_gquant_modules/nemo_util/trainNemo.py new file mode 100644 index 00000000..073fc8e8 --- /dev/null +++ b/modules/nemo_gquant_modules/nemo_util/trainNemo.py @@ -0,0 +1,1028 @@ +from gquant.dataframe_flow import Node +from gquant.dataframe_flow.portsSpecSchema import ConfSchema, PortsSpecSchema +from gquant.dataframe_flow.portsSpecSchema import NodePorts +from gquant.dataframe_flow._port_type_node import _PortTypesMixin + +from nemo.core.neural_types import NmTensor +import nemo +import numpy +import copy + +__all__ = ["NemoTrainNode"] + + +class CallBack(object): + + def __init__(self): + self.counter = 0 + + def __call__(self, global_vars): + import ray + from ray import tune + reports = {} + for key in global_vars.keys(): + value = numpy.array(global_vars[key]).mean() + print('eval:', key, value) + reports[key] = value + if ray.is_initialized(): + reports['training_iteration'] = self.counter + tune.report(**reports) + self.counter += 1 + + +class NemoTrainNode(Node, _PortTypesMixin): + def init(self): + _PortTypesMixin.init(self) + self.OUTPUT_PORT_NAME = 'checkpoint_dir' + + def ports_setup(self): + port_type = PortsSpecSchema.port_type + o_inports = {} + o_outports = {} + o_inports['input_tensor'] = {port_type: NmTensor} + if hasattr(self, 'inputs'): + for inp in self.inputs: + o_inports[inp['from_node'].uid+'@'+inp['from_port']] = { + port_type: NmTensor} + o_outports[self.OUTPUT_PORT_NAME] = {port_type: str} + return NodePorts(inports=o_inports, outports=o_outports) + + def columns_setup(self): + self.required = {} + output = {} + output['axes'] = [] + output['element'] = {} + output['element']['types'] = ['VoidType'] + output['element']['fields'] = 'None' + output['element']['parameters'] = '{}' + ports = self.ports_setup() + inports = ports.inports + if inports is not None: + for k in inports.keys(): + self.required[k] = output + return {} + + def conf_schema(self): + json = { + "title": "NeMo Train Node", + "type": "object", + "description": "Node used to train a NeMo neural network", + "properties": { + "parameters": { + "type": "object", + "description": "parameters for train method", + "properties": { + "tensors_to_optimize": { + "type": "array", + "items": { + "type": "string", + } + }, + "batches_per_step": { + "type": "number", + "default": None + }, + "stop_on_nan_loss": { + "type": "boolean", + "default": False + }, + "synced_batchnorm": { + "type": "boolean", + "default": False + }, + "synced_batchnorm_groupsize": { + "type": "number", + "default": 0 + }, + "gradient_predivide": { + "type": "boolean", + "default": False + }, + "amp_max_loss_scale": { + "type": "number", + "default": 16777216.0 + }, + "reset": { + "type": "boolean", + "default": False + } + } + }, + "check_point": { + "type": "object", + "description": "parameters for checkpoint method", + "properties": { + "folder": { + "type": "string", + "description": """A path where checkpoints are to + be stored and loaded from if load_from_folder + is None""" + }, + "load_from_folder": { + "type": ["string", "null"], + "description": """A path where checkpoints can be + loaded from""", + "default": None + }, + "step_freq": { + "type": "integer", + "description": """How often in terms of steps to + save checkpoints. One of step_freq or epoch_freq + is required""" + }, + "epoch_freq": { + "type": "integer", + "description": """How often in terms of epochs to + save checkpoints. One of step_freq or epoch_freq + is required.""" + }, + "checkpoints_to_keep": { + "type": "integer", + "description": """Number of most recent + checkpoints to keep. Older checkpoints will be + deleted.""", + "default": 4 + }, + "force_load": { + "type": "boolean", + "description": """Whether to crash if loading + is unsuccessful.""", + "default": False + } + } + }, + "simple_logger": { + "type": "object", + "description": """A simple callback that prints tensors + to screen. It's default option is to print the training + loss every 100 steps. Additional tensors can be printed + by adding them to the tensors_to_log argument.""", + "properties": { + "step_freq": { + "type": "integer", + "description": """The frequency of printing to + screen. Defaults to every 100 steps""", + "default": 100 + }, + "tensors_to_log": { + "type": "array", + "description": """A list of NmTensors which will + be printed every step_freq steps.""", + "items": { + "type": "string", + } + } + } + }, + "eval_callback": { + "type": "object", + "description": """Used to report the statistics of + evaluation dataset""", + "properties": { + "eval_step": { + "type": ["integer", "null"], + "description": """The frequency of running eval""", + "default": None + }, + "eval_tensors": { + "type": "array", + "description": """A list of NmTensors which will + be evaluated every eval_step steps.""", + "items": { + "type": "string", + } + } + } + }, + "warmup_policy": { + "type": "object", + "description": """Choose a warm up policy""", + "properties": { + "name": { + "type": "string", + "enum": ["WarmupPolicy", "WarmupHoldPolicy", + "SquareAnnealing", "SquareRootAnnealing", + "CosineAnnealing", "WarmupAnnealing", + "InverseSquareRootAnnealing", + "PolynomialDecayAnnealing", + "PolynomialHoldDecayAnnealing"] + }, + }, + "dependencies": { + "name": { + "oneOf": [ + { + "properties": { + "name": { + "type": "string", + "enum": ["WarmupPolicy"] + }, + "parameters": { + "type": "object", + "properties": { + "warmup_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps in + warmup stage""", + "default": None + }, + "warmup_ratio": { + "type": ["number", "null"], + "description": """Ratio of + warmup steps to total + steps""", + "default": None + }, + "total_steps": { + "type": ["integer", + "null"], + "description": """Total number + of steps while training or + `None` for infinite + training""", + "default": None + } + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["WarmupHoldPolicy"] + }, + "parameters": { + "type": "object", + "properties": { + "warmup_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps in + warmup stage""", + "default": None + }, + "warmup_ratio": { + "type": ["number", + "null"], + "description": """Ratio of + warmup steps to total + steps""", + "default": None + }, + "total_steps": { + "type": ["integer", + "null"], + "description": """Total number + of steps while training or + `None` for infinite + training""", + "default": None + }, + "hold_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps to hold the + learning rate after warm + up""", + "default": None + }, + "hold_ratio": { + "type": ["integer", + "null"], + "description": """Ratio of hold + steps to total steps""", + "default": None + }, + "min_lr": { + "type": "number", + "description": """minimum learing + rate""", + "default": 0.0 + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["SquareAnnealing"] + }, + "parameters": { + "type": "object", + "properties": { + "warmup_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps in + warmup stage""", + "default": None + }, + "warmup_ratio": { + "type": ["number", "null"], + "description": """Ratio of + warmup steps to total + steps""", + "default": None + }, + "total_steps": { + "type": ["integer", + "null"], + "description": """Total number + of steps while training or + `None` for infinite + training""", + "default": None + }, + "min_lr": { + "type": "number", + "description": """minimum learing + rate""", + "default": 0.00001 + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["SquareRootAnnealing"] + }, + "parameters": { + "type": "object", + "properties": { + "warmup_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps in + warmup stage""", + "default": None + }, + "warmup_ratio": { + "type": ["number", "null"], + "description": """Ratio of + warmup steps to total + steps""", + "default": None + }, + "total_steps": { + "type": ["integer", + "null"], + "description": """Total number + of steps while training or + `None` for infinite + training""", + "default": None + }, + "min_lr": { + "type": "number", + "description": """minimum learing + rate""", + "default": 0.0 + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["CosineAnnealing"] + }, + "parameters": { + "type": "object", + "properties": { + "warmup_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps in + warmup stage""", + "default": None + }, + "warmup_ratio": { + "type": ["number", "null"], + "description": """Ratio of + warmup steps to total + steps""", + "default": None + }, + "total_steps": { + "type": ["integer", + "null"], + "description": """Total number + of steps while training or + `None` for infinite + training""", + "default": None + }, + "min_lr": { + "type": "number", + "description": """minimum learing + rate""", + "default": 0.0 + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["WarmupAnnealing"] + }, + "parameters": { + "type": "object", + "properties": { + "warmup_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps in + warmup stage""", + "default": None + }, + "warmup_ratio": { + "type": ["number", "null"], + "description": """Ratio of + warmup steps to total + steps""", + "default": None + }, + "total_steps": { + "type": ["integer", + "null"], + "description": """Total number + of steps while training or + `None` for infinite + training""", + "default": None + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": [ + "InverseSquareRootAnnealing"] + }, + "parameters": { + "type": "object", + "properties": { + "warmup_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps in + warmup stage""", + "default": None + }, + "warmup_ratio": { + "type": ["number", "null"], + "description": """Ratio of + warmup steps to total + steps""", + "default": None + }, + "total_steps": { + "type": ["integer", + "null"], + "description": """Total number + of steps while training or + `None` for infinite + training""", + "default": None + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": [ + "PolynomialDecayAnnealing"] + }, + "parameters": { + "type": "object", + "properties": { + "warmup_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps in + warmup stage""", + "default": None + }, + "warmup_ratio": { + "type": ["number", "null"], + "description": """Ratio of + warmup steps to total + steps""", + "default": None + }, + "total_steps": { + "type": ["integer", + "null"], + "description": """Total number + of steps while training or + `None` for infinite + training""", + "default": None + }, + "min_lr": { + "type": "number", + "description": """minimum learing + rate""", + "default": 0.0 + }, + "power": { + "type": "number", + "default": 1.0 + }, + "cycle": { + "type": "boolean", + "default": False + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": [ + "PolynomialHoldDecayAnnealing"] + }, + "parameters": { + "type": "object", + "properties": { + "warmup_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps in + warmup stage""", + "default": None + }, + "warmup_ratio": { + "type": ["number", "null"], + "description": """Ratio of + warmup steps to total + steps""", + "default": None + }, + "total_steps": { + "type": ["integer", + "null"], + "description": """Total number + of steps while training or + `None` for infinite + training""", + "default": None + }, + "hold_steps": { + "type": ["integer", + "null"], + "description": """Number of + training steps to hold the + learning rate after warm + up""", + "default": None + }, + "hold_ratio": { + "type": ["integer", + "null"], + "description": """Ratio of hold + steps to total steps""", + "default": None + }, + "min_lr": { + "type": "number", + "description": """minimum learing + rate""", + "default": 0.0 + }, + "power": { + "type": "number", + "default": 1.0 + }, + "cycle": { + "type": "boolean", + "default": False + }, + } + }, + } + } + ] + } + } + }, + "optimizer": { + "type": "object", + "description": """The optimization algorithm""", + "properties": { + "name": { + "type": "string", + "enum": ["sgd", "adam", "fused_adam", "adam_w", + "novograd", "fused_novograd", + "fused_lamb"] + }, + }, + "dependencies": { + "name": { + "oneOf": [ + { + "properties": { + "name": { + "type": "string", + "enum": ["sgd"] + }, + "parameters": { + "type": "object", + "properties": { + "num_epochs": { + "type": "integer", + "default": 10 + }, + "lr": { + "type": "number", + "default": 0.001 + }, + "momentum": { + "type": "number", + "default": 0.9 + }, + "weight_decay": { + "type": "number", + "default": 0.0 + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["adam"] + }, + "parameters": { + "type": "object", + "properties": { + "num_epochs": { + "type": "integer", + "default": 10 + }, + "lr": { + "type": "number", + "default": 0.001 + }, + "betas": { + "type": "array", + "items": [ + { + "type": "number", + "default": 0.9 + }, + { + "type": "number", + "default": 0.999 + } + ] + }, + "eps": { + "type": "number", + "default": 0.000000001 + }, + "weight_decay": { + "type": "number", + "default": 0.0 + }, + "amsgrad": { + "type": "boolean", + "default": False + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["fused_adam"] + }, + "parameters": { + "type": "object", + "properties": { + "num_epochs": { + "type": "integer", + "default": 10 + }, + "lr": { + "type": "number", + "default": 0.001 + }, + "betas": { + "type": "array", + "items": [ + { + "type": "number", + "default": 0.9 + }, + { + "type": "number", + "default": 0.999 + } + ] + }, + "eps": { + "type": "number", + "default": 0.00000001, + }, + "weight_decay": { + "type": "number", + "default": 0.0 + } + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["adam_w"] + }, + "parameters": { + "type": "object", + "properties": { + "num_epochs": { + "type": "integer", + "default": 10 + }, + "lr": { + "type": "number", + "default": 0.001 + }, + "betas": { + "type": "array", + "items": [ + { + "type": "number", + "default": 0.9 + }, + { + "type": "number", + "default": 0.999 + } + ] + }, + "eps": { + "type": "number", + "default": 0.00000001, + }, + "weight_decay": { + "type": "number", + "default": 0.0 + }, + "amsgrad": { + "type": "boolean", + "default": False + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["novograd"] + }, + "parameters": { + "type": "object", + "properties": { + "num_epochs": { + "type": "integer", + "default": 10 + }, + "lr": { + "type": "number", + "default": 0.001 + }, + "betas": { + "type": "array", + "items": [ + { + "type": "number", + "default": 0.9 + }, + { + "type": "number", + "default": 0.999 + } + ] + }, + "luc": { + "type": "boolean", + "default": False + }, + "luc_eta": { + "type": "number", + "default": 0.001 + }, + "weight_decay": { + "type": "number", + "default": 0.0 + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["fused_novograd"] + }, + "parameters": { + "type": "object", + "properties": { + "num_epochs": { + "type": "integer", + "default": 10 + }, + "lr": { + "type": "number", + "default": 0.001 + }, + "betas": { + "type": "array", + "items": [ + { + "type": "number", + "default": 0.9 + }, + { + "type": "number", + "default": 0.999 + } + ] + }, + "reg_inside_moment": { + "type": "boolean", + "default": True + }, + "grad_averaging": { + "type": "boolean", + "default": False + }, + "weight_decay": { + "type": "number", + "default": 0.0 + }, + } + }, + } + }, + { + "properties": { + "name": { + "type": "string", + "enum": ["fused_lamb"] + }, + "parameters": { + "type": "object", + "properties": { + "num_epochs": { + "type": "integer", + "default": 10 + }, + "lr": { + "type": "number", + "default": 0.001 + }, + } + }, + } + } + ] + } + } + } + } + } + ui = { + "check_point": { + "folder": {"ui:widget": "PathSelector"}, + "load_from_folder": {"ui:widget": "PathSelector"}, + }, + "warmup_policy": { + "parameters": { + "warmup_steps": {"ui:widget": "updown"}, + "total_steps": {"ui:widget": "updown"}, + "warmup_ratio": {"ui:widget": "updown"}, + "hold_steps": {"ui:widget": "updown"}, + "hold_ratio": {"ui:widget": "updown"} + }, + }, + "eval_callback": { + "eval_step": {"ui:widget": "updown"}, + } + } + enum = [] + enumNames = [] + count = 1 + if hasattr(self, 'inputs'): + for i in self.inputs: + enum.append(i['from_node'].uid+'@'+i['from_port']) + enumNames.append(i['from_node'].uid+'.'+i['from_port']) + count += 1 + json['properties']['parameters'][ + 'properties']["tensors_to_optimize"][ + 'items']['enum'] = enum + json['properties']['parameters'][ + 'properties']["tensors_to_optimize"][ + 'items']['enumNames'] = enumNames + json['properties']['simple_logger'][ + 'properties']["tensors_to_log"][ + 'items']['enum'] = enum + json['properties']['simple_logger'][ + 'properties']["tensors_to_log"][ + 'items']['enumNames'] = enumNames + json['properties']['eval_callback'][ + 'properties']["eval_tensors"][ + 'items']['enum'] = enum + json['properties']['eval_callback'][ + 'properties']["eval_tensors"][ + 'items']['enumNames'] = enumNames + return ConfSchema(json=json, ui=ui) + + def process(self, inputs): + nf = nemo.core.NeuralModuleFactory.get_default_factory() + log_conf = copy.copy(self.conf["simple_logger"]) + log_conf['tensors_to_log'] = [ + inputs[i] for i in log_conf['tensors_to_log']] + log_callback = nemo.core.SimpleLogger(**log_conf) + check_callback = nemo.core.CheckpointCallback( + **self.conf['check_point']) + all_args = copy.copy(self.conf['parameters']) + all_args['tensors_to_optimize'] = [ + inputs[i] for i in all_args['tensors_to_optimize']] + + # eval callbacks + def eval_iter_callback(tensors, global_vars): + for e_name in eval_names: + if e_name not in global_vars: + global_vars[e_name] = [] + + for e_name in eval_names: + for kv, v in tensors.items(): + if kv.startswith(e_name): + global_vars[e_name].append(v[0].cpu().numpy().mean()) + + all_args['callbacks'] = [check_callback, log_callback] + if ('eval_callback' in self.conf and + 'eval_tensors' in self.conf['eval_callback'] and + len(self.conf['eval_callback']['eval_tensors']) > 0): + eval_conf = copy.copy(self.conf['eval_callback']) + eval_conf['eval_tensors'] = [ + inputs[i] for i in eval_conf['eval_tensors']] + eval_names = [i.name for i in eval_conf['eval_tensors']] + eval_conf['user_iter_callback'] = eval_iter_callback + eval_conf['user_epochs_done_callback'] = CallBack() + eval_callback = nemo.core.EvaluatorCallback(**eval_conf) + all_args['callbacks'].append(eval_callback) + all_args['optimizer'] = self.conf['optimizer']['name'] + all_args['optimization_params'] = self.conf['optimizer']['parameters'] + + if 'warmup_policy' in self.conf and 'name' in self.conf[ + "warmup_policy"]: + policy_name = self.conf["warmup_policy"]['name'] + from nemo.utils import lr_policies + policy_class = getattr(lr_policies, policy_name) + lr_policy = policy_class( + **self.conf["warmup_policy"]['parameters']) + all_args['lr_policy'] = lr_policy + nf.train(**all_args) + return {self.OUTPUT_PORT_NAME: self.conf['check_point']['folder']} diff --git a/modules/nemo_gquant_modules/nlp.py b/modules/nemo_gquant_modules/nlp.py new file mode 100644 index 00000000..57aa9367 --- /dev/null +++ b/modules/nemo_gquant_modules/nlp.py @@ -0,0 +1,154 @@ +from gquant.dataframe_flow import Node +from .nemoBaseNode import NeMoBase +import nemo +import nemo.collections.nlp.nm + + + +class BertInferDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.data_layers.bert_inference_datalayer.BertInferDataLayer) + + + +class BertPretrainingDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.data_layers.lm_bert_datalayer.BertPretrainingDataLayer) + + + +class BertPretrainingPreprocessedDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.data_layers.lm_bert_datalayer.BertPretrainingPreprocessedDataLayer) + + + +class TextDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.data_layers.text_datalayer.TextDataLayer) + + + +class MaskedLogLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.losses.masked_xentropy_loss.MaskedLogLoss) + + + +class SGDDialogueStateLossNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.losses.sgd_loss.SGDDialogueStateLossNM) + + + +class SmoothedCrossEntropyLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.losses.smoothed_cross_entropy_loss.SmoothedCrossEntropyLoss) + + + +class SpanningLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.losses.spanning_loss.SpanningLoss) + + + +class AlbertNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.huggingface.albert_nm.Albert) + + + +class BERTNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.huggingface.bert_nm.BERT) + + + +class BeamSearchTranslatorNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.transformer.transformer_nm.BeamSearchTranslatorNM) + + + +class BertTokenClassifierNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.token_classification_nm.BertTokenClassifier) + + + +class EncoderRNNNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.encoder_rnn.EncoderRNN) + + + +class GreedyLanguageGeneratorNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.transformer.transformer_nm.GreedyLanguageGeneratorNM) + + + +class JointIntentSlotClassifierNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.joint_intent_slot.joint_intent_slot_classifier_nm.JointIntentSlotClassifier) + + + +class PunctCapitTokenClassifierNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.punctuation_capitalization.punctuation_capitalization_classifier_nm.PunctCapitTokenClassifier) + + + +class RobertaNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.huggingface.roberta_nm.Roberta) + + + +class SGDDecoderNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.dialogue_state_tracking.sgd.sgd_decoder_nm.SGDDecoderNM) + + + +class SGDEncoderNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.dialogue_state_tracking.sgd.sgd_encoder_nm.SGDEncoderNM) + + + +class SequenceClassifierNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.sequence_classification_nm.SequenceClassifier) + + + +class SequenceRegressionNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.sequence_regression_nm.SequenceRegression) + + + +class TRADEGeneratorNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.dialogue_state_tracking.trade_generator_nm.TRADEGenerator) + + + +class TokenClassifierNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.token_classification_nm.TokenClassifier) + + + +class TransformerDecoderNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.transformer.transformer_nm.TransformerDecoderNM) + + + +class TransformerEncoderNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.nlp.nm.trainables.common.transformer.transformer_nm.TransformerEncoderNM) diff --git a/modules/nemo_gquant_modules/simple_gan.py b/modules/nemo_gquant_modules/simple_gan.py new file mode 100644 index 00000000..42373f86 --- /dev/null +++ b/modules/nemo_gquant_modules/simple_gan.py @@ -0,0 +1,46 @@ +from gquant.dataframe_flow import Node +from .nemoBaseNode import NeMoBase +import nemo +import nemo.collections.simple_gan + + + +class DiscriminatorLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.simple_gan.gan.DiscriminatorLoss) + + + +class GradientPenaltyNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.simple_gan.gan.GradientPenalty) + + + +class InterpolateImageNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.simple_gan.gan.InterpolateImage) + + + +class MnistGanDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.simple_gan.gan.MnistGanDataLayer) + + + +class RandomDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.simple_gan.gan.RandomDataLayer) + + + +class SimpleDiscriminatorNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.simple_gan.gan.SimpleDiscriminator) + + + +class SimpleGeneratorNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.simple_gan.gan.SimpleGenerator) diff --git a/modules/nemo_gquant_modules/tts.py b/modules/nemo_gquant_modules/tts.py new file mode 100644 index 00000000..3f7e2689 --- /dev/null +++ b/modules/nemo_gquant_modules/tts.py @@ -0,0 +1,118 @@ +from gquant.dataframe_flow import Node +from .nemoBaseNode import NeMoBase +import nemo +import nemo.collections.tts + + + +class AudioDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.data_layers.AudioDataLayer) + + + +class FastSpeechNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.fastspeech_modules.FastSpeech) + + + +class FastSpeechDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.fastspeech_modules.FastSpeechDataLayer) + + + +class FastSpeechLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.fastspeech_modules.FastSpeechLoss) + + + +class MakeGateNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.tacotron2_modules.MakeGate) + + + +class Tacotron2DecoderNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.tacotron2_modules.Tacotron2Decoder) + + + +class Tacotron2DecoderInferNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.tacotron2_modules.Tacotron2DecoderInfer) + + + +class Tacotron2EncoderNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.tacotron2_modules.Tacotron2Encoder) + + + +class Tacotron2LossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.tacotron2_modules.Tacotron2Loss) + + + +class Tacotron2PostnetNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.tacotron2_modules.Tacotron2Postnet) + + + +class TextEmbeddingNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.tacotron2_modules.TextEmbedding) + + + +class LenSamplerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.talknet_modules.LenSampler) + + + +class TalkNetNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.talknet_modules.TalkNet) + + + +class TalkNetDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.talknet_modules.TalkNetDataLayer) + + + +class TalkNetDursLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.talknet_modules.TalkNetDursLoss) + + + +class TalkNetMelsLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.talknet_modules.TalkNetMelsLoss) + + + +class WaveGlowInferNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.waveglow_modules.WaveGlowInferNM) + + + +class WaveGlowLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.waveglow_modules.WaveGlowLoss) + + + +class WaveGlowNMNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.collections.tts.waveglow_modules.WaveGlowNM) diff --git a/modules/nemo_gquant_modules/tutorials.py b/modules/nemo_gquant_modules/tutorials.py new file mode 100644 index 00000000..bc76198d --- /dev/null +++ b/modules/nemo_gquant_modules/tutorials.py @@ -0,0 +1,64 @@ +from gquant.dataframe_flow import Node +from .nemoBaseNode import NeMoBase +import nemo +import nemo.backends.pytorch.tutorials + + + +class DialogDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.tutorials.chatbot.modules.DialogDataLayer) + + + +class EncoderRNNNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.tutorials.chatbot.modules.EncoderRNN) + + + +class GreedyLuongAttnDecoderRNNNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.tutorials.chatbot.modules.GreedyLuongAttnDecoderRNN) + + + +class LuongAttnDecoderRNNNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.tutorials.chatbot.modules.LuongAttnDecoderRNN) + + + +class MaskedXEntropyLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.tutorials.chatbot.modules.MaskedXEntropyLoss) + + + +class CrossEntropyLossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.tutorials.toys.CrossEntropyLoss) + + + +class L1LossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.tutorials.toys.L1Loss) + + + +class MSELossNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.tutorials.toys.MSELoss) + + + +class RealFunctionDataLayerNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.tutorials.toys.RealFunctionDataLayer) + + + +class TaylorNetNode(NeMoBase, Node): + def init(self): + NeMoBase.init(self, nemo.backends.pytorch.tutorials.toys.TaylorNet) diff --git a/notebooks/03_simple_dask_example.ipynb b/notebooks/03_simple_dask_example.ipynb index 736c9796..88d1cf2e 100644 --- a/notebooks/03_simple_dask_example.ipynb +++ b/notebooks/03_simple_dask_example.ipynb @@ -150,6 +150,24 @@ "convert the sorted stock data into partitions and save it into csv files. Note, the data is slited in a way that the same asset belongs to the same partition" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "task_graph = TaskGraph.load_taskgraph('../taskgraphs/sort_stocks.gq.yaml')\n", + "input_cached, = task_graph.run()\n", + "task_graph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "convert the sorted stock data into partitions and save it into csv files. Note, the data is slited in a way that the same asset belongs to the same partition" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/05_customize_nodes_with_ports.ipynb b/notebooks/05_customize_nodes_with_ports.ipynb index 4f93963f..d62ec755 100644 --- a/notebooks/05_customize_nodes_with_ports.ipynb +++ b/notebooks/05_customize_nodes_with_ports.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -80,7 +80,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -164,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -254,7 +254,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -289,7 +289,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -306,7 +306,7 @@ "" ] }, - "execution_count": 26, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -318,7 +318,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -335,7 +335,7 @@ "" ] }, - "execution_count": 27, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -354,18 +354,25 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 11, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "039cd525ecd14a7f95c56cb369e61a75", + "model_id": "9c57b14189514a5b9d86a7016ac7bb32", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "Tab(children=(Output(), Output(), Output(), Output(layout=Layout(border='1px solid black'))), _titles={'0': 'p…" + "Tab(children=(Output(), Output(), Output(), Output(layout=Layout(border='1px solid black'), outputs=({'output_…" ] }, "metadata": {}, @@ -382,7 +389,9 @@ " 'distance_by_cudf.distance_df',\n", " 'distance_by_cudf.distance_abs_df'\n", "]\n", - "task_graph.run(outputs=outlist, formated=True)\n" + "r = task_graph.run(outputs=outlist)\n", + "points_df = r['points_task.points_df_out']\n", + "task_graph.run(outputs=outlist, formated=True)" ] }, { @@ -417,7 +426,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -527,7 +536,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -633,7 +642,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -643,7 +652,7 @@ "" ] }, - "execution_count": 32, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -695,18 +704,25 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 27, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "61c83bfe39ae4291a82392f0ac0f152e", + "model_id": "e7facdb676b04ee98f2ca978c2994cf2", "version_major": 2, "version_minor": 0 }, "text/plain": [ - "Tab(children=(Output(), Output(), Output(), Output(layout=Layout(border='1px solid black'))), _titles={'0': 'd…" + "Tab(children=(Output(), Output(), Output(), Output(layout=Layout(border='1px solid black'), outputs=({'output_…" ] }, "metadata": {}, @@ -719,6 +735,7 @@ " 'distance_by_numba.distance_df',\n", " 'distance_by_cupy.distance_df'\n", "]\n", + "(df_w_cudf, df_w_numba, df_w_cupy) = task_graph.run(out_list)\n", "task_graph.run(out_list, formated=True)" ] }, @@ -731,7 +748,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -759,7 +776,7 @@ }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -842,7 +859,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -852,7 +869,7 @@ "" ] }, - "execution_count": 46, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -890,7 +907,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 31, "metadata": {}, "outputs": [ { @@ -924,7 +941,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -935,7 +952,7 @@ "\n", "

Client

\n", "\n", "\n", @@ -951,10 +968,10 @@ "" ], "text/plain": [ - "" + "" ] }, - "execution_count": 61, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -984,7 +1001,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -1065,7 +1082,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -1140,7 +1157,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 35, "metadata": {}, "outputs": [ { @@ -1149,35 +1166,35 @@ "text": [ "HEAD points_ddf:\n", " x y\n", - "0 0.970135 0.878125\n", - "1 0.753458 0.189487\n", - "2 0.459017 0.508227\n", - "3 0.081538 0.938519\n", - "4 0.583010 0.605625\n", + "0 0.657520 0.846858\n", + "1 0.811397 0.260745\n", + "2 0.783395 0.541172\n", + "3 0.247166 0.779094\n", + "4 0.597549 0.083825\n", "\n", "HEAD df_w_cudf:\n", " x y distance_cudf\n", - "0 0.970135 0.878125 1.308535\n", - "1 0.753458 0.189487 0.776920\n", - "2 0.459017 0.508227 0.684829\n", - "3 0.081538 0.938519 0.942054\n", - "4 0.583010 0.605625 0.840644\n", + "0 0.657520 0.846858 1.072148\n", + "1 0.811397 0.260745 0.852263\n", + "2 0.783395 0.541172 0.952142\n", + "3 0.247166 0.779094 0.817361\n", + "4 0.597549 0.083825 0.603400\n", "\n", "HEAD df_w_numba:\n", " x y distance_numba\n", - "0 0.970135 0.878125 1.308535\n", - "1 0.753458 0.189487 0.776920\n", - "2 0.459017 0.508227 0.684829\n", - "3 0.081538 0.938519 0.942054\n", - "4 0.583010 0.605625 0.840644\n", + "0 0.657520 0.846858 1.072148\n", + "1 0.811397 0.260745 0.852263\n", + "2 0.783395 0.541172 0.952142\n", + "3 0.247166 0.779094 0.817361\n", + "4 0.597549 0.083825 0.603400\n", "\n", "HEAD df_w_cupy:\n", " x y distance_cupy\n", - "0 0.970135 0.878125 1.308535\n", - "1 0.753458 0.189487 0.776920\n", - "2 0.459017 0.508227 0.684829\n", - "3 0.081538 0.938519 0.942054\n", - "4 0.583010 0.605625 0.840644\n", + "0 0.657520 0.846858 1.072148\n", + "1 0.811397 0.260745 0.852263\n", + "2 0.783395 0.541172 0.952142\n", + "3 0.247166 0.779094 0.817361\n", + "4 0.597549 0.083825 0.603400\n", "\n", "Max Difference cudf to numba: 2.220446049250313e-16\n", "Max Difference cudf to cupy: 2.220446049250313e-16\n" @@ -1249,7 +1266,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -1259,7 +1276,7 @@ "" ] }, - "execution_count": 63, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } @@ -1359,7 +1376,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 37, "metadata": {}, "outputs": [ { @@ -1368,27 +1385,27 @@ "text": [ "HEAD df_w_cudf:\n", " x y distance_cudf\n", - "0 0.088949 0.821747 0.826547\n", - "1 0.857481 0.391407 0.942588\n", - "2 0.359685 0.148664 0.389197\n", - "3 0.142579 0.648509 0.663997\n", - "4 0.798294 0.130983 0.808968\n", + "0 0.130721 0.775092 0.786038\n", + "1 0.447255 0.293845 0.535147\n", + "2 0.424665 0.156408 0.452553\n", + "3 0.139219 0.048282 0.147353\n", + "4 0.753861 0.506722 0.908336\n", "\n", "HEAD df_w_numba:\n", " x y distance_numba\n", - "0 0.088949 0.821747 0.826547\n", - "1 0.857481 0.391407 0.942588\n", - "2 0.359685 0.148664 0.389197\n", - "3 0.142579 0.648509 0.663997\n", - "4 0.798294 0.130983 0.808968\n", + "0 0.130721 0.775092 0.786038\n", + "1 0.447255 0.293845 0.535147\n", + "2 0.424665 0.156408 0.452553\n", + "3 0.139219 0.048282 0.147353\n", + "4 0.753861 0.506722 0.908336\n", "\n", "HEAD df_w_cupy:\n", " x y distance_cupy\n", - "0 0.088949 0.821747 0.826547\n", - "1 0.857481 0.391407 0.942588\n", - "2 0.359685 0.148664 0.389197\n", - "3 0.142579 0.648509 0.663997\n", - "4 0.798294 0.130983 0.808968\n", + "0 0.130721 0.775092 0.786038\n", + "1 0.447255 0.293845 0.535147\n", + "2 0.424665 0.156408 0.452553\n", + "3 0.139219 0.048282 0.147353\n", + "4 0.753861 0.506722 0.908336\n", "\n", "Max Difference cudf to numba: 2.220446049250313e-16\n", "Max Difference cudf to cupy: 2.220446049250313e-16\n" @@ -1423,7 +1440,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -1441,13 +1458,13 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 39, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "89c732dfa8c1468785aec6a6b523dff9", + "model_id": "5a8d8093475e4a1f93cead8f866fdb5a", "version_major": 2, "version_minor": 0 }, @@ -1473,13 +1490,20 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 40, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a88ef0372ed9443faf93de720fab0273", + "model_id": "48bd5c65dbf947029c99b6af13a3a64f", "version_major": 2, "version_minor": 0 }, @@ -1515,7 +1539,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -1524,27 +1548,27 @@ "text": [ "HEAD df_w_cudf:\n", " x y distance_cudf\n", - "0 0.696019 0.197711 0.723555\n", - "1 0.615746 0.279821 0.676345\n", - "2 0.757476 0.044872 0.758804\n", - "3 0.584972 0.599957 0.837938\n", - "4 0.664956 0.060440 0.667698\n", + "0 0.130721 0.775092 0.786038\n", + "1 0.447255 0.293845 0.535147\n", + "2 0.424665 0.156408 0.452553\n", + "3 0.139219 0.048282 0.147353\n", + "4 0.753861 0.506722 0.908336\n", "\n", "HEAD df_w_numba:\n", " x y distance_numba\n", - "0 0.696019 0.197711 0.723555\n", - "1 0.615746 0.279821 0.676345\n", - "2 0.757476 0.044872 0.758804\n", - "3 0.584972 0.599957 0.837938\n", - "4 0.664956 0.060440 0.667698\n", + "0 0.130721 0.775092 0.786038\n", + "1 0.447255 0.293845 0.535147\n", + "2 0.424665 0.156408 0.452553\n", + "3 0.139219 0.048282 0.147353\n", + "4 0.753861 0.506722 0.908336\n", "\n", "HEAD df_w_cupy:\n", " x y distance_cupy\n", - "0 0.696019 0.197711 0.723555\n", - "1 0.615746 0.279821 0.676345\n", - "2 0.757476 0.044872 0.758804\n", - "3 0.584972 0.599957 0.837938\n", - "4 0.664956 0.060440 0.667698\n", + "0 0.130721 0.775092 0.786038\n", + "1 0.447255 0.293845 0.535147\n", + "2 0.424665 0.156408 0.452553\n", + "3 0.139219 0.048282 0.147353\n", + "4 0.753861 0.506722 0.908336\n", "\n" ] } @@ -1582,7 +1606,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ diff --git a/notebooks/08_gquant_machine_learning.ipynb b/notebooks/08_gquant_machine_learning.ipynb new file mode 100644 index 00000000..6bcdaaa4 --- /dev/null +++ b/notebooks/08_gquant_machine_learning.ipynb @@ -0,0 +1,471 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# GQuant tutorial: build a XGBoost model to predict the next day stock return in 30 minutes.\n", + "\n", + "\n", + "XGBoost is a decision-tree-based ensemble Machine Learning algorithm that uses a gradient boosting framework. Since its introduction, this algorithm has not only been credited with winning numerous Kaggle competitions but also for being the driving force under the hood for several cutting-edge industry applications. XGBoost natively supports the GPU acceleration, which speeds up the training and inference by orders of magnitude. \n", + "\n", + "gQuant is a graph computation tool that builds on top of RAPIDS which includes the XGBoost algorithm. gQuant project has a Jupyterlab extension that can guide the user to build data science workflows in the browser. In this tutorial, we will learn step by step how to use gQuant user interface and build a simple XGBoost model from scratch to predict positive or negative next day stock return. This tutorial is organized as follows:\n", + "\n", + " 1. Prepare the fake dataset with categorical variables\n", + " 2. Preprocess the dataset to be ready for XGBoost algorithm\n", + " 3. Train a XGBoost model and run inference\n", + " 4. Visualize the machine learning result\n", + " 5. Accelerate the XGBoost inference by tree inference library\n", + " 6. Change the dataset to stock dataset and predict the positive/negative next day stock return. \n", + " \n", + "In each step, there are animated gif files that show detailed steps. To effectively use the tutorial, we recommend follow the steps in the animation and try to reproduce the results. \n", + "\n", + "## Prepare the environment\n", + "\n", + "Let's import the necessary gQuant library. " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys; sys.path.insert(0, '..')\n", + "from gquant.dataframe_flow import TaskGraph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prepare for running in Dask environment\n", + "\n", + "Let's start the Dask local cluster environment for distributed computation.\n", + "\n", + "Dask provides a web-based dashboard to help to track progress, identify performance issues, and debug failures. To learn more about Dask dashboard, just follow this [link](https://distributed.dask.org/en/latest/web.html).\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "

Client

\n", + "\n", + "		\n", + "

Cluster

\n", + "
    \n", + "
  • Workers: 2
    • \n", + "
  • Cores: 2
    • \n", + "
  • Memory: 100.00 GB
    • \n", + "
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dask_cuda import LocalCUDACluster\n", + "cluster = LocalCUDACluster()\n", + "from dask.distributed import Client\n", + "client = Client(cluster)\n", + "client" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prepare the dataset\n", + "\n", + "### Add a data generator node\n", + "In this step, we add gQuant TaskGraph nodes, make connections and evaluate the graph\n", + "\n", + "\n", + "### Explore and visualize the data\n", + "In this step, we change the data generator configuration and visualize the result.\n", + "\n", + "\n", + "### Add categorical variable\n", + "To simulate categorical variables, we convert two of the continuous variables into categorical variables and encode them with one-hot encoding.\n", + "\n", + "\n", + "After this step, you should have a TaskGraph looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "890665de9a0a43e385c3830a2952a087", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'data_gen'), ('type', 'ClassificationData'), ('conf', {'n_…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "task_graph = TaskGraph.load_taskgraph('../taskgraphs/xgboost_example/data_generator.gq.yaml')\n", + "task_graph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Save the graph and create a composite node\n", + "We encapsulate the dataset create steps into a single composite node\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preprocess the data\n", + "\n", + "### Split the dataset into train and test\n", + "We split the dataset randomly into train and test so we can test the performance of the learned XGBoost model later.\n", + "\n", + "\n", + "### Normalize the features\n", + "Though it is not needed for XGBoost model, normalizing the features can be useful for other machine learning models. Think of this step as a placeholder for some preprocessing steps that are needed to clean up the dataset.\n", + "\n", + "\n", + "After this step, you should have a TaskGraph looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "3b2d1cd0c4894db0ad0e0afe5779c55b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', ''), ('type', 'Output_Collector'), ('conf', {}), ('inputs'…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "task_graph = TaskGraph.load_taskgraph('../taskgraphs/xgboost_example/ml_preprocess.gq.yaml')\n", + "task_graph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Machine Learning\n", + "### Train an XGBoost Model and run inference\n", + "In this step, we feed the prepared dataset to train an XGBoost model. The output model object is used to run inference for both train and test dataset.\n", + "\n", + "\n", + "\n", + "### gQuant evaluates a subgraph, no wasted computation\n", + "In this step, we show by switching to a dask_cudf output port, the graph can run in a distributed environment automatically. The graph only does computations on the nodes that are necessary for the results. \n", + "\n", + "\n", + "\n", + "After this step, you should have a TaskGraph looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bcb12ff81adf42bda35da31d76eb0a55", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'data_gen'), ('type', 'ClassificationData'), ('conf', {'n_…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "task_graph = TaskGraph.load_taskgraph('../taskgraphs/xgboost_example/xgboost_model.gq.yaml')\n", + "task_graph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Visualize the training result\n", + "gQuant provide analysis nodes to evaluate the XGBoost model. In this step, will check the ROC curve and feature importances\n", + "\n", + "\n", + "\n", + "After this step, you should have a TaskGraph looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b5b91f82dcb1486ca3e34a490ed596b0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'data_gen'), ('type', 'ClassificationData'), ('conf', {'n_…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "task_graph = TaskGraph.load_taskgraph('../taskgraphs/xgboost_example/metrics.gq.yaml')\n", + "task_graph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Forest inference for deployment\n", + "Forest inference library provides a great performance boost for XGBoost model inference as shown in this [blog](https://medium.com/rapids-ai/rapids-forest-inference-library-prediction-at-100-million-rows-per-second-19558890bc35). In this step, we export the XGBoost model to use a forest inference library. \n", + "\n", + "\n", + "### Distributed inference\n", + "We can run inference in a distributed environment. Usually, we have a lot of data to process in production.\n", + "\n", + "\n", + "After this step, you should have a TaskGraph looks like this:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "96b5730637b64d85b0510bf8aabac866", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'data_gen'), ('type', 'ClassificationData'), ('conf', {'n_…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "task_graph = TaskGraph.load_taskgraph('../taskgraphs/xgboost_example/tree_inference.gq.yaml')\n", + "task_graph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a custom node\n", + "Since we have a nice XGBoost model composite node, let's convert it to a normal gQuant node for future use without writing any Python code. How cool is that!\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Real life example\n", + "\n", + "We have been working on the fake data for predicting some binary classes. Let's change the dataset to something meaningful.\n", + "\n", + "### Get the stock data\n", + "We prepare a dataset that calculates the features using technical indicators. We convert the next day return into a binary label indicating positive or negative returns.\n", + "We re-use the TaskGraph from previous [06_xgboost_trade](https://github.com/rapidsai/gQuant/blob/master/notebooks/06_xgboost_trade.ipynb) notebook. \n", + "\n", + "\n", + "After this step, you should have a TaskGraph looks like this:\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "42f163f7df04427082b0bca9b4c5d146", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'stock_data'), ('type', 'CsvStockLoader'), ('conf', {'file…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "task_graph = TaskGraph.load_taskgraph('../taskgraphs/xgboost_example/stock_data.gq.yaml')\n", + "task_graph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run the XGBoost model on the stock data\n", + "This is the last step! We add our custom XGBoost node created before. We can now train and make predictions for our stock dataset easily. As you can see, the ROC value is not bad at all!\n", + "\n", + "\n", + "After this step, you should have a TaskGraph looks like this:\n", + "\n", + "Note, you need to create the custom node as shown before to see this graph." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "be0aff8a7f8e40f390044d377ae9e217", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'stock_data'), ('type', 'CsvStockLoader'), ('conf', {'file…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "task_graph = TaskGraph.load_taskgraph('../taskgraphs/xgboost_example/xgboost_stock.gq.yaml')\n", + "task_graph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clean Up" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'status': 'ok', 'restart': True}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import IPython\n", + "app = IPython.Application.instance()\n", + "app.kernel.do_shutdown(True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/09_gquant_machine_hpo.ipynb b/notebooks/09_gquant_machine_hpo.ipynb new file mode 100644 index 00000000..dcdb0e7c --- /dev/null +++ b/notebooks/09_gquant_machine_hpo.ipynb @@ -0,0 +1,211 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# gQuant Hyperparameter Tuning Example\n", + "\n", + "When data scientists are building machine learning models, there are a few magic numbers that are included in the model. The example numbers are depths in the tree, the learning rate, etc. The parameters that define the model architecture or training process are usually referred to as hyperparameters. They are magical because there is no good theory about what number to choose. Commonly, the hyperparameters can be searched to find a good set of them that results in the best model performance. The process of searching is referred to as hyperparameter tuning. \n", + "\n", + "There are a few popular Python-based hyperparameter tuning libraries existing: [Ray Tune](https://docs.ray.io/en/latest/tune/), [Optuna](https://optuna.org/), [HyperOpt](https://hyperopt.github.io/hyperopt/). Each library provides a set of search algorithms and schedule algorithms that is easy to use. \n", + "\n", + "Inside the gQuant, we implemented a `Context Composite Node` that can flexibly expose the hyperparameters that are interesting for tuning. The `Context Composite Node` makes hyperparameter tuning easy to do by leveraging the hyperparameter tuning libraries. Without loss of generality, we show in this tutorial an example of using `Ray Tune` to optimize the hyperparameters for an XGBoost model \n", + "\n", + "First, let's load the gQuant library" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import sys; sys.path.insert(0, '..')\n", + "from gquant.dataframe_flow import TaskGraph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`Ray Tune` is built on top of the library [ray](https://ray.io/), which is a distributed execution framework that makes it easy to do distributed computation. Let's setup the `ray` environment so we can utilize all the GPUs in the host node for hyperparameter search." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 16:55:56,065\tINFO resource_spec.py:231 -- Starting Ray with 52.69 GiB memory available for workers and up to 26.35 GiB for objects. You can adjust these settings with ray.init(memory=, object_store_memory=).\n", + "2020-10-09 16:55:56,629\tINFO services.py:1193 -- View the Ray dashboard at \u001b[1m\u001b[32m172.17.0.2:8265\u001b[39m\u001b[22m\n", + "2020-10-09 16:55:56,634\tWARNING services.py:1567 -- WARNING: The object store is using /tmp instead of /dev/shm because /dev/shm has only 67100672 bytes available. This may slow down performance! You may be able to free up space by deleting files in /dev/shm or terminating any running plasma_store_server processes. If you are inside a Docker container, you may need to pass an argument with the flag '--shm-size' to 'docker run'.\n", + "2020-10-09 16:55:56,649\tWARNING services.py:1567 -- WARNING: The object store is using /tmp instead of /dev/shm because /dev/shm has only 67100672 bytes available. This may slow down performance! You may be able to free up space by deleting files in /dev/shm or terminating any running plasma_store_server processes. If you are inside a Docker container, you may need to pass an argument with the flag '--shm-size' to 'docker run'.\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.17.0.2',\n", + " 'raylet_ip_address': '172.17.0.2',\n", + " 'redis_address': '172.17.0.2:6379',\n", + " 'object_store_address': '/tmp/ray/session_2020-10-09_16-55-56_064065_5720/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-10-09_16-55-56_064065_5720/sockets/raylet',\n", + " 'webui_url': '172.17.0.2:8265',\n", + " 'session_dir': '/tmp/ray/session_2020-10-09_16-55-56_064065_5720'}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ray\n", + "ray.init(dashboard_host='0.0.0.0')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the `09_gquant_machine_leanring` notebook, we constructed a computation graph in gQuant to train a XGBoost model and run inference. We will reuse that example computation graph `xgboost_with_metrics.gq.yaml` for hyper-parameter tuning. Let's load the hyperparameter search computation graph:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0667e46df07e4b3cb68992a59eee939a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'data_gen'), ('type', 'ClassificationData'), ('conf', {'n_…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "taskGraph = TaskGraph.load_taskgraph('../taskgraphs/xgboost_example/hyper_parameters_search.gq.yaml')\n", + "taskGraph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As you can see above, the `xgboost_with_metrics.gq.yaml` graph is loaded into a `Context Composite Node` with id `xgboost_model`. It exposes two input ports from the loaded graph nodes to get the training dataframe and testing dataframe. It also exposes the output ports from the loaded graph nodes to output the XGBoost training and inference results. So this composite node encapsulates the XGBoost computation graph into a single node. \n", + "\n", + "Most importantly, the context composite node can create some context parameters for the graph `xgboost_with_metrics`. The context parameter can map its value to a set of configuration parameters of the nodes inside the `xgboost_with_metrics` graph. So it is one-to-many mapping relation. When the context composite node running the computation, the values of the mapped configuration parameters will be substituted by the context parameters. Try to click on `xgboost_model` to see how to add a new context parameter and map it to a node configuration parameter.\n", + "\n", + "The context composite node has a default `conf_in` port which can take a configuration object as input. Once it is connected, the context composite node will ignore its default configuration and use the fed configuration. The default `conf_out` port passes the configuration for other nodes who can take `ConfData` object. \n", + "\n", + "The `GridRandomSearchNode` in the above graph is a subclass of `Context Composite Node`, so it can take the `ConfData` from other `Context Composite Node` and expose the same input and output ports. It knows what are the context parameters defined in the `xgboost_model` node and the user can select the ones that will be used for grid and random searches. In this example, we will do a grid search for `depth` and random search for `eta`. Once the best hyperparameter is found, it will be used to do the XGBoost computation. So the outputs from `hpo` node are the XGBoost training and inference results with the best hyperparameter.\n", + "\n", + "Click on the `run` button above and check the XGBoost results before and after the hyperparameter tuning. During the run the log console has informative information from the Ray Tune library. Click one the 'list' button to see the log or you can go to \"View -> Show Log Console\".\n", + "\n", + "Since you know how hyperparameter tuning is done by gQuant, we can work on a more interesting problem. In the following graph, we will do hyperparameter tuning to find the best XGBoost model to predict the positive/negative next stock return. Note, we keep everything the same but change the dataset in the graph. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f541a93ca3744ab1842cbbc1a19d2e07", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', ''), ('type', 'Output_Collector'), ('conf', {}), ('inputs'…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "taskGraph = TaskGraph.load_taskgraph('../taskgraphs/xgboost_example/xgboost_stock_hpo.gq.yaml')\n", + "taskGraph.draw()" + ] + }, + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "After the hyperparameter tuning, the AUC for the test dataset is improved to 0.54. Great!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clean up" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'status': 'ok', 'restart': True}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import IPython\n", + "app = IPython.Application.instance()\n", + "app.kernel.do_shutdown(True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/10_nemo_chatbot.ipynb b/notebooks/10_nemo_chatbot.ipynb new file mode 100644 index 00000000..9279493c --- /dev/null +++ b/notebooks/10_nemo_chatbot.ipynb @@ -0,0 +1,1252 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# gQuant NeMo Chatbot Example\n", + "\n", + "In the previous example notebooks, we use gQuant to perform ETL, backtesting, machine learning, hyperparameter tuning tasks. As we see, gQuant can be extended easily for other types of workflow accelerations as long as the computation can be organized as graph computations. \n", + "\n", + "[NeMo](https://github.com/NVIDIA/NeMo) is a toolkit for creating Conversational AI applications. Most importantly, NeMo abstracts the neural network into neural modules with input and output ports similar to gQuant nodes. So the neural network computation is organized as a computation graph made of neural modules. Because of the similarity between gQuant node and neural modules, the neural module can be easily wrapped up into a gQuant node. NeMo is integrated into the gQuant so the neural modules can be wired, trained, inferred visually with the help of gQuant UI. \n", + "\n", + "In this tutorial, we will show how to use gQuant to train an RNN model for chatbot applications. \n", + "\n", + "## NeMo gQuant integration\n", + "\n", + "Convert the NeMo neural module to gQuant node is easy. gQuant provides a base NeMo node that can be used for inheritance for neural modules. The child class just need to pass in the class name of the Neural Module in the constructor. Here is one example that converts the EncoderRNN Neural Module to gQuant node.\n", + "\n", + "```python\n", + "from gquant.plugin_nodes.nemo_util.nemoBaseNode import NeMoBase\n", + "import nemo\n", + "import nemo.backends.pytorch.tutorials\n", + "\n", + "class EncoderRNNNode(NeMoBase):\n", + " def init(self):\n", + " super().init(nemo.backends.pytorch.tutorials.chatbot.modules.EncoderRNN)\n", + "```\n", + "\n", + "`NeMoBase` inspects the `EncoderRNN` constructor signature and converts it as node configuration JSON schema. It automatically infers the types for each of the parameters. If the auto type inference has an error, it can be fixed by modifying the `self.fix_type` dictionary in the constructor. The keys of the dictionary are the parameter names while the values are the type strings.\n", + "\n", + "In the `util` directory, we provide a script `auto_gen.py` that automatically converts the NeMo Neural Modules into gQuant nodes. Currently, it converts 90% of the Neural Modules. The converted Python files are exported to the `modules` directory. gQuant by design can load any externally defined nodes by defining the node modules in the `gquantrc` file. Here is an example `gquantrc` file that loads the converted NeMo neural module files:\n", + "\n", + "```ini\n", + "nemo.asr= %(MODULEPATH)s/asr.py\n", + "nemo.common= %(MODULEPATH)s/common.py\n", + "nemo.cv= %(MODULEPATH)s/cv.py\n", + "nemo.nlp= %(MODULEPATH)s/nlp.py\n", + "nemo.simple_gan= %(MODULEPATH)s/simple_gan.py\n", + "nemo.tts= %(MODULEPATH)s/tts.py\n", + "nemo.tutorials= %(MODULEPATH)s/tutorials.py\n", + "```\n", + "Note, gQuant can automatically interpret the `.` in the module names to form hierarchical menus.\n", + "\n", + "\n", + "## Data Preparation\n", + "\n", + "Let's first prepare the chatbot dataset that will be used to train an RNN model." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import gzip\n", + "import os\n", + "import shutil\n", + "data_file = \"movie_data.txt\"\n", + "# Download the data file.\n", + "if not os.path.isfile(data_file):\n", + " with gzip.open(\"../../NeMo/tests/data/movie_lines.txt.gz\", 'rb') as f_in:\n", + " with open(data_file, 'wb') as f_out:\n", + " shutil.copyfileobj(f_in, f_out)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It assumes you are using the docker container to run this example. The `movie_lines.txt.gz` file is located at `../../NeMo/tests/data/`. If it is not the case, the data file can be fetched by the following command.\n", + "```bash\n", + "wget https://github.com/NVIDIA/NeMo/raw/v0.11.1/tests/data/movie_lines.txt.gz\n", + "gzip -d movie_lines.txt.gz\n", + "mv movie_lines.txt movie_data.txt\n", + "```\n", + "\n", + "Let's load the necessary NeMo module and gQuant module, and setup the Neural Module Factory environment. " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# start the neural module factory\n", + "import nemo\n", + "nemo.core.NeuralModuleFactory()\n", + "import json\n", + "from gquant.dataframe_flow import TaskGraph" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Later we will use Ray Tune for hyperparameter tuning. Let's setup the ray environment so we can utilize all the GPUs in the host node for hyperparameter search." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 18:19:07,391\tINFO resource_spec.py:231 -- Starting Ray with 52.64 GiB memory available for workers and up to 26.34 GiB for objects. You can adjust these settings with ray.init(memory=, object_store_memory=).\n", + "2020-10-09 18:19:07,958\tINFO services.py:1193 -- View the Ray dashboard at \u001b[1m\u001b[32m172.17.0.2:8265\u001b[39m\u001b[22m\n", + "2020-10-09 18:19:07,963\tWARNING services.py:1567 -- WARNING: The object store is using /tmp instead of /dev/shm because /dev/shm has only 67100672 bytes available. This may slow down performance! You may be able to free up space by deleting files in /dev/shm or terminating any running plasma_store_server processes. If you are inside a Docker container, you may need to pass an argument with the flag '--shm-size' to 'docker run'.\n", + "2020-10-09 18:19:07,981\tWARNING services.py:1567 -- WARNING: The object store is using /tmp instead of /dev/shm because /dev/shm has only 67100672 bytes available. This may slow down performance! You may be able to free up space by deleting files in /dev/shm or terminating any running plasma_store_server processes. If you are inside a Docker container, you may need to pass an argument with the flag '--shm-size' to 'docker run'.\n" + ] + }, + { + "data": { + "text/plain": [ + "{'node_ip_address': '172.17.0.2',\n", + " 'raylet_ip_address': '172.17.0.2',\n", + " 'redis_address': '172.17.0.2:6379',\n", + " 'object_store_address': '/tmp/ray/session_2020-10-09_18-19-07_390022_8297/sockets/plasma_store',\n", + " 'raylet_socket_name': '/tmp/ray/session_2020-10-09_18-19-07_390022_8297/sockets/raylet',\n", + " 'webui_url': '172.17.0.2:8265',\n", + " 'session_dir': '/tmp/ray/session_2020-10-09_18-19-07_390022_8297'}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ray\n", + "ray.init(dashboard_host='0.0.0.0')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Chatbot Model\n", + "\n", + "The chat bot model is taken from the NeMo tutorial example. It consists of a Data layer, a RNN encoder, RNN decoder for Cross Entropy loss and Greedy RNN decoder for text inference. Load the TaskGraph into gQuant:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8f7d22d0e518444189a4e71c70ba52d0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'data'), ('type', 'DialogDataLayerNode'), ('conf', {'batch…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "taskGraph=TaskGraph.load_taskgraph('../taskgraphs/nemo_examples/chatbot_example.gq.yaml')\n", + "taskGraph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You may notice that each trainable neural module has an `in_nm` input port that takes a neural module as input. This input port is mainly used for sharing weights between neural modules. The gQuant Neural Module wrapper supports all 3 weight sharing mechanisms mentioned in the [NeMo documents](https://docs.nvidia.com/deeplearning/nemo/user-guide/docs/en/v0.11.0/tutorials/weightsharing.html): Module Reuse, Weight Copying, and Weight Tying. Click on the trainable neural module node above, you will see that the weight sharing method can be configured easily in a dropdown list. In the above example, the encoder and decoder modules for thes evaluation dataset reuse the encoder and decoder modules for training. The greedy decoder shares the weights with the decoder by tying the weights. \n", + "\n", + "All the neural modules have an `out_nm` port that outputs the neural module instance itself for other nodes to consume. For example, it can be used for weight sharing. Or, it can be used to extract other important information from the neural module. Later in the inference stage, we use the outputted data layer neural module to extract the text dictionary information. \n", + "\n", + "There are two special nodes in the above graph: `NeMo Train Node` and `NeMo Infer Node`. As the name indicates, they are used for training and inference for a NeMo module graph. Both nodes can connect to any number of `NmTensors` which are the outputs from the Neural Modules. In the `NeMo Train Node`, the user can specify the `NmTensor` for training loss, and a list of `NmTensors` for logging. Users can select the optimization method from a list of pre-defined optimization methods. Users can also select the `WarmUp Policy` from a list of pre-defined policies. Other configurations like the checkpoints and training epochs can be configured in a similar way. The output port of the `NeMo Train Node` is the directory for the checkpoint files. In the above example, it is used by `NeMo Infer Node` to load the checkpoint files, and run inference. \n", + "\n", + "Click on the `run` button to see the training and inference of this chatbot model are in action. During the run the log console has informative information from the Ray Tune library. Click one the 'list' button to see the log or you can go to \"View -> Show Log Console\". It will train 3 epochs by default. If better accuracy is needed, try to increase the number of epochs in the `NeMo Train Node` and run it again.\n", + "\n", + "The chatbot graph is a little complicated. We can wrap the whole computation graph into a `Context Composite Node` as explained in the `08_gquant_machine_learning` notebook. The number of layers for encoders and decoders and the drop out rate parameters are exposed as context parameters for this context composite node. We will tune these two parameters later. \n", + "\n", + "Load this simplified graph:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2231d4203a84401ebf0d39e922404d36", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'rnn_train'), ('type', 'ContextCompositeNode'), ('conf', {…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "taskGraph=TaskGraph.load_taskgraph('../taskgraphs/nemo_examples/chatbot_simplified.gq.yaml')\n", + "taskGraph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The context composite node output the evaluation results from the inference node and the evaluation data layer neural module for extracting the text dictionary. It produces the same result as the previous complicated graph. The graph can be run in a programmatically way. Let's verify the result by the following command: " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NeMo I 2020-10-09 17:50:57 data:132] Start preparing training data ...\n", + "[NeMo I 2020-10-09 17:50:57 data:102] Reading lines...\n", + "[NeMo I 2020-10-09 17:51:09 data:134] Read 150000 sentence pairs\n", + "[NeMo I 2020-10-09 17:51:09 data:136] Trimmed to 43840 sentence pairs\n", + "[NeMo I 2020-10-09 17:51:09 data:137] Counting words...\n", + "[NeMo I 2020-10-09 17:51:09 data:141] Bad message (TypeError('not all arguments converted during string formatting')): {'name': 'nemo_logger', 'msg': 'Counted words:', 'args': (14455,), 'levelname': 'INFO', 'levelno': 20, 'pathname': '/home/quant/NeMo/nemo/backends/pytorch/tutorials/chatbot/data.py', 'filename': 'data.py', 'module': 'data', 'exc_info': None, 'exc_text': None, 'stack_info': None, 'lineno': 141, 'funcName': 'loadPrepareData', 'created': 1602265869.9934738, 'msecs': 993.4737682342529, 'relativeCreated': 267813.71688842773, 'thread': 140232394950464, 'threadName': 'MainThread', 'processName': 'MainProcess', 'process': 7025}\n", + "[NeMo I 2020-10-09 17:51:09 data:58] keep_words 6104 / 14452 = 0.4224\n", + "[NeMo I 2020-10-09 17:51:10 data:132] Start preparing training data ...\n", + "[NeMo I 2020-10-09 17:51:10 data:102] Reading lines...\n", + "[NeMo I 2020-10-09 17:51:22 data:134] Read 150000 sentence pairs\n", + "[NeMo I 2020-10-09 17:51:23 data:136] Trimmed to 43840 sentence pairs\n", + "[NeMo I 2020-10-09 17:51:23 data:137] Counting words...\n", + "[NeMo I 2020-10-09 17:51:23 data:141] Bad message (TypeError('not all arguments converted during string formatting')): {'name': 'nemo_logger', 'msg': 'Counted words:', 'args': (14455,), 'levelname': 'INFO', 'levelno': 20, 'pathname': '/home/quant/NeMo/nemo/backends/pytorch/tutorials/chatbot/data.py', 'filename': 'data.py', 'module': 'data', 'exc_info': None, 'exc_text': None, 'stack_info': None, 'lineno': 141, 'funcName': 'loadPrepareData', 'created': 1602265883.4942698, 'msecs': 494.26984786987305, 'relativeCreated': 281314.51296806335, 'thread': 140232394950464, 'threadName': 'MainThread', 'processName': 'MainProcess', 'process': 7025}\n", + "[NeMo I 2020-10-09 17:51:23 data:58] keep_words 6104 / 14452 = 0.4224\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[NeMo W 2020-10-09 17:51:24 callbacks:415] No checkpoints will be saved because step_freq and epoch_freq are both -1.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NeMo I 2020-10-09 17:51:24 callbacks:534] Found 3 modules with weights:\n", + "[NeMo I 2020-10-09 17:51:24 callbacks:536] GreedyLuongAttnDecoderRNN\n", + "[NeMo I 2020-10-09 17:51:24 callbacks:536] LuongAttnDecoderRNN\n", + "[NeMo I 2020-10-09 17:51:24 callbacks:536] EncoderRNN\n", + "[NeMo I 2020-10-09 17:51:24 callbacks:537] Total model parameters: 30876086\n", + "[NeMo I 2020-10-09 17:51:24 callbacks:473] Found checkpoint folder nemo_log. Will attempt to restore checkpoints from it.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[NeMo W 2020-10-09 17:51:24 callbacks:499] For module EncoderRNN, no file matches in nemo_log\n", + "[NeMo W 2020-10-09 17:51:24 callbacks:501] Checkpoint folder nemo_log was present but nothing was restored. Continuing training from random initialization.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NeMo I 2020-10-09 17:51:24 callbacks:232] loss: 8.730251\n", + "[NeMo I 2020-10-09 17:51:24 deprecated_callbacks:316] Doing Evaluation ..............................\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 17:51:26,659\tWARNING session.py:14 -- Session not detected. You should not be calling this function outside `tune.run` or while using the class API. \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "eval: loss 8.307969\n", + "[NeMo I 2020-10-09 17:51:26 deprecated_callbacks:321] Evaluation time: 2.3719964027404785 seconds\n", + "[NeMo I 2020-10-09 17:51:32 callbacks:232] loss: 4.24844\n", + "[NeMo I 2020-10-09 17:51:32 deprecated_callbacks:316] Doing Evaluation ..............................\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 17:51:34,396\tWARNING session.py:14 -- Session not detected. You should not be calling this function outside `tune.run` or while using the class API. \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "eval: loss 4.2280946\n", + "[NeMo I 2020-10-09 17:51:34 deprecated_callbacks:321] Evaluation time: 2.2655580043792725 seconds\n", + "[NeMo I 2020-10-09 17:51:39 callbacks:232] loss: 3.9356227\n", + "[NeMo I 2020-10-09 17:51:39 deprecated_callbacks:316] Doing Evaluation ..............................\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 17:51:42,315\tWARNING session.py:14 -- Session not detected. You should not be calling this function outside `tune.run` or while using the class API. \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "eval: loss 3.783767\n", + "[NeMo I 2020-10-09 17:51:42 deprecated_callbacks:321] Evaluation time: 2.654122829437256 seconds\n", + "[NeMo I 2020-10-09 17:51:47 callbacks:232] loss: 3.8796775\n", + "[NeMo I 2020-10-09 17:51:47 deprecated_callbacks:316] Doing Evaluation ..............................\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 17:51:49,971\tWARNING session.py:14 -- Session not detected. You should not be calling this function outside `tune.run` or while using the class API. \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "eval: loss 3.5885432\n", + "[NeMo I 2020-10-09 17:51:49 deprecated_callbacks:321] Evaluation time: 2.314290761947632 seconds\n", + "[NeMo I 2020-10-09 17:51:56 callbacks:232] loss: 3.4285002\n", + "[NeMo I 2020-10-09 17:51:56 deprecated_callbacks:316] Doing Evaluation ..............................\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 17:51:58,692\tWARNING session.py:14 -- Session not detected. You should not be calling this function outside `tune.run` or while using the class API. \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "eval: loss 3.3833182\n", + "[NeMo I 2020-10-09 17:51:58 deprecated_callbacks:321] Evaluation time: 2.6722679138183594 seconds\n", + "[NeMo I 2020-10-09 17:52:04 callbacks:232] loss: 3.3505182\n", + "[NeMo I 2020-10-09 17:52:04 deprecated_callbacks:316] Doing Evaluation ..............................\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 17:52:07,022\tWARNING session.py:14 -- Session not detected. You should not be calling this function outside `tune.run` or while using the class API. \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "eval: loss 3.2234194\n", + "[NeMo I 2020-10-09 17:52:07 deprecated_callbacks:321] Evaluation time: 2.358950138092041 seconds\n", + "[NeMo I 2020-10-09 17:52:12 callbacks:232] loss: 3.018219\n", + "[NeMo I 2020-10-09 17:52:12 deprecated_callbacks:316] Doing Evaluation ..............................\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 17:52:14,945\tWARNING session.py:14 -- Session not detected. You should not be calling this function outside `tune.run` or while using the class API. \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "eval: loss 3.0797098\n", + "[NeMo I 2020-10-09 17:52:14 deprecated_callbacks:321] Evaluation time: 2.337757110595703 seconds\n", + "[NeMo I 2020-10-09 17:52:20 callbacks:232] loss: 3.0476317\n", + "[NeMo I 2020-10-09 17:52:20 deprecated_callbacks:316] Doing Evaluation ..............................\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 17:52:23,207\tWARNING session.py:14 -- Session not detected. You should not be calling this function outside `tune.run` or while using the class API. \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "eval: loss 2.915382\n", + "[NeMo I 2020-10-09 17:52:23 deprecated_callbacks:321] Evaluation time: 2.6708412170410156 seconds\n", + "[NeMo I 2020-10-09 17:52:29 callbacks:232] loss: 2.988197\n", + "[NeMo I 2020-10-09 17:52:29 deprecated_callbacks:316] Doing Evaluation ..............................\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 17:52:31,503\tWARNING session.py:14 -- Session not detected. You should not be calling this function outside `tune.run` or while using the class API. \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "eval: loss 2.773505\n", + "[NeMo I 2020-10-09 17:52:31 deprecated_callbacks:321] Evaluation time: 2.327364444732666 seconds\n", + "[NeMo I 2020-10-09 17:52:32 deprecated_callbacks:339] Final Evaluation ..............................\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-10-09 17:52:34,811\tWARNING session.py:14 -- Session not detected. You should not be calling this function outside `tune.run` or while using the class API. \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "eval: loss 2.7484186\n", + "[NeMo I 2020-10-09 17:52:34 deprecated_callbacks:344] Evaluation time: 2.294581413269043 seconds\n", + "node inference, from oport checkpoint_dir not in out cols\n", + "{}\n", + "[NeMo I 2020-10-09 17:52:34 actions:695] Evaluating batch 0 out of 273\n", + "[NeMo I 2020-10-09 17:52:35 actions:695] Evaluating batch 27 out of 273\n", + "[NeMo I 2020-10-09 17:52:36 actions:695] Evaluating batch 54 out of 273\n", + "[NeMo I 2020-10-09 17:52:37 actions:695] Evaluating batch 81 out of 273\n", + "[NeMo I 2020-10-09 17:52:38 actions:695] Evaluating batch 108 out of 273\n", + "[NeMo I 2020-10-09 17:52:39 actions:695] Evaluating batch 135 out of 273\n", + "[NeMo I 2020-10-09 17:52:39 actions:695] Evaluating batch 162 out of 273\n", + "[NeMo I 2020-10-09 17:52:40 actions:695] Evaluating batch 189 out of 273\n", + "[NeMo I 2020-10-09 17:52:41 actions:695] Evaluating batch 216 out of 273\n", + "[NeMo I 2020-10-09 17:52:42 actions:695] Evaluating batch 243 out of 273\n", + "[NeMo I 2020-10-09 17:52:43 actions:695] Evaluating batch 270 out of 273\n" + ] + } + ], + "source": [ + "result = taskGraph.run()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The graph computation results are stored in the `result` variable. It can be used as a named tuple or a dictionary. The keys of the result dictionary can be queried by the following command:|" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "('rnn_train.inference@torch_tensor', 'rnn_train.eval_data@out_nm')" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result.get_keys()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The evaluated greedy tensors are indices of the predicted letters in the dictionary. We can translate the numbers to human-readable texts by the following convenient function:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from nemo.utils import logging\n", + "# Define the callback function which prints intermediate results to console.\n", + "def outputs2words(tensors, vocab):\n", + " source_ids = tensors[1][:, 0].cpu().numpy().tolist()\n", + " response_ids = tensors[2][:, 0].cpu().numpy().tolist()\n", + " tgt_ids = tensors[3][:, 0].cpu().numpy().tolist()\n", + " source = list(map(lambda x: vocab[x], source_ids))\n", + " response = list(map(lambda x: vocab[x], response_ids))\n", + " target = list(map(lambda x: vocab[x], tgt_ids))\n", + " source = ' '.join([s for s in source if s != 'EOS' and s != 'PAD'])\n", + " response = ' '.join([s for s in response if s != 'EOS' and s != 'PAD'])\n", + " target = ' '.join([s for s in target if s != 'EOS' and s != 'PAD'])\n", + " logging.info(f\"Train Loss:{str(tensors[0].item())}\")\n", + " logging.info(f\"SOURCE: {source} <---> PREDICTED RESPONSE: {response} \" f\"<---> TARGET: {target}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We are ready to check the quality of the model performance. Note, by default, only 3 epochs are trained. For better performance, please increase the number of epochs." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8878540992736816\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: oh . just not at school . . . <---> PREDICTED RESPONSE: what ? ? ? ? ? <---> TARGET: yeah\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8501431941986084\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you know what they re doin now lou . <---> PREDICTED RESPONSE: i m sorry . i m sorry . <---> TARGET: this i know benny .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.992612600326538\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: are you sure this is the river road ? <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: i saw the sign .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9164655208587646\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: it didn t hurt too much did it ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: ye . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.963688611984253\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i used to throw trash for the city . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: you lost that one ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8423519134521484\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i see he d walk out next month . <---> PREDICTED RESPONSE: i know . . . . <---> TARGET: that s right .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.822286605834961\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i don t recall ever seeing it before . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: well it was there .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.689497232437134\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: tommy stop it ! you re scaring lindsey . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: i saw him . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8311920166015625\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: don t you pull that shit with me . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: i didn t want to kill her will .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7959091663360596\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: bob . bet you had a late night . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: sleep is for the weak mr . president .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8443610668182373\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: so i see you re really into spooks . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: no . i never date outside my race .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.842954635620117\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you okay ? you seem a little off . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: nothing a good stiff drink can t fix .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.0126748085021973\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: tell ashley . . . i . . . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: you tell her !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8383681774139404\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: that means you won t have a gun . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: that s right .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.00457763671875\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: yeah . . . how ya doin pete ? <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: okay .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.763681173324585\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: nobody knows what we re talking about jack . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: you re so naive mister .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.933351755142212\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you know what s happened don t you ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: what ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.040013313293457\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: these disks . . . where are they ? <---> PREDICTED RESPONSE: i m here . . . . <---> TARGET: smith will know .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8634536266326904\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: but what about the past ? our past ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: we never really shared one .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.0505802631378174\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: we still have to get out of here . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: maybe if we gave the tapes back\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.159726142883301\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i . . . i don t understand . <---> PREDICTED RESPONSE: you know what i m doing . . <---> TARGET: agnes how are babies born ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.140392303466797\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: it s so good to see you sweetheart . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: you too mom .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.876481056213379\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: can i just ask you one other thing ? <---> PREDICTED RESPONSE: sure . . . . . <---> TARGET: maybe .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9994771480560303\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: just ask me what you want to know . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: oh you gonna make me say it .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.827291250228882\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you re going to miss the top ten . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: coming .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.882018804550171\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: if i may be so bold . . . <---> PREDICTED RESPONSE: i m sorry . . . . . <---> TARGET: for the gods spit it out !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.920955181121826\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: have you discussed this with anyone else john ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: no .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.0991899967193604\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: what s that got to do with it ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: you are aren t you ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9162800312042236\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: do you mind if i drink this here ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: sure . go ahead .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8232173919677734\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i had it on the island with me . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: must be a story there .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9020822048187256\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: could we talk about that a little later ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: of course .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.901149034500122\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: hell yeah . she s not my unicorn . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: move over . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7928922176361084\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: but i am not human . i am only <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: spock you want to know something ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9447402954101562\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: nope . can t . we re through . <---> PREDICTED RESPONSE: you re welcome . you re doing . <---> TARGET: then i ll call you a cab .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.984252452850342\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: a piece of ass wouldn t kill me . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: when ya go back on ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.790090322494507\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: do you mind if i drink this here ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: sure . go ahead .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8723599910736084\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: at least you pulled yourself out . . . <---> PREDICTED RESPONSE: i m sorry . i m sorry . <---> TARGET: things got a lot worse .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.0517563819885254\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: . . .where d you get the pistol ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: at the gettin place .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.0015506744384766\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you should . do you have a resume ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: i do .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.922788381576538\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: . . .you moved too didn t you ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: yeah .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.945343494415283\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: abrams . yes . how are you today ? <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: i m fine .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.0862414836883545\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: not today . i ve got other plans . <---> PREDICTED RESPONSE: why ? you re not . ? <---> TARGET: oh . well when then ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.81083345413208\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: why ? because you can do it alone ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: no sir !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9346556663513184\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: but no one knew until after . . . <---> PREDICTED RESPONSE: i know . . . . <---> TARGET: after i . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.0722873210906982\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: why don t you just go now okay ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: i told you i m not easily shocked .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8582704067230225\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: he s great right . is he great ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: he s alright .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9171805381774902\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: what the fuck are we going to do ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: hide him .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8178727626800537\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: this letter . i ve never opened it . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: why not ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8843209743499756\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: did you take anything out of those pants ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: oh no sir !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.900236129760742\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: go away i don t want you here . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: is someone in the room with you ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.0172202587127686\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: why do people have to love people anyway ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: yeah i know what you mean . queen .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7929372787475586\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: in your pocket . give it to me . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: i need it .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8310770988464355\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: in england i was a poor student but i <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: that was england .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7938899993896484\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i did my best but well you know me <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: sure\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.730086326599121\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: shit girl you couldn t a been eleven . <---> PREDICTED RESPONSE: i m sorry . i was . . <---> TARGET: hell i was game !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.88199782371521\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: out of your element aren t you captain ? <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: sort of .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9156126976013184\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: . . .you need to talk to me . <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: i don t need to talk to you .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9136767387390137\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: it s the lights . and the scotch . <---> PREDICTED RESPONSE: oh . . . . . <---> TARGET: how come you didn t catch your bus ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8318889141082764\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: u do you have your passport sir u ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: look maybe i should just . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9010751247406006\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: fergus . fergus my love light of my life <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: please dil\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.793943166732788\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i get real busy between two and four . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: okay .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.891098976135254\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i m going to get started on dishes . <---> PREDICTED RESPONSE: i m sorry . you re not . <---> TARGET: i ll help you .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.666313409805298\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: marion ! what did you tell the police ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: nothing about you .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.809729814529419\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i think jamie s been seeing karl again . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: great .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.856290340423584\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you understand which side the gelatin s on ? <---> PREDICTED RESPONSE: yes sir . . . . <---> TARGET: it tastes wonderful !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8982906341552734\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i don t understand why you do that . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: why i do what ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9323222637176514\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: only one in town . get in son . <---> PREDICTED RESPONSE: what s the matter ? ? . . <---> TARGET: ben when s delly due back ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8736207485198975\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: just like they say no fucking self control . <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: if you know that just back off .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8052327632904053\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: grandma death will teach me how . soon . <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: then how is time travel possible ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8905937671661377\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: how do you become something you re not ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: what would you like to be ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.797628402709961\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you re wrong . look at his face . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: one cannot love and kill .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8733725547790527\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: go out and plug the cord back in . <---> PREDICTED RESPONSE: what ? . . . ? ! <---> TARGET: what ? who pulled it out ? !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.899550676345825\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: shit ben ! how the shit are ya ? <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: fine frank . fine . how are you ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7686643600463867\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i don t want you to come anyway . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: what ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9078586101531982\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: they don t build them like this anymore . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: suppose not .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.939918041229248\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you getting into bed to think about yellow ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: i likes yellow .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.797706365585327\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: is her mother still lying on the floor ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: no she s fine .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6545722484588623\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: there s a lot you don t know . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: and don t let her know either .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.763164758682251\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: emmett ! am i glad to see you ! <---> PREDICTED RESPONSE: allo ? ! ! ! ! <---> TARGET: howdy jake . what s going on here ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.828334331512451\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: stop it sam . stop it right now ! <---> PREDICTED RESPONSE: what ? ! ! ! ! <---> TARGET: but mom . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8645946979522705\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you mean you don t want to go ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: well i m\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8904104232788086\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: where have you been for a week ? ! <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: four days . i left to think .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.022343873977661\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i m . . . i m alright . <---> PREDICTED RESPONSE: you re not gonna be here ? ? <---> TARGET: how was your date ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7746710777282715\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: no . i never date outside my race . <---> PREDICTED RESPONSE: you re not gonna be here ? ? <---> TARGET: i meant you re into ghosts .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7685344219207764\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: look kind o natural don t they fred ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: a sight for sore eyes !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6845545768737793\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you re in love with the fucking housekeeper ! <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: shut up !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.841613292694092\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i ll never see them again mister mulvaney . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: they look like good kids .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.776188850402832\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: tell ashley . . . i . . . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: you tell her !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9695048332214355\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i m sorry . i really shouldn t have <---> PREDICTED RESPONSE: i m sorry . you re not . <---> TARGET: i m glad you called .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.686812400817871\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: your dad s really gonna build his house ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: i don t know .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9639575481414795\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you ll probably be bigger than your brothers . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: yeah !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.851066827774048\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i hope you re still alive jean luc . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: i am .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9121861457824707\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: hmm so you re in the picture business ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: you could say that\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.788832187652588\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i thought for sure they would catch me . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: thanks .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9260122776031494\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: no he s not dangerous . not physically . <---> PREDICTED RESPONSE: why ? you re not . ? <---> TARGET: well what then ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.85560941696167\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: he means the last that we know of . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: this one walk you home too ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8693552017211914\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: they are at the gates of paris sire . <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: and where is the army ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9904074668884277\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i can t believe what i m seeing . <---> PREDICTED RESPONSE: you re not gonna be here ? . <---> TARGET: i know this is fucking nuts .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.691725254058838\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i thought you never sold cigarettes to kids . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: i don t you did .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.931187868118286\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: mrs . robinson i can t do this . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: you what ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8008804321289062\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: well i don t know . that depends . <---> PREDICTED RESPONSE: you know what i m doing . . <---> TARGET: on what ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6993141174316406\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: there are worse things than vampires out there . <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: like what ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.071131706237793\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: no . you ll be marked for death . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: let me die then .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7377281188964844\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: . . .i ll fix ya some eggs . <---> PREDICTED RESPONSE: yeah . . . . . <---> TARGET: aw norm .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8402462005615234\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: fuck you . fuck the three of you . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: hey cool it walter .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.004519462585449\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you hold the counter and i ll pull . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: usually i just turn the can upside down .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8814632892608643\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: what i never could figure about the movie ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: what ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7245051860809326\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i ve got a weak signal over there . <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: split up . find him .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8225152492523193\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: what do u you u use it for ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: my cannon .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.905247926712036\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i thought you d know me by now . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: what does it take to kill you ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8939192295074463\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: raise the glass and push the blue button . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: that s it ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.84401798248291\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: stop talking like a monk ! just fight ! <---> PREDICTED RESPONSE: what ? ! ! ! ! <---> TARGET: then tell me where jade fox is .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.834115982055664\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: this denial of death . you remember this ? <---> PREDICTED RESPONSE: yeah . . . . . <---> TARGET: oh\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.948885917663574\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: everyone loves the girl next door particularly farley . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: so it seems .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9753313064575195\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: do i have to tell you ? moss . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: that was easy wasn t it ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.81467866897583\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: what s happening ? what do you want ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: so very much .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7608695030212402\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: that car belonged to my brother in law . <---> PREDICTED RESPONSE: what ? it s a gun ! ! <---> TARGET: come on come on !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8423244953155518\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: so you re really gonna be a lawyer ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: and why not ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7845089435577393\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: get out ? ! what does it say ? <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: fucked me .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.0750746726989746\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: would you like to talk about this friend ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: his name is frank .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.750349521636963\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: and where do you think you re going ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: out .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8856892585754395\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: no it s not true you re lying . <---> PREDICTED RESPONSE: no . i m sorry . . <---> TARGET: not this time kat .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9307048320770264\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: . . .you moved too didn t you ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: yeah .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.918248414993286\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you didn t tell me about red deer table <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: what the hell is red deer table ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9041430950164795\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: okay . i just wanted you to know . <---> PREDICTED RESPONSE: what s your name ? ? . <---> TARGET: what is she shooting ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.9373600482940674\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i always hoped i would hit two meters . <---> PREDICTED RESPONSE: i know . . . . <---> TARGET: with a full head of hair .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6289374828338623\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: ah good . now turn to your left . <---> PREDICTED RESPONSE: what s your name ? ? ? <---> TARGET: she talk about the boys ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.77809739112854\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: just meet with the boy once a week . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: can we do it at my office ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6789133548736572\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: no i ain t got you . . . <---> PREDICTED RESPONSE: i m sorry . . . . . <---> TARGET: no i ain t got you . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7832393646240234\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i am the best man for the job . <---> PREDICTED RESPONSE: i know . . . . <---> TARGET: why don t you do it then ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8523900508880615\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: otherwise we d insist on your being here . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: you know it s true .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6502010822296143\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: what do u you u use it for ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: my cannon .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.800779342651367\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: otherwise we d insist on your being here . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: you know it s true .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6893861293792725\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i m sorry she hung up . what are <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: finding out where she is .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8984262943267822\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: how long can i afford to stay closed ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: oh two weeks maybe three .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.710831642150879\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: a man wouldn t do that . . . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: he isn t a man .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.795376777648926\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: and what did he come back now for ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: who ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7878425121307373\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: brad ! have mom or dad seen this ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: they re not home yet .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.62880802154541\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i m getting real tired of this maniac . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: maybe we better call that psychiatric clinic .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.859851598739624\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: it doesn t sound natural when i curse . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: just make noises then .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.839189291000366\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i just wanted to know where mom was ? <---> PREDICTED RESPONSE: you re not talking about it . . <---> TARGET: oh . sorry .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.854353666305542\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: can you say that my name is jack ? <---> PREDICTED RESPONSE: sure . . . . . <---> TARGET: you want her to call you jack ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6548728942871094\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: okay i m closing up . everybody relax . <---> PREDICTED RESPONSE: what s your name ? ? ? <---> TARGET: may i have the surgeon s name please ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.884765386581421\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: or the living fall in . who knows . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: and you believe her ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.939957618713379\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: how much you figure that s worth skipper ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: hard to say . maybe forty fifty million .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7053096294403076\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: we been seein you all along the way . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: yeah . i been seein you too .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.628314733505249\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i know . he s just the best . <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: i ve got jews at my table .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.794163942337036\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i thought i was allowed to defend myself . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: you re not allowed to lie .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.78108286857605\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: turn the power of the hunter against him . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: tie a fly . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.667147397994995\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i don t know how to thank you . <---> PREDICTED RESPONSE: i know . you re not . . <---> TARGET: i enjoyed every minute of it .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.771371603012085\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: all you got to do is name it . <---> PREDICTED RESPONSE: i m sorry to hear that . . <---> TARGET: give eve harrington job in you office .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.653848886489868\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: u one u set . everything from midnight on <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: no i think they re all in there .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7016780376434326\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: because you know you ll be killing us . <---> PREDICTED RESPONSE: i m sorry . i m sorry . <---> TARGET: don t lay this on me !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7924644947052\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: a man wouldn t do that . . . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: he isn t a man .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.74231219291687\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: call the code ! i want everyone out ! <---> PREDICTED RESPONSE: you re not gonna get out ! ! <---> TARGET: you gotta give me more to go on\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7866296768188477\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you re twenty two aren t you ray ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: say what and so what .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.982725143432617\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: there s some good things on it though . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: pardon me ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8013830184936523\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i think this is number three . . . <---> PREDICTED RESPONSE: you re not gonna be here ? . <---> TARGET: that s an opinion not evidence . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7286009788513184\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you re going to new york with us . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: when ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:3.0334956645965576\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: do you really want to buy those cigarettes ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: are you serious ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5736303329467773\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: yes sir . you wouldn t believe it . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: you re right .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.782520294189453\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: well what do you want to talk about ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: anything . anything at all .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6272618770599365\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: is there anything else you want to know ? <---> PREDICTED RESPONSE: no . i m not . . <---> TARGET: where you were when your husband died .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6342380046844482\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: harry . . . my god . . . <---> PREDICTED RESPONSE: i m glad you can . . . <---> TARGET: what happened ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.647120237350464\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: hell yeah . she s not my unicorn . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: move over . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.759434700012207\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: has your son ever told you about frank ? <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: come again ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7112436294555664\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: . . .yes i did . . . ? <---> PREDICTED RESPONSE: you know what i m doing . . <---> TARGET: are you learning how to play it ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6593172550201416\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: promise you ll be nice to the neighbors . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: i ll say as little as possible .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.605039596557617\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i just couldn t stand seeing him again . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: running away is no solution .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5933475494384766\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i get it walter . i get it . <---> PREDICTED RESPONSE: you re not gonna get out ? ! <---> TARGET: what do you get ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6661901473999023\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: liz left us red . take the hint . <---> PREDICTED RESPONSE: oh dmitri . . . . <---> TARGET: we don t take hints .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8322911262512207\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: it s still a little early for me . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: so be it .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.817857265472412\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you were the only neighbor i could tolerate . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: i did warn you .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.615877151489258\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: well he s sleeping now . better not . <---> PREDICTED RESPONSE: i know . . . . <---> TARGET: but he s all right ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.645284652709961\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you know what s happened don t you ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: what ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6602766513824463\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: this letter . i ve never opened it . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: why not ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.727707862854004\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: is that the walls of jericho going up ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: yep ! the walls of jericho .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6499249935150146\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: well then you must come back and visit . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: all right .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6696815490722656\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i want you to check this out ellie . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: we already did .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6891369819641113\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: hey pete . . . how ya doin ? <---> PREDICTED RESPONSE: i m glad you re gonna die . . <---> TARGET: who is this ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.520090341567993\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: jake you are never out of your head ! <---> PREDICTED RESPONSE: i m sorry . i m sorry . <---> TARGET: what s in here ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.748615026473999\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i wish you d get an answering machine . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: there s a phone here .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7560055255889893\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: yeah ? what have you been up to ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: i work for city hall .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6421234607696533\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: . actually . i ve been in twice . <---> PREDICTED RESPONSE: what s your name ? . . . <---> TARGET: twice . without her sensing anything ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8077006340026855\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: all right commander you wanna tell us anything ? <---> PREDICTED RESPONSE: no . i m not . . <---> TARGET: like what ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7220497131347656\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: what time s sailor s train get in ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: six .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.711522102355957\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: there s a first time for everything admiral . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: to be sure captain .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7883124351501465\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i don t think she can sing either . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: take care pal .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6747324466705322\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: yes robert . i know about the president . <---> PREDICTED RESPONSE: what ? . . . ? ? <---> TARGET: . . . oh ? you ve heard ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5223569869995117\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: they can deal with this sort of thing . <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: more soldiers ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.793869733810425\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: a bereavement . a death in the family . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: oh i m sorry to hear that .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6470611095428467\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: well you re two up on me now . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: hey you !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5742251873016357\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: who was the girl you walked off with ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: no one .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.640058755874634\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: are we going to go through this again ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: answer the question .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.715442180633545\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: no this is serious . say it backwards . <---> PREDICTED RESPONSE: i m sorry . you re not . <---> TARGET: no !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.722764253616333\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: no . of course not nothing like that . <---> PREDICTED RESPONSE: why ? you re not . ? <---> TARGET: are you sure ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.767812728881836\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i shouldn t drink when i m driving . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: you re so right .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.62459659576416\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: and what are we going to do now ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: we ll try again .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6753063201904297\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: really . do you think it s wrong ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: no .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.885138750076294\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: it s a plain end see ? so ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: take it off .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6587836742401123\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: a bereavement . a death in the family . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: oh i m sorry to hear that .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5465247631073\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i ve seen all that a hundred times . <---> PREDICTED RESPONSE: you re not leaving me ? ? . <---> TARGET: hey will you just leave me alone ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.69893741607666\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: well it s like this mr . kessler . <---> PREDICTED RESPONSE: what ? it s a lot . . <---> TARGET: brian .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5369489192962646\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: well they were right out in plain view . <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: behind the books .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.647768974304199\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: that s great . i didn t know . <---> PREDICTED RESPONSE: what s your name ? . . <---> TARGET: she saved my life .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7020466327667236\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: and you ve got me all figured out ? <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: sure .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.682431936264038\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: yes . money . which is what george ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: freedom .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6610305309295654\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: who else knew i had the baby ? ! <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: no one .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.691621780395508\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: in england i was a poor student but i <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: that was england .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6911919116973877\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: well what have we here ? a family . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: almost a christmas scene .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5504980087280273\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: it ll only work for one of us . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: yes sir .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5575671195983887\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: is that what your girlfriend does for you ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: sure . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.642374038696289\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i was lookin for you all day today . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: you were ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6029229164123535\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: . . .no she didn t . . . <---> PREDICTED RESPONSE: i m sorry . . . . . <---> TARGET: . . .yes she did .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6247215270996094\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: we re takin him . don t argue ! <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: he s my collar !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.618971586227417\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you don t have two million . three million <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: i ll borrow it from you . four million\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.524623155593872\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you can t just stay in your house . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: i know .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6991400718688965\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you know andy thinks you re doing this . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: doesn t that scare you ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6708881855010986\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i didn t mean for you to go ! <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: just radio for help okay ? ? ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.8072662353515625\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you sure that s how you spell it ? <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: yeah .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6390297412872314\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: so i ll see you at the rehearsal . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: yeah .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.632599115371704\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: fine . . . i guess . you ? <---> PREDICTED RESPONSE: yeah . . . . . . . <---> TARGET: good . i think about things .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6055521965026855\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: wait here . i ll be right back . <---> PREDICTED RESPONSE: what ? . . . ? ? <---> TARGET: where would we go ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.660280227661133\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: she came to talk to me . . . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: okay ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.574489116668701\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: well not yet . the gate was open ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: it s taken care of .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6359705924987793\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: what are they interested in . . . ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: wine women and song .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.689624071121216\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: yes . i was on the other side . <---> PREDICTED RESPONSE: i know . you re a rifle . <---> TARGET: you came back because of me ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.622332811355591\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: that s not really a very proper question . <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: how old are you ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5444705486297607\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: geez i forgot how heavy these things are . <---> PREDICTED RESPONSE: what ? . . . ? ! <---> TARGET: okay let s heat em up !\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.6890928745269775\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i take out my slim jim . . . <---> PREDICTED RESPONSE: what ? . . . ? . <---> TARGET: oh god . . .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5788426399230957\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: all right . no . that s nice . <---> PREDICTED RESPONSE: what s your name ? . . <---> TARGET: if there s a train .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.572592258453369\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i m fine hal . how are you ? <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: somebody said she was here .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7997095584869385\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i had to follow her to a store . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: what s wrong with your paisley tie ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.545987367630005\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: no . it had nothing on this afternoon . <---> PREDICTED RESPONSE: what ? ? ? ? ? <---> TARGET: oh you weren t scared .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.700650215148926\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: jeff again . has tom come in yet ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: not yet jeff .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.600351095199585\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: how long do you leave him in there ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: until he s done .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5984132289886475\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: i thought i was allowed to defend myself . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: you re not allowed to lie .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.4936509132385254\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: busy . but you ve got his heart . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: i want an hour with him .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.7349181175231934\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: clear the area . this is an order ! <---> PREDICTED RESPONSE: what ? ! ! ! ! <---> TARGET: what is wrong with you ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.561466932296753\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: who s setting you up in this place ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: brumby .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.4714972972869873\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: no . . . can i come in ? <---> PREDICTED RESPONSE: no . . . . . <---> TARGET: why don t we go to your room\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.570427894592285\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: how do you know you can trust me ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: got to trust someone .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5043978691101074\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: come on jack shall we go ? ! ! <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: apparently so .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.405250310897827\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you mean to tell me we re dead ! <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: i guess so .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.4471049308776855\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: you are one lucky sonofabitch you know that ? <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: i am ?\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.3557450771331787\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: well either you are or you aren t . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: right .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5180139541625977\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: are you nuts ? get out of there ! <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: i m going inside .\n", + "[NeMo I 2020-10-09 17:53:04 :13] Train Loss:2.5084352493286133\n", + "[NeMo I 2020-10-09 17:53:04 :14] SOURCE: no ! not me ! no other people ! <---> PREDICTED RESPONSE: what ? ! ! ! ! <---> TARGET: shut up ! just shut the\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.5081803798675537\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: does it make you . . . hard ? <---> PREDICTED RESPONSE: yeah . . . . . . . <---> TARGET: . . . hard ?\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.4649717807769775\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: good answer . same goes double for me . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: who s your friend ?\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.468278169631958\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: austin may i have a word with you ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: of course luv .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.532140016555786\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: no . . . thanks . . . listen <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: my name is parry .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.260477304458618\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: it s okay . the cops did it . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: i know .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.366523265838623\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: i don t wanna come along with you . <---> PREDICTED RESPONSE: i know . you can . . <---> TARGET: just say so .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.4216229915618896\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: i ve never used the word insane ted . <---> PREDICTED RESPONSE: i m sorry . you re not . <---> TARGET: what word would you use elaine ?\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.433523178100586\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: all right everybody ! that s ten minutes ! <---> PREDICTED RESPONSE: what ? ! ! ! ! <---> TARGET: no ! let me try this !\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.3968920707702637\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: you ve been out for about a day . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: say what ?\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.274991035461426\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: i was thinking of selling all the cards . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: really ? no kidding ?\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.301872491836548\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: yeah but you you got a nice apartment . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: i have a tiny apartment .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.4966228008270264\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: no i won t let you do it . <---> PREDICTED RESPONSE: i know . i m sorry . . <---> TARGET: alex what are you doing ?\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.369938611984253\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: andy stop ! i think he can help you <---> PREDICTED RESPONSE: allo ? ! ! ! ! <---> TARGET: allo ?\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.3272383213043213\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: you can t make me do a thing . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: sit down for a second .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.420529842376709\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: you never could have kept this from me . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: i was leaving .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.451397180557251\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: and you did that to protect my interest ? <---> PREDICTED RESPONSE: no . i m not . . <---> TARGET: in a way yes .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.431372880935669\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: tis not the time to speak of cheer . <---> PREDICTED RESPONSE: i ll be here . . . <---> TARGET: you ll visit again ?\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.238546848297119\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: your voice it s so familiar . . . <---> PREDICTED RESPONSE: what ? . . . . . . <---> TARGET: we have met . . .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.164865493774414\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: i don t want to be just your friend <---> PREDICTED RESPONSE: you know what i m doing . . <---> TARGET: you don t know what you want\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.4855642318725586\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: oh i suppose you is a doctor homer ? <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: almost .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.454925060272217\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: hello hello . it went well i thought . <---> PREDICTED RESPONSE: hello . . . . . <---> TARGET: very impressive ceremony . good speech .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.4613382816314697\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: fire department ? firemen don t carry guns . <---> PREDICTED RESPONSE: i don t know . . . <---> TARGET: oh yeah ? guess again .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.5438313484191895\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: i m sorry . i m so sorry . <---> PREDICTED RESPONSE: you re not leaving me . . . <---> TARGET: oh letty what happened ?\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.753873825073242\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: bobby you re in way over your head . <---> PREDICTED RESPONSE: i m not . . . . <---> TARGET: go to em jerry .\n", + "[NeMo I 2020-10-09 17:53:05 :13] Train Loss:2.6906731128692627\n", + "[NeMo I 2020-10-09 17:53:05 :14] SOURCE: this is sort of high class . . . <---> PREDICTED RESPONSE: i m sorry . . . . <---> TARGET: i mean porno movies .\n" + ] + } + ], + "source": [ + "for batch in range(len(result['rnn_train.inference@torch_tensor'][0])):\n", + " outputs2words([i[batch] for i in result['rnn_train.inference@torch_tensor']], result['rnn_train.eval_data@out_nm'].voc.index2word)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Chatbot model Hypyparameter Tuning\n", + "\n", + "If you haven't followed the `09_gquant_machine_hpo` notebook, it is recommended to go back and check it. It explains in detail how to use gQuant to perform hyperparameter tuning. We will use it to tune the hyperparameters for the chatbot model.\n", + "\n", + "Since the context composite node exposes the `number of layers` and `drop-out rate` as context parameters, the `hpo` `Nemo Hyper Tune Node` will do a grid search of the `number of layers` parameter and random uniform search on the `drop-out` parameter. The eval loss will be used as the hyperparameter tuning metric. \n", + "\n", + "In the following example, we will perform `1` sample hyperparameter tuning. Click on the `run` button to see the hyperparameter tuning in action." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "357d14b76d7b4bffabc17e591dc7af5c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'rnn_train'), ('type', 'ContextCompositeNode'), ('conf', {…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "taskGraph=TaskGraph.load_taskgraph('../taskgraphs/nemo_examples/chatbot_hpo.gq.yaml')\n", + "taskGraph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is satisfying to see all the GPUs in the host node are busy searching for different hyperparameters. The best set of hyperparameters are reported in the end. It can also be fed back to other context composite node to use. \n", + "\n", + "Since we are dealing with deep learning model training, the model is trained in a batched fashion. It is sometimes wasteful to run the training to end to get a metric. Luckily, the `Ray Tune` library provides a few scheduler algorithms that can perform the early stop in the optimization process. The `Nemo Hyper Tune Node` integrates a few scheduler algorithms from the `Ray Tune` lib that are ready to be used. \n", + "\n", + "In the following example, we will use [ASHA](https://docs.ray.io/en/latest/tune/api_docs/schedulers.html#tune-scheduler-hyperband), a scalable algorithm for [principled early stopping](https://blog.ml.cmu.edu/2018/12/12/massively-parallel-hyperparameter-optimization/). On a high level, ASHA terminates trials that are less promising and allocates more time and resources to more promising trials. As our optimization process becomes more efficient, we can afford to increase the search space by 5x, by adjusting the parameter `num_samples`. In the following example, we use `10` as `num_samples`.\n", + "\n", + "Click on the `run` button to see the large scale hyperparameter search in action." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + } + }, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "8a7f8821561c49138578538642158ebb", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "GQuantWidget(sub=HBox(), value=[OrderedDict([('id', 'rnn_train'), ('type', 'ContextCompositeNode'), ('conf', {…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "taskGraph=TaskGraph.load_taskgraph('../taskgraphs/nemo_examples/chatbot_large_hpo_search.gq.yaml')\n", + "taskGraph.draw()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Checking the dynamically updated logs, you can see there are `20` search trails running in total because of the `10` samples and `2` types of grid search. Most of the trials are terminated early at `1` iteration. Only a few are running untill the end. \n", + "\n", + "After tuning is done, the best hyperparameter set is used to train the model again and run the inference. It should give a better model performance." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Clean up" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'status': 'ok', 'restart': True}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import IPython\n", + "app = IPython.Application.instance()\n", + "app.kernel.do_shutdown(True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebooks/images/xgboost/categorical_variable.gif b/notebooks/images/xgboost/categorical_variable.gif new file mode 100644 index 00000000..cb897215 Binary files /dev/null and b/notebooks/images/xgboost/categorical_variable.gif differ diff --git a/notebooks/images/xgboost/create_composite_node.gif b/notebooks/images/xgboost/create_composite_node.gif new file mode 100644 index 00000000..2da1e684 Binary files /dev/null and b/notebooks/images/xgboost/create_composite_node.gif differ diff --git a/notebooks/images/xgboost/create_node.gif b/notebooks/images/xgboost/create_node.gif new file mode 100644 index 00000000..1b55ed0a Binary files /dev/null and b/notebooks/images/xgboost/create_node.gif differ diff --git a/notebooks/images/xgboost/custom_node.gif b/notebooks/images/xgboost/custom_node.gif new file mode 100644 index 00000000..14f4b377 Binary files /dev/null and b/notebooks/images/xgboost/custom_node.gif differ diff --git a/notebooks/images/xgboost/dask_and_sub_graph.gif b/notebooks/images/xgboost/dask_and_sub_graph.gif new file mode 100644 index 00000000..c587a54a Binary files /dev/null and b/notebooks/images/xgboost/dask_and_sub_graph.gif differ diff --git a/notebooks/images/xgboost/distributed_inference.gif b/notebooks/images/xgboost/distributed_inference.gif new file mode 100644 index 00000000..c961610f Binary files /dev/null and b/notebooks/images/xgboost/distributed_inference.gif differ diff --git a/notebooks/images/xgboost/forest_inference.gif b/notebooks/images/xgboost/forest_inference.gif new file mode 100644 index 00000000..267d248e Binary files /dev/null and b/notebooks/images/xgboost/forest_inference.gif differ diff --git a/notebooks/images/xgboost/normalize.gif b/notebooks/images/xgboost/normalize.gif new file mode 100644 index 00000000..71bba5c4 Binary files /dev/null and b/notebooks/images/xgboost/normalize.gif differ diff --git a/notebooks/images/xgboost/prepare_stock_data.gif b/notebooks/images/xgboost/prepare_stock_data.gif new file mode 100644 index 00000000..3a16b881 Binary files /dev/null and b/notebooks/images/xgboost/prepare_stock_data.gif differ diff --git a/notebooks/images/xgboost/split_the_dataset.gif b/notebooks/images/xgboost/split_the_dataset.gif new file mode 100644 index 00000000..a09a9251 Binary files /dev/null and b/notebooks/images/xgboost/split_the_dataset.gif differ diff --git a/notebooks/images/xgboost/train_and_infer.gif b/notebooks/images/xgboost/train_and_infer.gif new file mode 100644 index 00000000..59df919b Binary files /dev/null and b/notebooks/images/xgboost/train_and_infer.gif differ diff --git a/notebooks/images/xgboost/visualize_data.gif b/notebooks/images/xgboost/visualize_data.gif new file mode 100644 index 00000000..361073e6 Binary files /dev/null and b/notebooks/images/xgboost/visualize_data.gif differ diff --git a/notebooks/images/xgboost/xgboost_metrics.gif b/notebooks/images/xgboost/xgboost_metrics.gif new file mode 100644 index 00000000..9ae94e39 Binary files /dev/null and b/notebooks/images/xgboost/xgboost_metrics.gif differ diff --git a/notebooks/images/xgboost/xgboost_stock_data.gif b/notebooks/images/xgboost/xgboost_stock_data.gif new file mode 100644 index 00000000..bcd90f12 Binary files /dev/null and b/notebooks/images/xgboost/xgboost_stock_data.gif differ diff --git a/taskgraphs/nemo_examples/chatbot_example.gq.yaml b/taskgraphs/nemo_examples/chatbot_example.gq.yaml new file mode 100644 index 00000000..2723c581 --- /dev/null +++ b/taskgraphs/nemo_examples/chatbot_example.gq.yaml @@ -0,0 +1,168 @@ +- id: data + type: DialogDataLayerNode + conf: + batch_size: 128 + corpus_name: cornell + datafile: notebooks/movie_data.txt + min_count: 3 + inputs: {} + module: nemo_modules +- id: encoder + type: EncoderRNNNode + conf: + voc_size: 6107 + encoder_n_layers: 2 + hidden_size: 512 + dropout: 0.1 + bidirectional: true + inputs: + input_lengths: data.src_lengths + input_seq: data.src + module: nemo_modules +- id: greedy_decoder + type: GreedyLuongAttnDecoderRNNNode + conf: + attn_model: dot + hidden_size: 512 + voc_size: 6107 + decoder_n_layers: 2 + dropout: 0.1 + max_dec_steps: 10 + inputs: + encoder_outputs: encoder_eval.outputs + in_nm: decoder.out_nm + module: nemo_modules +- id: decoder + type: LuongAttnDecoderRNNNode + conf: + attn_model: dot + hidden_size: 512 + voc_size: 6107 + decoder_n_layers: 2 + dropout: 0.1 + inputs: + targets: data.tgt + encoder_outputs: encoder.outputs + max_target_len: data.max_tgt_lengths + module: nemo_modules +- id: loss + type: MaskedXEntropyLossNode + conf: {} + inputs: + predictions: decoder.outputs + target: data.tgt + mask: data.mask + module: nemo_modules +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: inference.torch_tensor +- id: train + type: NemoTrainNode + conf: + parameters: + tensors_to_optimize: + - loss@loss + batches_per_step: 1 + stop_on_nan_loss: false + synced_batchnorm: false + synced_batchnorm_groupsize: 0 + gradient_predivide: false + amp_max_loss_scale: 16777216 + reset: false + check_point: + load_from_folder: null + checkpoints_to_keep: 4 + force_load: false + folder: nemo_log + simple_logger: + step_freq: 100 + tensors_to_log: + - loss@loss + eval_callback: + eval_step: 100 + eval_tensors: + - eval_loss@loss + warmup_policy: {} + optimizer: + parameters: + num_epochs: 3 + lr: 0.001 + betas: + - 0.9 + - 0.999 + eps: 1e-8 + weight_decay: 0 + amsgrad: false + momentum: 0.9 + name: adam + inputs: + loss@loss: loss.loss + eval_loss@loss: eval_loss.loss +- id: inference + type: NemoInferNode + conf: + tensors: + - eval_loss@loss + - eval_data@src + - greedy_decoder@outputs + - eval_data@tgt + checkpoint_dir: null + ckpt_pattern: "" + verbose: true + cache: false + use_cache: false + offload_to_cpu: true + inputs: + log_dir: train.checkpoint_dir + eval_data@tgt: eval_data.tgt + eval_data@src: eval_data.src + greedy_decoder@outputs: greedy_decoder.outputs + eval_loss@loss: eval_loss.loss +- id: eval_data + type: DialogDataLayerNode + conf: + batch_size: 128 + corpus_name: cornell_eval + datafile: notebooks/movie_data.txt + min_count: 3 + inputs: {} + module: nemo_modules +- id: encoder_eval + type: EncoderRNNNode + conf: + voc_size: 6107 + encoder_n_layers: 2 + hidden_size: 512 + dropout: 0.1 + bidirectional: true + share_weight: Reuse + inputs: + input_seq: eval_data.src + input_lengths: eval_data.src_lengths + in_nm: encoder.out_nm + module: nemo_modules +- id: decoder_eval + type: LuongAttnDecoderRNNNode + conf: + attn_model: dot + hidden_size: 512 + voc_size: 6107 + decoder_n_layers: 2 + dropout: 0.1 + share_weight: Reuse + inputs: + targets: eval_data.tgt + encoder_outputs: encoder_eval.outputs + max_target_len: eval_data.max_tgt_lengths + in_nm: decoder.out_nm + module: nemo_modules +- id: eval_loss + type: MaskedXEntropyLossNode + conf: {} + inputs: + predictions: decoder_eval.outputs + target: eval_data.tgt + mask: eval_data.mask + module: nemo_modules diff --git a/taskgraphs/nemo_examples/chatbot_hpo.gq.yaml b/taskgraphs/nemo_examples/chatbot_hpo.gq.yaml new file mode 100644 index 00000000..9d021199 --- /dev/null +++ b/taskgraphs/nemo_examples/chatbot_hpo.gq.yaml @@ -0,0 +1,87 @@ +- id: rnn_train + type: ContextCompositeNode + conf: + input: [] + output: + - inference.torch_tensor + - eval_data.out_nm + context: + num_layer: + type: number + value: 2 + map: + - node_id: encoder + xpath: encoder.conf.encoder_n_layers + - node_id: decoder + xpath: decoder.conf.decoder_n_layers + - node_id: greedy_decoder + xpath: greedy_decoder.conf.decoder_n_layers + drop_out: + type: number + value: 0.1 + map: + - node_id: encoder + xpath: encoder.conf.dropout + - node_id: decoder + xpath: decoder.conf.dropout + taskgraph: taskgraphs/nemo_examples/chatbot_example.gq.yaml + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: hpo.inference@torch_tensor + in2: hpo.eval_data@out_nm + in3: hpo.conf_out +- id: hpo + type: NemoHyperTuneNode + conf: + parameters: + - search: + function: grid_search + args: + - 1 + - 2 + name: num_layer + - search: + function: uniform + args: + - 0 + - 0.3 + name: drop_out + best: + mode: min + metric: loss + tune: + local_dir: ./ray + name: exp + num_samples: 1 + resources_per_trial: + cpu: 1 + gpu: 1 + input: [] + output: + - inference.torch_tensor + - eval_data.out_nm + context: + num_layer: + type: number + value: 2 + map: + - node_id: encoder + xpath: encoder.conf.encoder_n_layers + - node_id: decoder + xpath: decoder.conf.decoder_n_layers + - node_id: greedy_decoder + xpath: greedy_decoder.conf.decoder_n_layers + drop_out: + type: number + value: 0.1 + map: + - node_id: encoder + xpath: encoder.conf.dropout + - node_id: decoder + xpath: decoder.conf.dropout + taskgraph: taskgraphs/nemo_examples/chatbot_example.gq.yaml + inputs: + conf_in: rnn_train.conf_out diff --git a/taskgraphs/nemo_examples/chatbot_large_hpo_search.gq.yaml b/taskgraphs/nemo_examples/chatbot_large_hpo_search.gq.yaml new file mode 100644 index 00000000..ee256da8 --- /dev/null +++ b/taskgraphs/nemo_examples/chatbot_large_hpo_search.gq.yaml @@ -0,0 +1,114 @@ +- id: rnn_train + type: ContextCompositeNode + conf: + input: [] + output: + - inference.torch_tensor + - eval_data.out_nm + context: + num_layer: + type: number + value: 2 + map: + - node_id: encoder + xpath: encoder.conf.encoder_n_layers + - node_id: decoder + xpath: decoder.conf.decoder_n_layers + - node_id: greedy_decoder + xpath: greedy_decoder.conf.decoder_n_layers + drop_out: + type: number + value: 0.1 + map: + - node_id: encoder + xpath: encoder.conf.dropout + - node_id: decoder + xpath: decoder.conf.dropout + learning_rate: + type: number + value: 0.001 + map: + - node_id: train + xpath: train.conf.optimizer.parameters.lr + taskgraph: taskgraphs/nemo_examples/chatbot_example.gq.yaml + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: hpo.inference@torch_tensor + in2: hpo.eval_data@out_nm + in3: hpo.conf_out +- id: hpo + type: NemoHyperTuneNode + conf: + parameters: + - search: + function: grid_search + args: + - 1 + - 2 + name: num_layer + - search: + function: uniform + args: + - 0 + - 0.3 + name: drop_out + - search: + function: loguniform + args: + - 0.0001 + - 0.01 + - 10 + name: learning_rate + best: + mode: min + metric: loss + tune: + local_dir: ./ray + name: exp + num_samples: 10 + resources_per_trial: + cpu: 1 + gpu: 1 + scheduler: + parameters: + time_attr: training_iteration + max_t: 100 + grace_period: 1 + reduction_factor: 4 + brackets: 1 + name: AsyncHyperBandScheduler + input: [] + output: + - inference.torch_tensor + - eval_data.out_nm + context: + num_layer: + type: number + value: 2 + map: + - node_id: encoder + xpath: encoder.conf.encoder_n_layers + - node_id: decoder + xpath: decoder.conf.decoder_n_layers + - node_id: greedy_decoder + xpath: greedy_decoder.conf.decoder_n_layers + drop_out: + type: number + value: 0.1 + map: + - node_id: encoder + xpath: encoder.conf.dropout + - node_id: decoder + xpath: decoder.conf.dropout + learning_rate: + type: number + value: 0.001 + map: + - node_id: train + xpath: train.conf.optimizer.parameters.lr + taskgraph: taskgraphs/nemo_examples/chatbot_example.gq.yaml + inputs: + conf_in: rnn_train.conf_out diff --git a/taskgraphs/nemo_examples/chatbot_simplified.gq.yaml b/taskgraphs/nemo_examples/chatbot_simplified.gq.yaml new file mode 100644 index 00000000..68630385 --- /dev/null +++ b/taskgraphs/nemo_examples/chatbot_simplified.gq.yaml @@ -0,0 +1,34 @@ +- id: rnn_train + type: ContextCompositeNode + conf: + input: [] + output: + - inference.torch_tensor + - eval_data.out_nm + context: + num_layer: + type: number + value: 2 + map: + - node_id: encoder + xpath: encoder.conf.encoder_n_layers + - node_id: decoder + xpath: decoder.conf.decoder_n_layers + - node_id: greedy_decoder + xpath: greedy_decoder.conf.decoder_n_layers + drop_out: + type: number + value: 0.1 + map: + - node_id: encoder + xpath: encoder.conf.dropout + - node_id: decoder + xpath: decoder.conf.dropout + taskgraph: taskgraphs/nemo_examples/chatbot_example.gq.yaml + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: rnn_train.inference@torch_tensor + in2: rnn_train.eval_data@out_nm diff --git a/taskgraphs/nemo_examples/nemo_train_composite.gq.yaml b/taskgraphs/nemo_examples/nemo_train_composite.gq.yaml new file mode 100644 index 00000000..8a790185 --- /dev/null +++ b/taskgraphs/nemo_examples/nemo_train_composite.gq.yaml @@ -0,0 +1,80 @@ +- id: data + type: RealFunctionDataLayerNode + conf: + batch_size: 128 + f_name: sin + n: 10000 + x_lo: -4 + x_hi: 4 + name: data + inputs: {} + module: nemo_modules +- id: train + type: NemoTrainNode + conf: + parameters: + tensors_to_optimize: + - network@loss@loss + batches_per_step: 1 + stop_on_nan_loss: false + synced_batchnorm: false + synced_batchnorm_groupsize: 0 + gradient_predivide: false + amp_max_loss_scale: 16777216 + reset: false + check_point: + load_from_folder: nemo_log + checkpoints_to_keep: 4 + force_load: false + folder: nemo_log + epoch_freq: 1 + simple_logger: + step_freq: 100 + tensors_to_log: + - network@loss@loss + eval_callback: + ? eval_step + warmup_policy: {} + optimizer: + parameters: + num_epochs: 10 + lr: 0.001 + momentum: 0.9 + weight_decay: 0 + name: sgd + inputs: + network@loss@loss: network.loss@loss +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: train.checkpoint_dir + in2: infer.torch_tensor +- id: infer + type: NemoInferNode + conf: + tensors: + - network@net@y_pred + checkpoint_dir: nemo_log + ckpt_pattern: "" + verbose: true + cache: false + use_cache: false + offload_to_cpu: true + inputs: + log_dir: train.checkpoint_dir + network@net@y_pred: network.net@y_pred +- id: network + type: CompositeNode + conf: + input: + - net.x + - loss.target + output: + - net.y_pred + - loss.loss + subnodes_conf: {} + taskgraph: taskgraphs/nemo_examples/nemo_train_infer.gq.yaml + inputs: + net@x: data.x + loss@target: data.y diff --git a/taskgraphs/nemo_examples/nemo_train_infer.gq.yaml b/taskgraphs/nemo_examples/nemo_train_infer.gq.yaml new file mode 100644 index 00000000..71c5fa36 --- /dev/null +++ b/taskgraphs/nemo_examples/nemo_train_infer.gq.yaml @@ -0,0 +1,81 @@ +- id: data + type: RealFunctionDataLayerNode + conf: + batch_size: 128 + f_name: sin + n: 10000 + x_lo: -4 + x_hi: 4 + name: data + inputs: {} + module: nemo_modules +- id: net + type: TaylorNetNode + conf: + dim: 4 + name: net + inputs: + x: data.x + module: nemo_modules +- id: loss + type: MSELossNode + conf: + name: loss + inputs: + predictions: net.y_pred + target: data.y + module: nemo_modules +- id: train + type: NemoTrainNode + conf: + parameters: + tensors_to_optimize: + - loss@loss + batches_per_step: 1 + stop_on_nan_loss: false + synced_batchnorm: false + synced_batchnorm_groupsize: 0 + gradient_predivide: false + amp_max_loss_scale: 16777216 + reset: false + check_point: + load_from_folder: nemo_log + checkpoints_to_keep: 4 + force_load: false + folder: nemo_log + epoch_freq: 1 + simple_logger: + step_freq: 100 + tensors_to_log: + - loss@loss + warmup_policy: {} + optimizer: + parameters: + num_epochs: 10 + lr: 0.001 + momentum: 0.9 + weight_decay: 0 + name: sgd + inputs: + loss@loss: loss.loss +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: train.checkpoint_dir + in2: loss.loss + in3: infer.torch_tensor +- id: infer + type: NemoInferNode + conf: + tensors: + - net@y_pred + checkpoint_dir: nemo_log + ckpt_pattern: "" + verbose: true + cache: false + use_cache: false + offload_to_cpu: true + inputs: + net@y_pred: net.y_pred + log_dir: train.checkpoint_dir diff --git a/taskgraphs/nemo_examples/simplified.gq.yaml b/taskgraphs/nemo_examples/simplified.gq.yaml new file mode 100644 index 00000000..95b28b6e --- /dev/null +++ b/taskgraphs/nemo_examples/simplified.gq.yaml @@ -0,0 +1,15 @@ +- id: chat + type: CompositeNode + conf: + output: + - inference.torch_tensor + - data.out_nm + subnodes_conf: {} + taskgraph: taskgraphs/nemo_examples/chatbot_example.gq.yaml + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: chat.inference@torch_tensor + in2: chat.data@out_nm diff --git a/taskgraphs/xgboost_example/data_generator.gq.yaml b/taskgraphs/xgboost_example/data_generator.gq.yaml new file mode 100644 index 00000000..64d4c286 --- /dev/null +++ b/taskgraphs/xgboost_example/data_generator.gq.yaml @@ -0,0 +1,68 @@ +- id: data_gen + type: ClassificationData + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.01 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: drop_x2_x3.out +- id: x2_to_sign + type: AddSignIndicatorNode + conf: + sign: x2_sign + column: x2 + inputs: + in: data_gen.cudf_out +- id: x3_to_sign + type: AddSignIndicatorNode + conf: + sign: x3_sign + column: x3 + inputs: + in: x2_to_sign.out +- id: drop_x2_x3 + type: DropNode + conf: + columns: + - x2 + - x3 + - x3_sign + - x2_sign + inputs: + in: one_hot_encoding.out +- id: one_hot_encoding + type: OneHotEncodingNode + conf: + - cats: + - 0 + - 1 + prefix_sep: _ + dtype: float64 + column: x3_sign + prefix: x3 + - cats: + - 0 + - 1 + prefix_sep: _ + dtype: float64 + column: x2_sign + prefix: x2 + inputs: + in: x3_to_sign.out diff --git a/taskgraphs/xgboost_example/hyper_parameters_search.gq.yaml b/taskgraphs/xgboost_example/hyper_parameters_search.gq.yaml new file mode 100644 index 00000000..99c3c605 --- /dev/null +++ b/taskgraphs/xgboost_example/hyper_parameters_search.gq.yaml @@ -0,0 +1,208 @@ +- id: data_gen + type: ClassificationData + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.01 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: xgboost_model.train_roc@roc_curve + in2: xgboost_model.test_roc@roc_curve + in3: hpo.conf_out + in4: hpo.train_roc@roc_curve + in5: hpo.test_roc@roc_curve +- id: data_generator + type: CompositeNode + conf: + input: + - x2_to_sign.in + output: + - drop_x2_x3.out + subnode_ids: + - data_gen + subnodes_conf: + data_gen: + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.1 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + random_state: 10 + taskgraph: taskgraphs/xgboost_example/data_generator.gq.yaml + inputs: + x2_to_sign@in: data_gen.cudf_out +- id: data_splitter + type: DataSplittingNode + conf: + train_size: 0.8 + target: y + inputs: + in: data_generator.drop_x2_x3@out +- id: xgboost_model + type: ContextCompositeNode + conf: + input: + - train_norm.df_in + - test_norm.df_in + output: + - train_infer.out + - test_infer.out + - train_roc.roc_curve + - test_roc.roc_curve + context: + target: + type: string + value: y + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.target + - node_id: train_roc + xpath: train_roc.conf.label + - node_id: test_roc + xpath: test_roc.conf.label + features: + type: array_string + value: + - y + map: + - node_id: train_norm + xpath: train_norm.conf.columns + - node_id: train_xgboost + xpath: train_xgboost.conf.columns + inclusive: + type: boolean + map: + - node_id: train_norm + xpath: train_norm.conf.include + - node_id: train_xgboost + xpath: train_xgboost.conf.include + value: false + depth: + type: number + value: 1 + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.xgboost_parameters.max_depth + eta: + type: number + value: 0.1 + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.xgboost_parameters.eta + subnodes_conf: {} + taskgraph: taskgraphs/xgboost_example/xgboost_model_with_metrics.gq.yaml + inputs: + test_norm@df_in: data_splitter.test + train_norm@df_in: data_splitter.train +- id: hpo + type: GridRandomSearchNode + conf: + parameters: + - search: + function: grid_search + args: + - 1 + - 3 + - 5 + name: depth + - search: + function: uniform + args: + - 0.1 + - 0.8 + name: eta + metrics: + - train_roc.value + - test_roc.value + best: + mode: max + metric: test_roc.value + tune: + local_dir: ./ray + name: exp + num_samples: 1 + resources_per_trial: + cpu: 1 + gpu: 1 + input: + - train_norm.df_in + - test_norm.df_in + output: + - train_infer.out + - test_infer.out + - train_roc.roc_curve + - test_roc.roc_curve + context: + target: + type: string + value: y + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.target + - node_id: train_roc + xpath: train_roc.conf.label + - node_id: test_roc + xpath: test_roc.conf.label + features: + type: array_string + value: + - y + map: + - node_id: train_norm + xpath: train_norm.conf.columns + - node_id: train_xgboost + xpath: train_xgboost.conf.columns + inclusive: + type: boolean + map: + - node_id: train_norm + xpath: train_norm.conf.include + - node_id: train_xgboost + xpath: train_xgboost.conf.include + value: false + depth: + type: number + value: 3 + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.xgboost_parameters.max_depth + eta: + type: number + value: 0.2719547419145216 + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.xgboost_parameters.eta + subnodes_conf: {} + taskgraph: taskgraphs/xgboost_example/xgboost_model_with_metrics.gq.yaml + inputs: + conf_in: xgboost_model.conf_out + train_norm@df_in: data_splitter.train + test_norm@df_in: data_splitter.test diff --git a/taskgraphs/xgboost_example/metrics.gq.yaml b/taskgraphs/xgboost_example/metrics.gq.yaml new file mode 100644 index 00000000..3aff569b --- /dev/null +++ b/taskgraphs/xgboost_example/metrics.gq.yaml @@ -0,0 +1,101 @@ +- id: data_gen + type: ClassificationData + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.01 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: train_roc.roc_curve + in2: test_roc.roc_curve + in3: feature_importance.importance_curve +- id: data_generator + type: CompositeNode + conf: + input: + - x2_to_sign.in + output: + - drop_x2_x3.out + subnode_ids: + - data_gen + subnodes_conf: + data_gen: + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.1 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + random_state: 10 + taskgraph: taskgraphs/xgboost_example/data_generator.gq.yaml + inputs: + x2_to_sign@in: data_gen.cudf_out +- id: xgboost_model + type: CompositeNode + conf: + input: + - train_norm.df_in + - test_norm.df_in + output: + - train_infer.out + - test_infer.out + - train_xgboost.model_out + subnodes_conf: {} + taskgraph: taskgraphs/xgboost_example/xgboost_model.gq.yaml + inputs: + test_norm@df_in: data_splitter.test + train_norm@df_in: data_splitter.train +- id: data_splitter + type: DataSplittingNode + conf: + train_size: 0.8 + target: y + inputs: + in: data_generator.drop_x2_x3@out +- id: train_roc + type: RocCurveNode + conf: + label: y + prediction: predict + inputs: + in: xgboost_model.train_infer@out +- id: test_roc + type: RocCurveNode + conf: + label: y + prediction: predict + inputs: + in: xgboost_model.test_infer@out +- id: feature_importance + type: ImportanceCurveNode + conf: + type: gain + inputs: + in: xgboost_model.train_xgboost@model_out diff --git a/taskgraphs/xgboost_example/ml_preprocess.gq.yaml b/taskgraphs/xgboost_example/ml_preprocess.gq.yaml new file mode 100644 index 00000000..1923ca0c --- /dev/null +++ b/taskgraphs/xgboost_example/ml_preprocess.gq.yaml @@ -0,0 +1,61 @@ +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: train_norm.df_out + in2: test_norm.df_out +- id: data_generator + type: CompositeNode + conf: + output: + - drop_x2_x3.out + subnode_ids: + - data_gen + subnodes_conf: + data_gen: + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.1 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + random_state: 10 + taskgraph: taskgraphs/xgboost_example/data_generator.gq.yaml + inputs: {} +- id: data_splitter + type: DataSplittingNode + conf: + train_size: 0.8 + target: y + inputs: + in: data_generator.drop_x2_x3@out +- id: train_norm + type: NormalizationNode + conf: + columns: + - x3_0 + - x3_1 + - x2_0 + - x2_1 + - y + include: false + inputs: + df_in: data_splitter.train +- id: test_norm + type: NormalizationNode + conf: + include: true + inputs: + norm_data_in: train_norm.norm_data_out + df_in: data_splitter.test diff --git a/taskgraphs/xgboost_example/stock_data.gq.yaml b/taskgraphs/xgboost_example/stock_data.gq.yaml new file mode 100644 index 00000000..1b5288a6 --- /dev/null +++ b/taskgraphs/xgboost_example/stock_data.gq.yaml @@ -0,0 +1,226 @@ +- id: stock_data + type: CsvStockLoader + conf: + file: notebooks/data/stock_price_hist.csv.gz + path: notebooks/many-small + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: drop_col.out +- id: stock_feature + type: CompositeNode + conf: + input: + - preprocess.sort_node@in + output: + - technical_indicator.stock_out + subnode_ids: + - technical_indicator + subnodes_conf: + technical_indicator: + conf: + indicators: + - function: port_bollinger_bands + args: + - 10 + columns: + - close + - function: port_chaikin_oscillator + args: + - 2 + - 3 + columns: + - high + - low + - close + - volume + - function: port_macd + args: + - 2 + - 3 + columns: + - close + - function: port_relative_strength_index + args: + - 2 + columns: + - high + - low + - function: port_average_true_range + args: + - 2 + columns: + - high + - low + - close + - function: port_stochastic_oscillator_k + args: + - 2 + columns: + - high + - low + - close + - function: port_stochastic_oscillator_d + args: + - 2 + columns: + - high + - low + - close + - function: port_money_flow_index + args: + - 2 + columns: + - high + - low + - close + - volume + - function: port_force_index + args: + - 2 + columns: + - close + - volume + - function: port_ultimate_oscillator + args: + - 2 + columns: + - high + - low + - close + - function: port_accumulation_distribution + args: + - 2 + columns: + - high + - low + - close + - volume + - function: port_commodity_channel_index + args: + - 2 + columns: + - high + - low + - close + - function: port_on_balance_volume + args: + - 2 + columns: + - close + - volume + - function: port_vortex_indicator + args: + - 2 + columns: + - high + - low + - close + - function: port_kst_oscillator + args: + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + columns: + - close + - function: port_mass_index + args: + - 2 + - 3 + columns: + - high + - low + - function: port_true_strength_index + args: + - 2 + - 3 + columns: + - close + - function: port_ease_of_movement + args: + - 2 + columns: + - high + - low + - volume + - function: port_coppock_curve + args: + - 2 + columns: + - close + - function: port_keltner_channel + args: + - 2 + columns: + - high + - low + - close + - function: port_ppsr + args: + - 2 + columns: + - high + - low + - close + - function: port_fractional_diff + args: + - 0.9 + columns: + - close + - function: port_fractional_diff + args: + - 0.7 + columns: + - close + - function: port_fractional_diff + args: + - 0.5 + columns: + - close + - function: port_fractional_diff + args: + - 0.3 + columns: + - close + - function: port_fractional_diff + args: + - 0.1 + columns: + - close + - function: port_shift + args: + - -1 + columns: + - returns + remove_na: true + taskgraph: taskgraphs/xgboost_trade.gq.yaml + inputs: + preprocess@sort_node@in: stock_data.cudf_out +- id: pos_neg_return + type: AddSignIndicatorNode + conf: + sign: sign + column: SHIFT_-1 + inputs: + in: stock_feature.technical_indicator@stock_out +- id: drop_col + type: DropNode + conf: + columns: + - indicator + - datetime + - asset + - SHIFT_-1 + - open + - high + - low + - close + inputs: + in: pos_neg_return.out diff --git a/taskgraphs/xgboost_example/tree_inference.gq.yaml b/taskgraphs/xgboost_example/tree_inference.gq.yaml new file mode 100644 index 00000000..45755c93 --- /dev/null +++ b/taskgraphs/xgboost_example/tree_inference.gq.yaml @@ -0,0 +1,117 @@ +- id: data_gen + type: ClassificationData + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.01 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: model_export.filename + in2: test_data.out + in3: test_infer.out + in4: xgboost_infer.out +- id: data_generator + type: CompositeNode + conf: + input: + - x2_to_sign.in + output: + - drop_x2_x3.out + subnode_ids: + - data_gen + subnodes_conf: + data_gen: + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.1 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + random_state: 10 + taskgraph: taskgraphs/xgboost_example/data_generator.gq.yaml + inputs: + x2_to_sign@in: data_gen.dask_cudf_out +- id: xgboost_model + type: CompositeNode + conf: + input: + - train_norm.df_in + - test_norm.df_in + output: + - train_infer.out + - test_infer.out + - train_xgboost.model_out + - train_norm.df_out + - test_norm.df_out + subnode_ids: [] + subnodes_conf: {} + taskgraph: taskgraphs/xgboost_example/xgboost_model.gq.yaml + inputs: + test_norm@df_in: data_splitter.test + train_norm@df_in: data_splitter.train +- id: data_splitter + type: DataSplittingNode + conf: + train_size: 0.8 + target: y + inputs: + in: data_generator.drop_x2_x3@out +- id: model_export + type: XGBoostExportNode + conf: + path: xgboost_model_file + inputs: + model_in: xgboost_model.train_xgboost@model_out +- id: tree_inference + type: ForestInferenceNode + conf: + columns: + - y + include: false + prediction: predict + file: xgboost_model_file + inputs: + model_file: model_export.filename + data_in: xgboost_model.test_norm@df_out +- id: test_data + type: DaskComputeNode + conf: {} + inputs: + in: xgboost_model.test_norm@df_out +- id: test_infer + type: DaskComputeNode + conf: {} + inputs: + in: tree_inference.out +- id: xgboost_infer + type: DaskComputeNode + conf: {} + inputs: + in: xgboost_model.test_infer@out diff --git a/taskgraphs/xgboost_example/xgboost_model.gq.yaml b/taskgraphs/xgboost_example/xgboost_model.gq.yaml new file mode 100644 index 00000000..fe7517d3 --- /dev/null +++ b/taskgraphs/xgboost_example/xgboost_model.gq.yaml @@ -0,0 +1,126 @@ +- id: data_gen + type: ClassificationData + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.01 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: test_infer.out + in2: train_infer.out +- id: data_generator + type: CompositeNode + conf: + input: + - x2_to_sign.in + output: + - drop_x2_x3.out + subnode_ids: + - data_gen + subnodes_conf: + data_gen: + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.1 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + random_state: 10 + taskgraph: taskgraphs/xgboost_example/data_generator.gq.yaml + inputs: + x2_to_sign@in: data_gen.cudf_out +- id: data_splitter + type: DataSplittingNode + conf: + train_size: 0.8 + target: y + inputs: + in: data_generator.drop_x2_x3@out +- id: train_norm + type: NormalizationNode + conf: + columns: + - x3_0 + - x3_1 + - x2_0 + - x2_1 + - y + include: false + inputs: + df_in: data_splitter.train +- id: test_norm + type: NormalizationNode + conf: + include: true + inputs: + norm_data_in: train_norm.norm_data_out + df_in: data_splitter.test +- id: train_xgboost + type: TrainXGBoostNode + conf: + num_of_rounds: 100 + columns: + - y + include: false + xgboost_parameters: + eta: 0.3 + min_child_weight: 1 + subsample: 1 + sampling_method: uniform + colsample_bytree: 1 + colsample_bylevel: 1 + colsample_bynode: 1 + max_depth: 8 + max_leaves: 256 + grow_policy: depthwise + gamma: 0 + lambda: 1 + alpha: 0 + tree_method: gpu_hist + single_precision_histogram: false + deterministic_histogram: false + objective: binary:logistic + target: y + inputs: + in: train_norm.df_out +- id: train_infer + type: InferXGBoostNode + conf: + prediction: predict + inputs: + data_in: train_norm.df_out + model_in: train_xgboost.model_out +- id: test_infer + type: InferXGBoostNode + conf: + prediction: predict + inputs: + data_in: test_norm.df_out + model_in: train_xgboost.model_out diff --git a/taskgraphs/xgboost_example/xgboost_model_with_metrics.gq.yaml b/taskgraphs/xgboost_example/xgboost_model_with_metrics.gq.yaml new file mode 100644 index 00000000..9955951f --- /dev/null +++ b/taskgraphs/xgboost_example/xgboost_model_with_metrics.gq.yaml @@ -0,0 +1,140 @@ +- id: data_gen + type: ClassificationData + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.01 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: test_roc.roc_curve + in2: train_roc.roc_curve +- id: data_generator + type: CompositeNode + conf: + input: + - x2_to_sign.in + output: + - drop_x2_x3.out + subnode_ids: + - data_gen + subnodes_conf: + data_gen: + conf: + n_samples: 10000 + n_features: 10 + n_informative: 4 + n_redundant: 0 + n_repeated: 0 + n_classes: 2 + n_clusters_per_class: 2 + flip_y: 0.1 + class_sep: 1 + hypercube: true + shift: 0 + scale: 1 + shuffle: true + order: F + dtype: float64 + n_parts: 4 + random_state: 10 + taskgraph: taskgraphs/xgboost_example/data_generator.gq.yaml + inputs: + x2_to_sign@in: data_gen.cudf_out +- id: data_splitter + type: DataSplittingNode + conf: + train_size: 0.8 + target: y + inputs: + in: data_generator.drop_x2_x3@out +- id: train_norm + type: NormalizationNode + conf: + columns: + - x3_0 + - x3_1 + - x2_0 + - x2_1 + - y + include: false + inputs: + df_in: data_splitter.train +- id: test_norm + type: NormalizationNode + conf: + include: true + inputs: + norm_data_in: train_norm.norm_data_out + df_in: data_splitter.test +- id: train_xgboost + type: TrainXGBoostNode + conf: + num_of_rounds: 100 + columns: + - y + include: false + xgboost_parameters: + eta: 0.3 + min_child_weight: 1 + subsample: 1 + sampling_method: uniform + colsample_bytree: 1 + colsample_bylevel: 1 + colsample_bynode: 1 + max_depth: 8 + max_leaves: 256 + grow_policy: depthwise + gamma: 0 + lambda: 1 + alpha: 0 + tree_method: gpu_hist + single_precision_histogram: false + deterministic_histogram: false + objective: binary:logistic + target: y + inputs: + in: train_norm.df_out +- id: train_infer + type: InferXGBoostNode + conf: + prediction: predict + inputs: + data_in: train_norm.df_out + model_in: train_xgboost.model_out +- id: test_infer + type: InferXGBoostNode + conf: + prediction: predict + inputs: + data_in: test_norm.df_out + model_in: train_xgboost.model_out +- id: train_roc + type: RocCurveNode + conf: + label: y + prediction: predict + inputs: + in: train_infer.out +- id: test_roc + type: RocCurveNode + conf: + label: y + prediction: predict + inputs: + in: test_infer.out diff --git a/taskgraphs/xgboost_example/xgboost_stock.gq.yaml b/taskgraphs/xgboost_example/xgboost_stock.gq.yaml new file mode 100644 index 00000000..6fbeb368 --- /dev/null +++ b/taskgraphs/xgboost_example/xgboost_stock.gq.yaml @@ -0,0 +1,293 @@ +- id: stock_data + type: CsvStockLoader + conf: + file: notebooks/data/stock_price_hist.csv.gz + path: notebooks/many-small + inputs: {} +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: test_roc.roc_curve + in2: train_roc.roc_curve + in3: feature_importance.importance_curve + in4: xgboost_model.train_xgboost@model_out +- id: stock_feature + type: CompositeNode + conf: + input: + - preprocess.sort_node@in + output: + - technical_indicator.stock_out + subnode_ids: + - technical_indicator + subnodes_conf: + technical_indicator: + conf: + indicators: + - function: port_bollinger_bands + args: + - 10 + columns: + - close + - function: port_chaikin_oscillator + args: + - 2 + - 3 + columns: + - high + - low + - close + - volume + - function: port_macd + args: + - 2 + - 3 + columns: + - close + - function: port_relative_strength_index + args: + - 2 + columns: + - high + - low + - function: port_average_true_range + args: + - 2 + columns: + - high + - low + - close + - function: port_stochastic_oscillator_k + args: + - 2 + columns: + - high + - low + - close + - function: port_stochastic_oscillator_d + args: + - 2 + columns: + - high + - low + - close + - function: port_money_flow_index + args: + - 2 + columns: + - high + - low + - close + - volume + - function: port_force_index + args: + - 2 + columns: + - close + - volume + - function: port_ultimate_oscillator + args: + - 2 + columns: + - high + - low + - close + - function: port_accumulation_distribution + args: + - 2 + columns: + - high + - low + - close + - volume + - function: port_commodity_channel_index + args: + - 2 + columns: + - high + - low + - close + - function: port_on_balance_volume + args: + - 2 + columns: + - close + - volume + - function: port_vortex_indicator + args: + - 2 + columns: + - high + - low + - close + - function: port_kst_oscillator + args: + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + columns: + - close + - function: port_mass_index + args: + - 2 + - 3 + columns: + - high + - low + - function: port_true_strength_index + args: + - 2 + - 3 + columns: + - close + - function: port_ease_of_movement + args: + - 2 + columns: + - high + - low + - volume + - function: port_coppock_curve + args: + - 2 + columns: + - close + - function: port_keltner_channel + args: + - 2 + columns: + - high + - low + - close + - function: port_ppsr + args: + - 2 + columns: + - high + - low + - close + - function: port_fractional_diff + args: + - 0.9 + columns: + - close + - function: port_fractional_diff + args: + - 0.7 + columns: + - close + - function: port_fractional_diff + args: + - 0.5 + columns: + - close + - function: port_fractional_diff + args: + - 0.3 + columns: + - close + - function: port_fractional_diff + args: + - 0.1 + columns: + - close + - function: port_shift + args: + - -1 + columns: + - returns + remove_na: true + taskgraph: taskgraphs/xgboost_trade.gq.yaml + inputs: + preprocess@sort_node@in: stock_data.dask_cudf_out +- id: pos_neg_return + type: AddSignIndicatorNode + conf: + sign: sign + column: SHIFT_-1 + inputs: + in: stock_feature.technical_indicator@stock_out +- id: drop_col + type: DropNode + conf: + columns: + - indicator + - datetime + - asset + - SHIFT_-1 + - open + - high + - low + - close + inputs: + in: pos_neg_return.out +- id: split_data + type: DataSplittingNode + conf: + train_size: 0.8 + target: sign + inputs: + in: drop_col.out +- id: xgboost_model + type: CustXGBoostNode + conf: + train_norm: + conf: + columns: + - sign + include: false + train_xgboost: + conf: + num_of_rounds: 100 + columns: + - sign + include: false + xgboost_parameters: + eta: 0.3 + min_child_weight: 1 + subsample: 1 + sampling_method: uniform + colsample_bytree: 1 + colsample_bylevel: 1 + colsample_bynode: 1 + max_depth: 8 + max_leaves: 256 + grow_policy: depthwise + gamma: 0 + lambda: 1 + alpha: 0 + tree_method: gpu_hist + single_precision_histogram: false + deterministic_histogram: false + objective: binary:logistic + target: sign + inputs: + test_norm@df_in: split_data.test + train_norm@df_in: split_data.train + module: my_node +- id: train_roc + type: RocCurveNode + conf: + label: sign + prediction: predict + inputs: + in: xgboost_model.train_infer@out +- id: test_roc + type: RocCurveNode + conf: + label: sign + prediction: predict + inputs: + in: xgboost_model.test_infer@out +- id: feature_importance + type: ImportanceCurveNode + conf: + type: gain + inputs: + in: xgboost_model.train_xgboost@model_out diff --git a/taskgraphs/xgboost_example/xgboost_stock_hpo.gq.yaml b/taskgraphs/xgboost_example/xgboost_stock_hpo.gq.yaml new file mode 100644 index 00000000..e0695785 --- /dev/null +++ b/taskgraphs/xgboost_example/xgboost_stock_hpo.gq.yaml @@ -0,0 +1,377 @@ +- id: "" + type: Output_Collector + conf: {} + inputs: + in1: xgboost_model.train_roc@roc_curve + in2: xgboost_model.test_roc@roc_curve + in3: hpo.conf_out + in4: hpo.train_roc@roc_curve + in5: hpo.test_roc@roc_curve +- id: xgboost_model + type: ContextCompositeNode + conf: + input: + - train_norm.df_in + - test_norm.df_in + output: + - train_infer.out + - test_infer.out + - train_roc.roc_curve + - test_roc.roc_curve + context: + target: + type: string + value: sign + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.target + - node_id: train_roc + xpath: train_roc.conf.label + - node_id: test_roc + xpath: test_roc.conf.label + features: + type: array_string + value: + - sign + map: + - node_id: train_norm + xpath: train_norm.conf.columns + - node_id: train_xgboost + xpath: train_xgboost.conf.columns + inclusive: + type: boolean + map: + - node_id: train_norm + xpath: train_norm.conf.include + - node_id: train_xgboost + xpath: train_xgboost.conf.include + value: false + depth: + type: number + value: 1 + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.xgboost_parameters.max_depth + eta: + type: number + value: 0.1 + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.xgboost_parameters.eta + subnodes_conf: {} + taskgraph: taskgraphs/xgboost_example/xgboost_model_with_metrics.gq.yaml + inputs: + train_norm@df_in: split_data.train + test_norm@df_in: split_data.test +- id: hpo + type: GridRandomSearchNode + conf: + parameters: + - search: + function: grid_search + args: + - 1 + - 3 + - 5 + name: depth + - search: + function: uniform + args: + - 0.1 + - 0.8 + name: eta + metrics: + - train_roc.value + - test_roc.value + best: + mode: max + metric: test_roc.value + tune: + local_dir: ./ray + name: stock + num_samples: 1 + resources_per_trial: + cpu: 1 + gpu: 1 + input: + - train_norm.df_in + - test_norm.df_in + output: + - train_infer.out + - test_infer.out + - train_roc.roc_curve + - test_roc.roc_curve + context: + target: + type: string + value: sign + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.target + - node_id: train_roc + xpath: train_roc.conf.label + - node_id: test_roc + xpath: test_roc.conf.label + features: + type: array_string + value: + - sign + map: + - node_id: train_norm + xpath: train_norm.conf.columns + - node_id: train_xgboost + xpath: train_xgboost.conf.columns + inclusive: + type: boolean + map: + - node_id: train_norm + xpath: train_norm.conf.include + - node_id: train_xgboost + xpath: train_xgboost.conf.include + value: false + depth: + type: number + value: 1 + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.xgboost_parameters.max_depth + eta: + type: number + value: 0.1 + map: + - node_id: train_xgboost + xpath: train_xgboost.conf.xgboost_parameters.eta + subnodes_conf: {} + taskgraph: taskgraphs/xgboost_example/xgboost_model_with_metrics.gq.yaml + inputs: + conf_in: xgboost_model.conf_out + train_norm@df_in: split_data.train + test_norm@df_in: split_data.test +- id: stock_data + type: CsvStockLoader + conf: + file: notebooks/data/stock_price_hist.csv.gz + path: notebooks/many-small + inputs: {} +- id: stock_feature + type: CompositeNode + conf: + input: + - preprocess.sort_node@in + output: + - technical_indicator.stock_out + subnode_ids: + - technical_indicator + subnodes_conf: + technical_indicator: + conf: + indicators: + - function: port_bollinger_bands + args: + - 10 + columns: + - close + - function: port_chaikin_oscillator + args: + - 2 + - 3 + columns: + - high + - low + - close + - volume + - function: port_macd + args: + - 2 + - 3 + columns: + - close + - function: port_relative_strength_index + args: + - 2 + columns: + - high + - low + - function: port_average_true_range + args: + - 2 + columns: + - high + - low + - close + - function: port_stochastic_oscillator_k + args: + - 2 + columns: + - high + - low + - close + - function: port_stochastic_oscillator_d + args: + - 2 + columns: + - high + - low + - close + - function: port_money_flow_index + args: + - 2 + columns: + - high + - low + - close + - volume + - function: port_force_index + args: + - 2 + columns: + - close + - volume + - function: port_ultimate_oscillator + args: + - 2 + columns: + - high + - low + - close + - function: port_accumulation_distribution + args: + - 2 + columns: + - high + - low + - close + - volume + - function: port_commodity_channel_index + args: + - 2 + columns: + - high + - low + - close + - function: port_on_balance_volume + args: + - 2 + columns: + - close + - volume + - function: port_vortex_indicator + args: + - 2 + columns: + - high + - low + - close + - function: port_kst_oscillator + args: + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + columns: + - close + - function: port_mass_index + args: + - 2 + - 3 + columns: + - high + - low + - function: port_true_strength_index + args: + - 2 + - 3 + columns: + - close + - function: port_ease_of_movement + args: + - 2 + columns: + - high + - low + - volume + - function: port_coppock_curve + args: + - 2 + columns: + - close + - function: port_keltner_channel + args: + - 2 + columns: + - high + - low + - close + - function: port_ppsr + args: + - 2 + columns: + - high + - low + - close + - function: port_fractional_diff + args: + - 0.9 + columns: + - close + - function: port_fractional_diff + args: + - 0.7 + columns: + - close + - function: port_fractional_diff + args: + - 0.5 + columns: + - close + - function: port_fractional_diff + args: + - 0.3 + columns: + - close + - function: port_fractional_diff + args: + - 0.1 + columns: + - close + - function: port_shift + args: + - -1 + columns: + - returns + remove_na: true + taskgraph: taskgraphs/xgboost_trade.gq.yaml + inputs: + preprocess@sort_node@in: stock_data.cudf_out +- id: pos_neg_return + type: AddSignIndicatorNode + conf: + sign: sign + column: SHIFT_-1 + inputs: + in: stock_feature.technical_indicator@stock_out +- id: drop_col + type: DropNode + conf: + columns: + - indicator + - datetime + - asset + - SHIFT_-1 + - open + - high + - low + - close + inputs: + in: pos_neg_return.out +- id: split_data + type: DataSplittingNode + conf: + train_size: 0.8 + target: sign + inputs: + in: drop_col.out diff --git a/util/auto_gen.py b/util/auto_gen.py new file mode 100644 index 00000000..acf4afa5 --- /dev/null +++ b/util/auto_gen.py @@ -0,0 +1,105 @@ +import sys +import inspect +from pathlib import Path + +file_ = Path(__file__) +modulespath = '{}/modules'.format(file_.resolve().parents[1]) +sys.path.insert(1, modulespath) + +from nemo.backends.pytorch.nm import NeuralModule +from nemo_gquant_modules.nemoBaseNode import FeedProperty + +TEMPLATE = """from gquant.dataframe_flow import Node +from .nemoBaseNode import NeMoBase +import nemo +import {} +""" + +CLASS_TEMP = """ + + +class {}(NeMoBase, Node): + def init(self): + NeMoBase.init(self, {}) +""" + + +def gen_module_file(module, overwrite=None): + file_str = TEMPLATE.format(module.__name__) + + nodecls_list = [] + + for item in inspect.getmembers(module): + if inspect.ismodule(item[1]): + if item[1].__package__.startswith('nemo'): + for node in inspect.getmembers(item[1]): + if inspect.isclass(node[1]): + nodecls = node[1] + if nodecls in nodecls_list: + continue + + if issubclass(nodecls, NeuralModule): + if nodecls.__module__ == 'nemo.backends.pytorch.nm': + continue + try: + # p_inports = node[1].input_ports + # p_outports = node[1].output_ports + # feeder = FeedProperty({}) + # inports = p_inports.fget(feeder) + # outports = p_outports.fget(feeder) + + init_fun = node[1].__init__ + sig = inspect.signature(init_fun) + skip = False + for key in sig.parameters.keys(): + if key == 'self': + # ignore the self + continue + para = sig.parameters[key] + if para.default != inspect._empty: + if para.default.__class__.__name__ == 'type' or para.default.__class__.__name__ == 'DataCombination': + print(para.default, para) + skip = True + break + if skip: + print(node[0], 'find class arg', para.default.__class__.__name__) + continue + + class_name = node[1].__module__ + '.' + node[1].__name__ + file_str += CLASS_TEMP.format(node[0] + "Node", + class_name) + nodecls_list.append(nodecls) + except Exception as e: + print(e) + print(node[0], 'is not compatible, as it uses instance for input/output ports') + continue + + if overwrite is not None: + module_name = overwrite + else: + module_name = module.__name__.split('.')[-1] + with open('../modules/nemo_gquant_modules/' + module_name + '.py', 'w') as f: + f.write(file_str) + + +import nemo.backends.pytorch.tutorials +gen_module_file(nemo.backends.pytorch.tutorials) + +import nemo.backends.pytorch.common +gen_module_file(nemo.backends.pytorch.common) + +import nemo.collections.asr +gen_module_file(nemo.collections.asr) + +import nemo.collections.cv +gen_module_file(nemo.collections.cv) + +import nemo.collections.nlp.nm +gen_module_file(nemo.collections.nlp.nm, 'nlp') + +import nemo.collections.simple_gan +gen_module_file(nemo.collections.simple_gan) + +import nemo.collections.tts +gen_module_file(nemo.collections.tts) +