Merge of PR #1645

PiperOrigin-RevId: 260743903

tensor2tensor/layers/common_attention.py

@@ -619,7 +619,7 @@ def add_timing_signal_nd(x, min_timescale=1.0, max_timescale=1.0e4):
   memory inputs to attention.
 
   The use of relative position is possible because sin(a+b) and cos(a+b) can be
-  expressed in terms of b, sin(a) and cos(a).
+  experessed in terms of b, sin(a) and cos(a).
 
   x is a Tensor with n "positional" dimensions, e.g. one dimension for a
   sequence or two dimensions for an image

tensor2tensor/utils/optimize.py

@@ -18,7 +18,6 @@
 from __future__ import division
 from __future__ import print_function
 import numpy as np
-import os
 
 from tensor2tensor.layers import common_layers
 from tensor2tensor.utils import adafactor as adafactor_lib
@@ -41,7 +40,7 @@ def _mixed_precision_is_enabled(hparams):
   return activation_dtype == tf.float16 and weight_dtype == tf.float32
 
 
-def optimize(loss, learning_rate, hparams, use_tpu=False, variables=None, gpu_auto_mixed_precision=False):
+def optimize(loss, learning_rate, hparams, use_tpu=False, variables=None):
   """Minimize loss."""
   loss = weight_decay_and_noise(loss, hparams, learning_rate)
   loss = tf.identity(loss, name="total_loss")
@@ -66,16 +65,6 @@ def optimize(loss, learning_rate, hparams, use_tpu=False, variables=None, gpu_au
   opt = ConditionalOptimizer(hparams.optimizer, learning_rate, hparams, use_tpu)
   if use_tpu:
     opt = tf.contrib.tpu.CrossShardOptimizer(opt)
-  if os.environ.get('TF_ENABLE_AUTO_MIXED_PRECISION', default='0') == '1' or gpu_auto_mixed_precision:
-      if use_tpu:
-          raise(RuntimeError("GPU auto mixed precision cannot be used with TPU"))
-      elif _mixed_precision_is_enabled(hparams):
-          raise(RuntimeError("GPU auto mixed precision cannot be used with manual mixed precision"))
-      else:
-          setattr(opt, '_use_locking', 'True')
-          setattr(opt, '_name', 'ConditionalOptimizer')
-          opt = tf.train.experimental.enable_mixed_precision_graph_rewrite(opt)
-
   opt_summaries = []
   if common_layers.should_generate_summaries():
     tf.summary.scalar("learning_rate", learning_rate)

tensor2tensor/visualization/visualization.py

@@ -58,17 +58,17 @@ def encode(self, input_str):
   def decode(self, integers):
     """List of ints to str."""
     integers = list(np.squeeze(integers))
-    return self.encoders['targets'].decode(integers)
+    return self.encoders["targets"].decode(integers)
 
   def encode_list(self, integers):
     """List of ints to list of str."""
     integers = list(np.squeeze(integers))
-    return self.encoders['inputs'].decode_list(integers)
+    return self.encoders["inputs"].decode_list(integers)
 
   def decode_list(self, integers):
     """List of ints to list of str."""
     integers = list(np.squeeze(integers))
-    return self.encoders['targets'].decode_list(integers)
+    return self.encoders["targets"].decode_list(integers)
 
   def get_vis_data_from_string(self, sess, input_string):
     """Constructs the data needed for visualizing attentions.