Merge pull request #1 from tensorflow/master

update from original

Author: oderdene

.gitignore

@@ -90,3 +90,8 @@ ENV/
 
 # PyCharm
 .idea/
+
+# For mac
+.DS_Store
+
+samples/outreach/blogs/segmentation_blogpost/carvana-image-masking-challenge/

CODEOWNERS

@@ -6,10 +6,12 @@
 /research/adv_imagenet_models/ @AlexeyKurakin
 /research/attention_ocr/ @alexgorban
 /research/audioset/ @plakal @dpwe
+/research/autoaugment/* @barretzoph
 /research/autoencoders/ @snurkabill
 /research/brain_coder/ @danabo
 /research/cognitive_mapping_and_planning/ @s-gupta
 /research/compression/ @nmjohn
+/research/cvt_text/ @clarkkev @lmthang
 /research/deep_contextual_bandits/ @rikel
 /research/deeplab/ @aquariusjay @yknzhu @gpapan
 /research/delf/ @andrefaraujo
@@ -28,6 +30,7 @@
 /research/lfads/ @jazcollins @susillo
 /research/lm_1b/ @oriolvinyals @panyx0718
 /research/lm_commonsense/ @thtrieu
+/research/lstm_object_detection/ @dreamdragon @masonliuw @yinxiaoli
 /research/marco/ @vincentvanhoucke
 /research/maskgan/ @a-dai
 /research/morph_net/ @gariel-google
@@ -46,6 +49,7 @@
 /research/slim/ @sguada @nathansilberman
 /research/steve/ @buckman-google
 /research/street/ @theraysmith
+/research/struct2depth/ @aneliaangelova
 /research/swivel/ @waterson
 /research/syntaxnet/ @calberti @andorardo @bogatyy @markomernick
 /research/tcn/ @coreylynch @sermanet

official/README.md

@@ -14,11 +14,21 @@ If you are on a version of TensorFlow earlier than 1.4, please [update your inst
 ## Requirements
 Please follow the below steps before running models in this repo:
 
-1. Add the top-level ***/models*** folder to the Python path with the command:
+
+1. TensorFlow [nightly binaries](https://github.com/tensorflow/tensorflow#installation)
+
+2. Add the top-level ***/models*** folder to the Python path with the command:
    ```
    export PYTHONPATH="$PYTHONPATH:/path/to/models"
    ```
-2. Install dependencies:
+   
+   Using Colab:
+   ```
+   import os
+   os.environ['PYTHONPATH'] += ":/path/to/models"
+   ```
+
+3. Install dependencies:
    ```
    pip3 install --user -r official/requirements.txt
    ```
@@ -54,7 +64,7 @@ our objectives of readable, usable, and maintainable code.
 * Runnable from a blank environment with relative ease.
 * Trainable on: single GPU/CPU (baseline), multiple GPUs, TPU
 * Compatible with Python 2 and 3 (using [six](https://pythonhosted.org/six/) when necessary)
-* Conform to [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html)
+* Conform to [Google Python Style Guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md)
 
 **Implementation guidelines**
 

official/boosted_trees/data_download.py

@@ -47,7 +47,7 @@
 
 def _download_higgs_data_and_save_npz(data_dir):
   """Download higgs data and store as a numpy compressed file."""
-  input_url = os.path.join(URL_ROOT, INPUT_FILE)
+  input_url = URL_ROOT + "/" + INPUT_FILE
   np_filename = os.path.join(data_dir, NPZ_FILE)
   if tf.gfile.Exists(np_filename):
     raise ValueError("data_dir already has the processed data file: {}".format(

official/boosted_trees/train_higgs.py

@@ -249,7 +249,8 @@ def train_boosted_trees(flags_obj):
         _make_csv_serving_input_receiver_fn(
             column_names=feature_names,
             # columns are all floats.
-            column_defaults=[[0.0]] * len(feature_names)))
+            column_defaults=[[0.0]] * len(feature_names)),
+        strip_default_attrs=True)
 
 
 def main(_):

official/datasets/movielens.py

@@ -72,7 +72,7 @@
     ML_20M: 138493,
 }
 
-# Note: Users are indexed [1, k], not [0, k-1]
+# Note: Movies are indexed [1, k], not [0, k-1]
 # Both the 1m and 20m datasets use the same movie set.
 NUM_ITEM_IDS = 3952
 
@@ -111,12 +111,7 @@ def _download_and_clean(dataset, data_dir):
   temp_dir = tempfile.mkdtemp()
   try:
     zip_path = os.path.join(temp_dir, "{}.zip".format(dataset))
-    def _progress(count, block_size, total_size):
-      sys.stdout.write("\r>> Downloading {} {:.1f}%".format(
-          zip_path, 100.0 * count * block_size / total_size))
-      sys.stdout.flush()
-
-    zip_path, _ = urllib.request.urlretrieve(url, zip_path, _progress)
+    zip_path, _ = urllib.request.urlretrieve(url, zip_path)
     statinfo = os.stat(zip_path)
     # A new line to clear the carriage return from download progress
     # tf.logging.info is not applicable here
@@ -133,8 +128,12 @@ def _progress(count, block_size, total_size):
       _regularize_20m_dataset(temp_dir)
 
     for fname in tf.gfile.ListDirectory(temp_dir):
-      tf.gfile.Copy(os.path.join(temp_dir, fname),
-                    os.path.join(data_subdir, fname))
+      if not tf.gfile.Exists(os.path.join(data_subdir, fname)):
+        tf.gfile.Copy(os.path.join(temp_dir, fname),
+                      os.path.join(data_subdir, fname))
+      else:
+        tf.logging.info("Skipping copy of {}, as it already exists in the "
+                        "destination folder.".format(fname))
 
   finally:
     tf.gfile.DeleteRecursively(temp_dir)

official/keras_application_models/README.md

@@ -19,10 +19,19 @@ Synthetic dataset is used for the benchmark.
 Two custom callbacks are provided for model benchmarking: ExamplesPerSecondCallback and LoggingMetricCallback. For each callback, `epoch_based` and `batch_based` options are available to set the benchmark level. Check [model_callbacks.py](model_callbacks.py) for more details.
 
 ## Running Code
-To benchmark a model, use `--model` to specify the model name, and issue the following command:
+To benchmark a model, use `--model` to specify the model name. To perform the benchmark with eager execution, issue the following command:
 ```
-python benchmark_main.py --model=resnet
+python benchmark_main.py --model resnet50 --eager
 ```
+Note that, if eager execution is enabled, only one GPU is utilized even if multiple GPUs are provided and multi_gpu_model is used.
+
+
+To use distribution strategy in the benchmark, run the following:
+```
+python benchmark_main.py --model resnet50 --dist_strat
+```
+Currently, only one of the --eager and --dist_strat arguments can be defined, as DistributionStrategy is not supported in Eager execution now.
+
 Arguments:
   * `--model`: Which model to be benchmarked. The model name is defined as the keys of `MODELS` in [benchmark_main.py](benchmark_main.py).
   * `--callbacks`: To specify a list of callbacks.

official/keras_application_models/benchmark_main.py

@@ -28,6 +28,7 @@
 from official.keras_application_models import model_callbacks
 from official.utils.flags import core as flags_core
 from official.utils.logs import logger
+from official.utils.misc import distribution_utils
 
 # Define a dictionary that maps model names to their model classes inside Keras
 MODELS = {
@@ -41,9 +42,8 @@
     "densenet121": tf.keras.applications.DenseNet121,
     "densenet169": tf.keras.applications.DenseNet169,
     "densenet201": tf.keras.applications.DenseNet201,
-    # TODO(b/80431378)
-    # "nasnetlarge": tf.keras.applications.NASNetLarge,
-    # "nasnetmobile": tf.keras.applications.NASNetMobile,
+    "nasnetlarge": tf.keras.applications.NASNetLarge,
+    "nasnetmobile": tf.keras.applications.NASNetMobile,
 }
 
 
@@ -62,7 +62,6 @@ def run_keras_model_benchmark(_):
   # Load the model
   tf.logging.info("Benchmark on {} model...".format(FLAGS.model))
   keras_model = MODELS[FLAGS.model]
-  model = keras_model(weights=None)
 
   # Get dataset
   dataset_name = "ImageNet"
@@ -73,31 +72,49 @@ def run_keras_model_benchmark(_):
         FLAGS.model, FLAGS.batch_size)
     val_dataset = dataset.generate_synthetic_input_dataset(
         FLAGS.model, FLAGS.batch_size)
+    model = keras_model(weights=None)
   else:
-    raise ValueError("Only synthetic dataset is supported!")
+    tf.logging.info("Using CIFAR-10 dataset...")
+    dataset_name = "CIFAR-10"
+    ds = dataset.Cifar10Dataset(FLAGS.batch_size)
+    train_dataset = ds.train_dataset
+    val_dataset = ds.test_dataset
+    model = keras_model(
+        weights=None, input_shape=ds.input_shape, classes=ds.num_classes)
 
-  # If run with multiple GPUs
-  # If eager execution is enabled, only one GPU is utilized even if multiple
-  # GPUs are provided.
   num_gpus = flags_core.get_num_gpus(FLAGS)
-  if num_gpus > 1:
+
+  distribution = None
+  # Use distribution strategy
+  if FLAGS.dist_strat:
+    distribution = distribution_utils.get_distribution_strategy(
+        num_gpus=num_gpus)
+  elif num_gpus > 1:
+    # Run with multi_gpu_model
+    # If eager execution is enabled, only one GPU is utilized even if multiple
+    # GPUs are provided.
     if FLAGS.eager:
       tf.logging.warning(
           "{} GPUs are provided, but only one GPU is utilized as "
           "eager execution is enabled.".format(num_gpus))
     model = tf.keras.utils.multi_gpu_model(model, gpus=num_gpus)
 
+  # Adam optimizer and some other optimizers doesn't work well with
+  # distribution strategy (b/113076709)
+  # Use GradientDescentOptimizer here
+  optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
   model.compile(loss="categorical_crossentropy",
-                optimizer=tf.train.AdamOptimizer(),
-                metrics=["accuracy"])
+                optimizer=optimizer,
+                metrics=["accuracy"],
+                distribute=distribution)
 
   # Create benchmark logger for benchmark logging
   run_params = {
       "batch_size": FLAGS.batch_size,
       "synthetic_data": FLAGS.use_synthetic_data,
       "train_epochs": FLAGS.train_epochs,
-      "num_train_images": FLAGS.num_images,
-      "num_eval_images": FLAGS.num_images,
+      "num_train_images": FLAGS.num_train_images,
+      "num_eval_images": FLAGS.num_eval_images,
   }
 
   benchmark_logger = logger.get_benchmark_logger()
@@ -118,8 +135,8 @@ def run_keras_model_benchmark(_):
       epochs=FLAGS.train_epochs,
       callbacks=callbacks,
       validation_data=val_dataset,
-      steps_per_epoch=int(np.ceil(FLAGS.num_images / FLAGS.batch_size)),
-      validation_steps=int(np.ceil(FLAGS.num_images / FLAGS.batch_size))
+      steps_per_epoch=int(np.ceil(FLAGS.num_train_images / FLAGS.batch_size)),
+      validation_steps=int(np.ceil(FLAGS.num_eval_images / FLAGS.batch_size))
   )
 
   tf.logging.info("Logging the evaluation results...")
@@ -128,7 +145,7 @@ def run_keras_model_benchmark(_):
         "accuracy": history.history["val_acc"][epoch],
         "loss": history.history["val_loss"][epoch],
         tf.GraphKeys.GLOBAL_STEP: (epoch + 1) * np.ceil(
-            FLAGS.num_images/FLAGS.batch_size)
+            FLAGS.num_eval_images/FLAGS.batch_size)
     }
     benchmark_logger.log_evaluation_result(eval_results)
 
@@ -157,17 +174,29 @@ def define_keras_benchmark_flags():
           "Model to be benchmarked."))
 
   flags.DEFINE_integer(
-      name="num_images", default=1000,
+      name="num_train_images", default=1000,
+      help=flags_core.help_wrap(
+          "The number of synthetic images for training. The default value is "
+          "1000."))
+
+  flags.DEFINE_integer(
+      name="num_eval_images", default=50,
       help=flags_core.help_wrap(
-          "The number of synthetic images for training and evaluation. The "
-          "default value is 1000."))
+          "The number of synthetic images for evaluation. The default value is "
+          "50."))
 
   flags.DEFINE_boolean(
       name="eager", default=False, help=flags_core.help_wrap(
           "To enable eager execution. Note that if eager execution is enabled, "
           "only one GPU is utilized even if multiple GPUs are provided and "
           "multi_gpu_model is used."))
 
+  flags.DEFINE_boolean(
+      name="dist_strat", default=False, help=flags_core.help_wrap(
+          "To enable distribution strategy for model training and evaluation. "
+          "Number of GPUs used for distribution strategy can be set by the "
+          "argument --num_gpus."))
+
   flags.DEFINE_list(
       name="callbacks",
       default=["ExamplesPerSecondCallback", "LoggingMetricCallback"],
@@ -176,6 +205,15 @@ def define_keras_benchmark_flags():
           "callbacks. For example: `--callbacks ExamplesPerSecondCallback,"
           "LoggingMetricCallback`"))
 
+  @flags.multi_flags_validator(
+      ["eager", "dist_strat"],
+      message="Both --eager and --dist_strat were set. Only one can be "
+              "defined, as DistributionStrategy is not supported in Eager "
+              "execution currently.")
+  # pylint: disable=unused-variable
+  def _check_eager_dist_strat(flag_dict):
+    return not(flag_dict["eager"] and flag_dict["dist_strat"])
+
 
 def main(_):
   with logger.benchmark_context(FLAGS):

official/keras_application_models/dataset.py

@@ -17,8 +17,8 @@
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
 import tensorflow as tf
-
 from official.utils.misc import model_helpers  # pylint: disable=g-bad-import-order
 
 # Default values for dataset.
@@ -29,7 +29,7 @@
 def _get_default_image_size(model):
   """Provide default image size for each model."""
   image_size = (224, 224)
-  if model in ["inception", "xception", "inceptionresnet"]:
+  if model in ["inceptionv3", "xception", "inceptionresnetv2"]:
     image_size = (299, 299)
   elif model in ["nasnetlarge"]:
     image_size = (331, 331)
@@ -42,8 +42,33 @@ def generate_synthetic_input_dataset(model, batch_size):
   image_shape = (batch_size,) + image_size + (_NUM_CHANNELS,)
   label_shape = (batch_size, _NUM_CLASSES)
 
-  return model_helpers.generate_synthetic_data(
+  dataset = model_helpers.generate_synthetic_data(
       input_shape=tf.TensorShape(image_shape),
-      input_dtype=tf.float32,
       label_shape=tf.TensorShape(label_shape),
-      label_dtype=tf.float32)
+  )
+  return dataset
+
+
+class Cifar10Dataset(object):
+  """CIFAR10 dataset, including train and test set.
+
+  Each sample consists of a 32x32 color image, and label is from 10 classes.
+  """
+
+  def __init__(self, batch_size):
+    """Initializes train/test datasets.
+
+    Args:
+      batch_size: int, the number of batch size.
+    """
+    self.input_shape = (32, 32, 3)
+    self.num_classes = 10
+    (x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()
+    x_train, x_test = x_train / 255.0, x_test / 255.0
+    y_train, y_test = y_train.astype(np.int64), y_test.astype(np.int64)
+    y_train = tf.keras.utils.to_categorical(y_train, self.num_classes)
+    y_test = tf.keras.utils.to_categorical(y_test, self.num_classes)
+    self.train_dataset = tf.data.Dataset.from_tensor_slices(
+        (x_train, y_train)).shuffle(2000).batch(batch_size).repeat()
+    self.test_dataset = tf.data.Dataset.from_tensor_slices(
+        (x_test, y_test)).shuffle(2000).batch(batch_size).repeat()