Generate _toc.yaml and make cell_id generation deterministic (keras-t…

…eam#264) * Generate _toc.yaml * changes * changes * changes * Add PyYAML * Delete * Add toc * changes * Hash the content with cell_count * Use cell_id variable * Changes
Akora-IngDKB · Sep 25, 2020 · b1fa63c · b1fa63c
1 parent 6a4098c
commit b1fa63c
Show file tree

Hide file tree

Showing 16 changed files with 716 additions and 697 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -7,3 +7,4 @@ sphinx~=3.0.3
 black==19.10b0
 pathlib
 tf-nightly~=2.3.0.dev20200610
+PyYAML
diff --git a/scripts/generate_tf_guides.py b/scripts/generate_tf_guides.py
@@ -2,26 +2,53 @@
 import tutobooks
 import copy
 import json
-import random
+import hashlib
 import string
 import re
+import yaml
 
+# The order of CONFIG is also used to generate the _toc.yaml for tensorflow.org.
 CONFIG = [
+    {
+        "title": "The Sequential model",
+        "source_name": "sequential_model",
+        "target_name": "sequential_model",
+    },
     {
         "title": "The Functional API",
         "source_name": "functional_api",
         "target_name": "functional",
     },
     {
-        "title": "Training & evaluation with the built-in methods",
+        "title": "Training and evaluation with the built-in methods",
         "source_name": "training_with_built_in_methods",
         "target_name": "train_and_evaluate",
     },
     {
-        "title": "Making new Layers & Models via subclassing",
+        "title": "Making new Layers and Models via subclassing",
         "source_name": "making_new_layers_and_models_via_subclassing",
         "target_name": "custom_layers_and_models",
     },
+    {
+        "title": "Save and load Keras models",
+        "source_name": "serialization_and_saving",
+        "target_name": "save_and_serialize",
+    },
+    {
+        "title": "Working with preprocessing layers",
+        "source_name": "preprocessing_layers",
+        "target_name": "preprocessing_layers",
+    },
+    {
+        "title": "Customize what happens in Model.fit",
+        "source_name": "customizing_what_happens_in_fit",
+        "target_name": "customizing_what_happens_in_fit",
+    },
+    {
+        "title": "Writing a training loop from scratch",
+        "source_name": "writing_a_training_loop_from_scratch",
+        "target_name": "writing_a_training_loop_from_scratch",
+    },
     {
         "title": "Recurrent Neural Networks (RNN) with Keras",
         "source_name": "working_with_rnns",
@@ -32,46 +59,21 @@
         "source_name": "understanding_masking_and_padding",
         "target_name": "masking_and_padding",
     },
-    {
-        "title": "Save and load Keras models",
-        "source_name": "serialization_and_saving",
-        "target_name": "save_and_serialize",
-    },
     {
         "title": "Writing your own callbacks",
         "source_name": "writing_your_own_callbacks",
         "target_name": "custom_callback",
     },
     {
-        "title": "Writing a training loop from scratch",
-        "source_name": "writing_a_training_loop_from_scratch",
-        "target_name": "writing_a_training_loop_from_scratch",
-    },
-    {
-        "title": "Transfer learning & fine-tuning",
+        "title": "Transfer learning and fine-tuning",
         "source_name": "transfer_learning",
         "target_name": "transfer_learning",
     },
-    {
-        "title": "The Sequential model",
-        "source_name": "sequential_model",
-        "target_name": "sequential_model",
-    },
-    {
-        "title": "Customizing what happens in `fit()`",
-        "source_name": "customizing_what_happens_in_fit",
-        "target_name": "customizing_what_happens_in_fit",
-    },
     {
         "title": "Training Keras models with TensorFlow Cloud",
         "source_name": "training_keras_models_on_cloud",
         "target_name": "training_keras_models_on_cloud",
     },
-    {
-        "title": "Working with preprocessing layers",
-        "source_name": "preprocessing_layers",
-        "target_name": "preprocessing_layers",
-    },
 ]
 
 
@@ -205,8 +207,13 @@ def generate_single_tf_guide(source_dir, target_dir, title, source_name, target_
         buttons["source"][i] = buttons["source"][i].replace("SOURCE_NAME", source_name)
     header_cells.append(buttons)
     cells = header_cells + cells
+
+    cell_count = 0
     for cell in cells:
-        cell["metadata"]["id"] = random_id()
+        cell_count += 1
+        str_to_hash = f"{cell_count} {cell['source']}"
+        cell_id = hashlib.sha256(str_to_hash.encode('utf-8')).hexdigest()
+        cell["metadata"]["id"] = cell_id[:12]
 
     notebook = {}
     for key in TF_IPYNB_BASE.keys():
@@ -259,14 +266,26 @@ def generate_single_tf_guide(source_dir, target_dir, title, source_name, target_
     f.close()
 
 
-def random_id():
-    length = 12
-    letters = string.ascii_lowercase + string.ascii_uppercase + "0123456789"
-    return "".join(random.choice(letters) for i in range(length))
+def generate_toc(target_dir):
+    target_dir = Path(target_dir)
+
+    toc = []
+    for config in CONFIG:
+        toc.append(
+            {
+                "title": config["title"],
+                "path": str(Path("/guide/keras") / config["target_name"]),
+            }
+        )
+    toc_dict = {"toc": toc}
+
+    with open(str(target_dir / "_toc.yaml"), "w") as toc_file:
+        yaml.dump(toc_dict, toc_file, sort_keys=False)
 
 
 def generate_tf_guides():
-    random.seed(1337)
+    generate_toc(target_dir="../tf")
+
     for entry in CONFIG:
         generate_single_tf_guide(
             source_dir="../guides/ipynb/",

diff --git a/tf/_toc.yaml b/tf/_toc.yaml
@@ -1,28 +1,27 @@
 toc:
-- title: "Sequential API"
+- title: The Sequential model
   path: /guide/keras/sequential_model
-- title: "Keras functional API"
+- title: The Functional API
   path: /guide/keras/functional
-- title: "Train and evaluate"
+- title: Training and evaluation with the built-in methods
   path: /guide/keras/train_and_evaluate
-- title: "Write custom layers and models"
+- title: Making new Layers and Models via subclassing
   path: /guide/keras/custom_layers_and_models
-- title: "Save and serialize models"
+- title: Save and load Keras models
   path: /guide/keras/save_and_serialize
-- title: "Working with preprocessing layers"
+- title: Working with preprocessing layers
   path: /guide/keras/preprocessing_layers
-- title: "Customizing what happens in fit"
+- title: Customize what happens in Model.fit
   path: /guide/keras/customizing_what_happens_in_fit
-- title: "Writing a training loop from scratch"
+- title: Writing a training loop from scratch
   path: /guide/keras/writing_a_training_loop_from_scratch
-- title: "Keras Recurrent Neural Networks"
+- title: Recurrent Neural Networks (RNN) with Keras
   path: /guide/keras/rnn
-- title: "Masking and padding"
+- title: Masking and padding with Keras
   path: /guide/keras/masking_and_padding
-- title: "Write custom callbacks"
+- title: Writing your own callbacks
   path: /guide/keras/custom_callback
-- title: "Transfer learning"
+- title: Transfer learning and fine-tuning
   path: /guide/keras/transfer_learning
-- title: "Training Keras models with TensorFlow Cloud"
+- title: Training Keras models with TensorFlow Cloud
   path: /guide/keras/training_keras_models_on_cloud
-  status: new
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,3 +7,4 @@ sphinx~=3.0.3 @@
     black==19.10b0
     pathlib
     tf-nightly~=2.3.0.dev20200610
+    PyYAML